#!/bin/bash

set -o nounset
set -o errexit
set -o pipefail

trap 'FRC=$?; createArchMigrationJunit; debug' EXIT TERM

# Print cv, failed node, co, mcp information for debug purpose
function debug() {
    if (( FRC != 0 )); then
        echo -e "oc get clusterversion/version -oyaml\n$(oc get clusterversion/version -oyaml)"
        echo -e "Describing abnormal nodes...\n"
        oc get node --no-headers | awk '$2 != "Ready" {print $1}' | while read node; do echo -e "\n#####oc describe node ${node}#####\n$(oc describe node ${node})"; done
        echo -e "Describing abnormal operators...\n"
        oc get co --no-headers | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}' | while read co; do echo -e "\n#####oc describe co ${co}#####\n$(oc describe co ${co})"; done
        echo -e "Describing abnormal mcp...\n"
        oc get machineconfigpools --no-headers | awk '$3 != "True" || $4 != "False" || $5 != "False" {print $1}' | while read mcp; do echo -e "\n#####oc describe mcp ${mcp}#####\n$(oc describe mcp ${mcp})"; done
    fi
    echo -e "oc get co\n$(oc get co)"
    echo -e "oc get machineconfigpools\n$(oc get machineconfigpools)"
}

# Generate the Junit for migration
function createArchMigrationJunit() {
    echo "Generating the Junit for arch migration"
    if (( FRC == 0 )); then
      cat >"${ARTIFACT_DIR}/junit_arch_migration.xml" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="cluster arch migration" tests="1" failures="0">
  <testcase classname="cluster arch migration" name="arch migration should succeed"/>
</testsuite>
EOF
    else
      cat >"${ARTIFACT_DIR}/junit_arch_migration.xml" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="cluster arch_migration" tests="1" failures="1">
  <testcase classname="cluster arch migration" name="arch migration should succeed">
    <failure message="">openshift cluster arch migration failed</failure>
  </testcase>
</testsuite>
EOF
    fi
}

function run_command_oc() {
    local try=0 max=40 ret_val

    if [[ "$#" -lt 1 ]]; then
        return 0
    fi

    while (( try < max )); do
        if ret_val=$(oc "$@" 2>&1); then
            break
        fi
        (( try += 1 ))
        sleep 3
    done

    if (( try == max )); then
        echo >&2 "Run:[oc $*]"
        echo >&2 "Get:[$ret_val]"
        return 255
    fi

    echo "${ret_val}"
}

# Wait upgrade to start, https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-25473
# Following conditions will be used to determine if an upgrade is started
#   Progressing=True
#   ReleaseAccepted=True
#   Upgradeable=False (for ocp 4.19 and later)
# If not all above conditions check passed, we print a message and returns 1
function wait_upgrade_start(){
    local retry=0 output
    echo "Wait for the upgrade to start"
    while [[ retry -lt 10 ]]; do
        output="$(oc get clusterversion version -ojson)"
        if [[ "$(echo "$output" | jq -r '.status.conditions[] | select(.type == "Progressing").status')" == "True" ]] \
            && [[ "$(echo "$output" | jq -r '.status.conditions[] | select(.type == "ReleaseAccepted").status')" == "True" ]];  then
            
            if [[ "$SOURCE_MINOR_VERSION" -gt 18 ]]; then
                if [[ "$(echo "$output" | jq -r '.status.conditions[] | select(.type == "Upgradeable").status')" != "False" ]]; then
                    echo "The upgrade has not started yet, retry ${retry}"
                    retry=$(( retry+1 ))
                    sleep 1m
                    continue
                fi
            fi

            echo "Upgrade is processing"
            return 0
        fi
        retry=$(( retry+1 ))
        sleep 1m
    done
    echo "After 10 minutes, the upgrade has not started yet"
    return 1
}

function check_clusteroperators() {
    local tmp_ret=0 tmp_clusteroperator input column last_column_name tmp_clusteroperator_1 rc null_version unavailable_operator degraded_operator

    echo "Make sure every operator do not report empty column"
    tmp_clusteroperator=$(mktemp /tmp/health_check-script.XXXXXX)
    input="${tmp_clusteroperator}"
    ${OC} get clusteroperator >"${tmp_clusteroperator}"
    column=$(head -n 1 "${tmp_clusteroperator}" | awk '{print NF}')
    last_column_name=$(head -n 1 "${tmp_clusteroperator}" | awk '{print $NF}')
    if [[ ${last_column_name} == "MESSAGE" ]]; then
        (( column -= 1 ))
        tmp_clusteroperator_1=$(mktemp /tmp/health_check-script.XXXXXX)
        awk -v end=${column} '{for(i=1;i<=end;i++) printf $i"\t"; print ""}' "${tmp_clusteroperator}" > "${tmp_clusteroperator_1}"
        input="${tmp_clusteroperator_1}"
    fi

    while IFS= read -r line
    do
        rc=$(echo "${line}" | awk '{print NF}')
        if (( rc != column )); then
            echo >&2 "The following line have empty column"
            echo >&2 "${line}"
            (( tmp_ret += 1 ))
        fi
    done < "${input}"
    rm -f "${tmp_clusteroperator}"

    echo "Make sure every operator column reports version"
    if null_version=$(${OC} get clusteroperator -o json | jq '.items[] | select(.status.versions == null) | .metadata.name') && [[ ${null_version} != "" ]]; then
        echo >&2 "Null Version: ${null_version}"
        (( tmp_ret += 1 ))
    fi

    echo "Make sure every operator reports correct version"
    if incorrect_version=$(${OC} get clusteroperator --no-headers | awk -v var="${SOURCE_VERSION}" '$2 != var') && [[ ${incorrect_version} != "" ]]; then
        echo >&2 "Incorrect CO Version: ${incorrect_version}"
        (( tmp_ret += 1 ))
    fi

    echo "Make sure every operator's AVAILABLE column is True"
    if unavailable_operator=$(${OC} get clusteroperator | awk '$3 == "False"' | grep "False"); then
        echo >&2 "Some operator's AVAILABLE is False"
        echo >&2 "$unavailable_operator"
        (( tmp_ret += 1 ))
    fi
    if ${OC} get clusteroperator -o json | jq '.items[].status.conditions[] | select(.type == "Available") | .status' | grep -iv "True"; then
        echo >&2 "Some operators are unavailable, pls run 'oc get clusteroperator -o json' to check"
        (( tmp_ret += 1 ))
    fi

    echo "Make sure every operator's PROGRESSING column is False"
    if progressing_operator=$(${OC} get clusteroperator | awk '$4 == "True"' | grep "True"); then
        echo >&2 "Some operator's PROGRESSING is True"
        echo >&2 "$progressing_operator"
        (( tmp_ret += 1 ))
    fi
    if ${OC} get clusteroperator -o json | jq '.items[].status.conditions[] | select(.type == "Progressing") | .status' | grep -iv "False"; then
        echo >&2 "Some operators are Progressing, pls run 'oc get clusteroperator -o json' to check"
        (( tmp_ret += 1 ))
    fi

    echo "Make sure every operator's DEGRADED column is False"
    if degraded_operator=$(${OC} get clusteroperator | awk '$5 == "True"' | grep "True"); then
        echo >&2 "Some operator's DEGRADED is True"
        echo >&2 "$degraded_operator"
        (( tmp_ret += 1 ))
    fi
    if ${OC} get clusteroperator -o json | jq '.items[].status.conditions[] | select(.type == "Degraded") | .status'  | grep -iv 'False'; then
        echo >&2 "Some operators are Degraded, pls run 'oc get clusteroperator -o json' to check"
        (( tmp_ret += 1 ))
    fi

    return $tmp_ret
}

function wait_clusteroperators_continous_success() {
    local try=0 continous_successful_check=0 passed_criteria=3 max_retries=40
    while (( try < max_retries && continous_successful_check < passed_criteria )); do
        echo "Checking #${try}"
        if check_clusteroperators; then
            echo "Passed #${continous_successful_check}"
            (( continous_successful_check += 1 ))
        else
            echo "cluster operators are not ready yet, wait and retry..."
            continous_successful_check=0
        fi
        sleep 60
        (( try += 1 ))
    done
    if (( continous_successful_check != passed_criteria )); then
        echo >&2 "Some cluster operator does not get ready or not stable"
        echo "Debug: current CO output is:"
        oc get co
        return 1
    else
        echo "All cluster operators status check PASSED"
        return 0
    fi
}

function check_mcp() {
    local updating_mcp unhealthy_mcp tmp_output

    tmp_output=$(mktemp)
    oc get machineconfigpools -o custom-columns=NAME:metadata.name,CONFIG:spec.configuration.name,UPDATING:status.conditions[?\(@.type==\"Updating\"\)].status --no-headers > "${tmp_output}" || true
    # using the size of output to determinate if oc command is executed successfully
    if [[ -s "${tmp_output}" ]]; then
        updating_mcp=$(cat "${tmp_output}" | grep -v "False")
        if [[ -n "${updating_mcp}" ]]; then
            echo "Some mcp is updating..."
            echo "${updating_mcp}"
            return 1
        fi
    else
        echo "Did not run 'oc get machineconfigpools' successfully!"
        return 1
    fi
    # Do not check UPDATED on purpose, beause some paused mcp would not update itself until unpaused
    oc get machineconfigpools -o custom-columns=NAME:metadata.name,CONFIG:spec.configuration.name,UPDATING:status.conditions[?\(@.type==\"Updating\"\)].status,DEGRADED:status.conditions[?\(@.type==\"Degraded\"\)].status,DEGRADEDMACHINECOUNT:status.degradedMachineCount --no-headers > "${tmp_output}" || true
    # using the size of output to determinate if oc command is executed successfully
    if [[ -s "${tmp_output}" ]]; then
        unhealthy_mcp=$(cat "${tmp_output}" | grep -v "False.*False.*0")
        if [[ -n "${unhealthy_mcp}" ]]; then
            echo "Detected unhealthy mcp:"
            echo "${unhealthy_mcp}"
            echo "Real-time detected unhealthy mcp:"
            oc get machineconfigpools -o custom-columns=NAME:metadata.name,CONFIG:spec.configuration.name,UPDATING:status.conditions[?\(@.type==\"Updating\"\)].status,DEGRADED:status.conditions[?\(@.type==\"Degraded\"\)].status,DEGRADEDMACHINECOUNT:status.degradedMachineCount | grep -v "False.*False.*0"
            echo "Real-time full mcp output:"
            oc get machineconfigpools
            echo ""
            unhealthy_mcp_names=$(echo "${unhealthy_mcp}" | awk '{print $1}')
            echo "Using oc describe to check status of unhealthy mcp ..."
            for mcp_name in ${unhealthy_mcp_names}; do
              echo "Name: $mcp_name"
              oc describe mcp $mcp_name || echo "oc describe mcp $mcp_name failed"
            done
            return 2
        fi
    else
        echo "Did not run 'oc get machineconfigpools' successfully!"
        return 1
    fi
    return 0
}

function wait_mcp_continous_success() {
    local try=0 continous_successful_check=0 passed_criteria max_retries ret=0 interval=30
    num=$(oc get node --no-headers | wc -l)
    max_retries=$(expr $num \* 20 \* 60 \/ $interval) # Wait 20 minutes for each node, try 60/interval times per minutes
    passed_criteria=$(expr 5 \* 60 \/ $interval) # We consider mcp to be updated if its status is updated for 5 minutes
    local continous_degraded_check=0 degraded_criteria=5
    while (( try < max_retries && continous_successful_check < passed_criteria )); do
        echo "Checking #${try}"
        ret=0
        check_mcp || ret=$?
        if [[ "$ret" == "0" ]]; then
            continous_degraded_check=0
            echo "Passed #${continous_successful_check}"
            (( continous_successful_check += 1 ))
        elif [[ "$ret" == "1" ]]; then
            echo "Some machines are updating..."
            continous_successful_check=0
            continous_degraded_check=0
        else
            continous_successful_check=0
            echo "Some machines are degraded #${continous_degraded_check}..."
            (( continous_degraded_check += 1 ))
            if (( continous_degraded_check >= degraded_criteria )); then
                break
            fi
        fi
        echo "wait and retry..."
        sleep ${interval}
        (( try += 1 ))
    done
    if (( continous_successful_check != passed_criteria )); then
        echo >&2 "Some mcp does not get ready or not stable"
        echo "Debug: current mcp output is:"
        oc get machineconfigpools
        return 1
    else
        echo "All mcp status check PASSED"
        return 0
    fi
}

function check_node() {
    local not_ready_number try=0 interval=60 max_retries=40
    while (( try < max_retries )); do
        echo "Checking #${try}"
        not_ready_number=$(${OC} get node |grep -v STATUS | awk '$2 != "Ready"' | wc -l)
        if (( not_ready_number != 0 )); then
            sleep ${interval}
        else
            break
        fi
        (( try += 1 ))
    done
    if (( try == max_retries )); then
        echo >&2 "Timeout waiting for all nodes ready"
        return 1
    else
        echo "All nodes are ready"
        return 0
    fi
}

function check_pod() {
    echo "Show all pods status for reference/debug"
    oc get pods --all-namespaces
}

function health_check() {
    echo "Step #1: Make sure no degrated or updating mcp"
    wait_mcp_continous_success

    echo "Step #2: check all cluster operators get stable and ready"
    wait_clusteroperators_continous_success

    echo "Step #3: Make sure every machine is in 'Ready' status"
    check_node

    echo "Step #4: check all pods are in status running or complete"
    check_pod
}

# Monitor the migration status
function check_migrate_status() {
    local wait_migrate="${TIMEOUT}" out avail progress
    while (( wait_migrate > 0 )); do
        sleep 5m
        (( wait_migrate -= 5 ))
        if ! ( echo "oc get clusterversion" && oc get clusterversion ); then
            continue
        fi
        if ! out="$(oc get clusterversion --no-headers)"; then continue; fi
        avail="$(echo "${out}" | awk '{print $3}')"
        progress="$(echo "${out}" | awk '{print $4}')"
        if [[ ${avail} == "True" && ${progress} == "False" && ${out} == *"Cluster version is ${SOURCE_VERSION}" ]]; then
            if check_clusteroperators; then
                echo -e "Migrate succeed\n\n"
                return 0
            else
                echo -e "CVO reports upgrade completed, but some operators are not\n\n"
                return 1
            fi
        fi
    done
    if (( wait_migrate <= 0 )); then
        echo >&2 "Migrate timeout, exiting" && return 1
    fi

    if [[ "$(oc get clusterversion version -oyaml | yq -r '.status.history[0].state')" != "Completed" ]]; then
        echo >&2 "history state is not Completed, exiting" && return 1
    fi
}

# Check ClusterVersion has architecture="Multi"
function check_arch() {
    local msg
    msg=$(oc get clusterversion version -ojson|jq -r '.status.conditions[]|select(.type == "ReleaseAccepted")|.message')
    if [[ ${msg} == *"architecture=\"Multi\""* ]]; then
        echo "ClusterVersion architecture check PASSED"
    else
        echo >&2 "OCP-53921: ClusterVersion architecture check FAILED, exiting" && return 1
    fi
}

function switch_channel() {
    echo "Switch upgrade channel to candidate-${SOURCE_XY_VERSION}..."
    oc adm upgrade channel candidate-${SOURCE_XY_VERSION}
    ret=$(oc get clusterversion/version -ojson | jq -r '.spec.channel')
    if [[ "${ret}" != "candidate-${SOURCE_XY_VERSION}" ]]; then
        echo >&2 "Failed to switch channel, exiting" && return 1
    fi
}

function migrate() {
    echo "Arch migrating"
    cmd="oc adm upgrade --to-multi-arch"
    if [[ "${ENABLE_OTA_TEST}" == "OCP-60396" ]]; then
        cmd+=" --allow-upgrade-with-warnings"
    fi
    echo "Migrating cluster: ${cmd}"
    if ! eval "${cmd}"; then
        echo >&2 "Failed to request arch migration, exiting" && return 1
    fi
}

if [[ -f "${SHARED_DIR}/kubeconfig" ]] ; then
    export KUBECONFIG=${SHARED_DIR}/kubeconfig
fi

# Setup proxy if it's present in the shared dir
if [[ -f "${SHARED_DIR}/proxy-conf.sh" ]]; then
    # shellcheck disable=SC1091
    source "${SHARED_DIR}/proxy-conf.sh"
fi

export OC="run_command_oc"

SOURCE_VERSION="$(oc get clusterversion --no-headers | awk '{print $2}')"
export SOURCE_VERSION
SOURCE_MINOR_VERSION="$(echo "${SOURCE_VERSION}" | cut -f2 -d.)"
export SOURCE_MINOR_VERSION
SOURCE_XY_VERSION="$(echo "${SOURCE_VERSION}" | cut -f1,2 -d.)"
export SOURCE_XY_VERSION

switch_channel
migrate
if [[ "${SOURCE_MINOR_VERSION}" -gt 17 ]]; then
    wait_upgrade_start
fi
if [[ "${SOURCE_MINOR_VERSION}" -lt 18 ]]; then
    health_check
fi

check_migrate_status
check_arch
