#!/usr/bin/env bash

# SPDX-FileCopyrightText: 2025 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

set -ue
set -o pipefail

. utils/lib/common.sh

# Consts
BUCKET_REGION="us-west-2"
VPC_DEFAULT_CIDR_PREFIX="192.168."
VPC_FIRST_CIDR="192.168.248.0/21"
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) # The repository root directory
SAVE_DIR=$(realpath "${SAVE_DIR:-${ROOT_DIR}/SAVEME}")

# The following line is generated by "cat isrgrootx1.pem| base64 |tr -d '\n'""
LETSENCRYPT_isrgrootx1="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZhekNDQTFPZ0F3SUJBZ0lSQUlJUXo3RFNRT05aUkdQZ3UyT0Npd0F3RFFZSktvWklodmNOQVFFTEJRQXcKVHpFTE1Ba0dBMVVFQmhNQ1ZWTXhLVEFuQmdOVkJBb1RJRWx1ZEdWeWJtVjBJRk5sWTNWeWFYUjVJRkpsYzJWaApjbU5vSUVkeWIzVndNUlV3RXdZRFZRUURFd3hKVTFKSElGSnZiM1FnV0RFd0hoY05NVFV3TmpBME1URXdORE00CldoY05NelV3TmpBME1URXdORE00V2pCUE1Rc3dDUVlEVlFRR0V3SlZVekVwTUNjR0ExVUVDaE1nU1c1MFpYSnUKWlhRZ1UyVmpkWEpwZEhrZ1VtVnpaV0Z5WTJnZ1IzSnZkWEF4RlRBVEJnTlZCQU1UREVsVFVrY2dVbTl2ZENCWQpNVENDQWlJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dJUEFEQ0NBZ29DZ2dJQkFLM29KSFAwRkRmem01NHJWeWdjCmg3N2N0OTg0a0l4dVBPWlhvSGozZGNLaS92VnFidllBVHlqYjNtaUdiRVNUdHJGai9SUVNhNzhmMHVveG15RisKMFRNOHVrajEzWG5mczdqL0V2RWhta3ZCaW9aeGFVcG1abXlQZmp4d3Y2MHBJZ2J6NU1EbWdLN2lTNCszbVg2VQpBNS9UUjVkOG1VZ2pVK2c0cms4S2I0TXUwVWxYaklCMHR0b3YwRGlOZXdOd0lSdDE4akE4K28rdTNkcGpxK3NXClQ4S09FVXQrend2by83VjNMdlN5ZTByZ1RCSWxESENOQXltZzRWTWs3QlBaN2htL0VMTktqRCtKbzJGUjNxeUgKQjVUMFkzSHNMdUp2VzVpQjRZbGNOSGxzZHU4N2tHSjU1dHVrbWk4bXhkQVE0UTdlMlJDT0Z2dTM5NmozeCtVQwpCNWlQTmdpVjUrSTNsZzAyZFo3N0RuS3hIWnU4QS9sSkJkaUIzUVcwS3RaQjZhd0JkcFVLRDlqZjFiMFNIelV2CktCZHMwcGpCcUFsa2QyNUhON3JPckZsZWFKMS9jdGFKeFFaQktUNVpQdDBtOVNUSkVhZGFvMHhBSDBhaG1iV24KT2xGdWhqdWVmWEtuRWdWNFdlMCtVWGdWQ3dPUGpkQXZCYkkrZTBvY1MzTUZFdnpHNnVCUUUzeERrM1N6eW5UbgpqaDhCQ05BdzFGdHhOclFIdXNFd01GeEl0NEk3bUtaOVlJcWlveW1DekxxOWd3UWJvb01EUWFIV0JmRWJ3cmJ3CnFIeUdPMGFvU0NxSTNIYWFkcjhmYXFVOUdZL3JPUE5rM3NnckRRb28vL2ZiNGhWQzFDTFFKMTNoZWY0WTUzQ0kKclU3bTJZczZ4dDBuVVc3L3ZHVDFNME5QQWdNQkFBR2pRakJBTUE0R0ExVWREd0VCL3dRRUF3SUJCakFQQmdOVgpIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJSNXRGbm1lN2JsNUFGemdBaUl5QnBZOXVtYmJqQU5CZ2txCmhraUc5dzBCQVFzRkFBT0NBZ0VBVlI5WXFieXlxRkRRRExIWUdta2dKeWtJckdGMVhJcHUrSUxsYVMvVjlsWkwKdWJoekVGblRJWmQrNTB4eCs3TFNZSzA1cUF2cUZ5RldoZkZRRGxucnp1Qlo2YnJKRmUrR25ZK0VnUGJrNlpHUQozQmViWWh0RjhHYVYwbnh2d3VvNzd4L1B5OWF1Si9HcHNNaXUvWDErbXZvaUJPdi8yWC9xa1NzaXNSY09qL0tLCk5GdFkyUHdCeVZTNXVDYk1pb2d6aVV3dGhEeUMzKzZXVndXNkxMdjN4TGZIVGp1Q3ZqSElJbk56a3RIQ2dLUTUKT1JBekk0Sk1QSitHc2xXWUhiNHBob3dpbTU3aWF6dFhPb0p3VGR3Sng0bkxDZ2ROYk9oZGpzbnZ6cXZIdTdVcgpUa1hXU3RBbXpPVnl5Z2hxcFpYakZhSDNwTzNKTEYrbCsvK3NLQUl1dnRkN3UrTnhlNUFXMHdkZVJsTjhOd2RDCmpOUEVscHpWbWJVcTRKVWFnRWl1VERrSHpzeEhwRktWSzdxNCs2M1NNMU45NVIxTmJkV2hzY2RDYitaQUp6VmMKb3lpM0I0M25qVE9RNXlPZisxQ2NlV3hHMWJRVnM1WnVmcHNNbGpxNFVpMC8xbHZoK3dqQ2hQNGtxS09KMnF4cQo0Umdxc2FoRFlWdlRIOXc3alhieUxlaU5kZDhYTTJ3OVUvdDd5MEZmLzl5aTBHRTQ0WmE0ckYyTE45ZDExVFBBCm1SR3VuVUhCY25XRXZnSkJRbDluSkVpVTBac252Z2MvdWJoUGdYUlI0WHEzN1owajRyN2cxU2dFRXp3eEE1N2QKZW15UHhnY1l4bi9lUjQ0L0tKNEVCcytsVkRSM3ZleUptK2tYUTk5YjIxLytqaDVYb3MxQW5YNWlJdHJlR0NjPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCgo="

# Global variables to be updated
COMMAND=""
ENV_NAME=""
AWS_REGION="${AWS_REGION:-}"
AWS_ACCOUNT="${AWS_ACCOUNT:-}"
CUSTOMER_STATE_PREFIX="${CUSTOMER_STATE_PREFIX:-}"
BUCKET_NAME=""
PARENT_DOMAIN=""
ROOT_DOMAIN=""
ROOT_DOMAIN_HOST=""
ROUTE53=true
NEW_AWS_ACCOUNT=false
EMAIL=""
TLS_CERT=""
TLS_KEY=""
TLS_CA_CERT=""
SOCKS_PROXY="${SOCKS_PROXY:-}"
FULLCHAIN=""
CHAIN=""
PRIVKEY=""
SKIP_APPLY_LOADBALANCER=false
SKIP_APPLY_CLUSTER=false
SKIP_APPLY_VPC=false
SKIP_APPLY_ROUTE53=false
SKIP_DESTROY_LOADBALANCER=false
SKIP_DESTROY_CLUSTER=false
SKIP_DESTROY_VPC=false
SKIP_DESTROY_ROUTE53=false
SKIP_QUOTA_CHECKS="${SKIP_QUOTA_CHECKS:-false}"
AUTO_APPROVE=""
OUTPUT=""
KUBECONFIG=""
VALUES=""
VARIABLE_TFVAR=""
PROFILE_TFVAR=""
VALUES_CHANGED=""
JUMPHOST_SSHKEY=""
JUMPHOST_IP=""
VPC_CIDR=""
VPC_ID=""
SAVE_DIR_S3=""
AUTO_CERT=false
AZUREAD_CLIENTID=""
AZUREAD_USER=""
AZUREAD_PASS=""
RS_TOKEN="release-service-token-prod"  # The default value could be replaced
SM_PROXY_REGION="us-west-2"
RESET_AWS_ACCOUNT=false
AZUREAD_REFRESH_TOKEN=""
AZUREAD_TOKEN_ENDPOINT="https://registry-rs.edgeorchestration.intel.com/oauth/token"
CREATE_ROOT_DOMAIN="true"
ACCOUNT_BUCKET_PREFIX="account"
PROVISION_CONTEXT="true"
EKS_DESIRED_SIZE=3
EKS_MIN_SIZE=3
EKS_MAX_SIZE=3
EKS_NODE_TYPE="t3.2xlarge"
EKS_O11Y_DESIRED_SIZE=3
EKS_O11Y_MIN_SIZE=3
EKS_O11Y_MAX_SIZE=3
EKS_O11Y_NODE_TYPE="t3.2xlarge"
MAX_RDS_ACU=2
MIN_RDS_ACU=0.5
OVERRIDE_EKS_SIZE=false
OVERRIDE_EKS_O11Y_SIZE=false
OVERRIDE_EKS_NODE_TYPE=false
OVERRIDE_EKS_O11Y_NODE_TYPE=false
OVERRIDE_RDS_ACU=false
INTERNAL=false
AWS_ADMIN_ROLES=""
EKS_VERSION="1.32"
JUMPHOST_IP_ALLOW_LIST=""
REDUCE_NS_TTL=false
NUM_RDS_INSTANCES=1
CLUSTER_PROFILE=""
S3_PENDING=""
CUSTOMER_TAG=""
ENABLE_CACHE_REGISTRY="false"
EKS_CLUSTER_DNS_IP=""
EKS_HTTP_PROXY=""
EKS_HTTPS_PROXY=""
EKS_NO_PROXY=""
EKS_USER_SCRIPT_PRE_CLOUD_INIT=""
EKS_USER_SCRIPT_POST_CLOUD_INIT=""

OPTIONS_LIST=(
    "auto"
    "auto-cert"
    "aws-account:"
    "aws-admin-roles:"
    "azuread-clientid:"
    "azuread-pass:"
    "azuread-refresh-token:"
    "azuread-token-endpoint:"
    "azuread-user:"
    "cidr-block:"
    "customer-state-prefix:"
    "customer-tag:"
    "desired-nodes:"
    "desired-o11y-nodes:"
    "eks-cluster-dns-ip:"
    "eks-http-proxy:"
    "eks-https-proxy:"
    "eks-no-proxy:"
    "eks-user-script-pre-cloud-init-file:"
    "eks-user-script-post-cloud-init-file:"
    "email:"
    "enable-cache-registry"
    "environment:"
    "help"
    "internal"
    "jumphost-ip:"
    "jumphost-ip-allow-list:"
    "max-rds-acu:"
    "max-nodes:"
    "max-o11y-nodes:"
    "min-rds-acu:"
    "min-nodes:"
    "min-o11y-nodes:"
    "new-aws-account"
    "no-create-root-domain"
    "no-route53"
    "node-type:"
    "num-rds-instances:"
    "o11y-node-type:"
    "parent-domain:"
    "profile:"
    "reduce-ns-ttl"
    "region:"
    "reset-aws-account"
    "root-domain:"
    "rs-token:"
    "skip-apply-cluster"
    "skip-apply-loadbalancer"
    "skip-apply-route53"
    "skip-apply-vpc"
    "skip-destroy-cluster"
    "skip-destroy-loadbalancer"
    "skip-destroy-route53"
    "skip-destroy-vpc"
    "skip-quota-checks"
    "sm-proxy-region:"
    "socks-proxy:"
    "vpc-id:"
)

usage() {
        echo "Usage:"
        echo "utils/provision.sh < install | uninstall | pull-savedir | push-savedir | config | upgrade | account | update-cluster-setting > \\"
        echo "    [ --auto ] \\"
        echo "    [ --auto-cert ] \\"
        echo "    --aws-account {AWS ACCOUNT NUMBER} \\"
        echo "    [ --aws-admin-roles {AWS ADMIN ROLES} ] \\"
        echo "    [ --azuread-clientid {AZUREAD_CLIENTID} ] \\"
        echo "    [ --azuread-pass {AZUREAD_PASS} ] \\"
        echo "    [ --azuread-user {AZUREAD_USER} ] \\"
        echo "    [ --azuread-refresh-token {AZUREAD_REFRESH_TOKEN} ] \\"
        echo "    [ --azuread-token-endpoint {AZUREAD_TOKEN_ENDPOINT} ] \\"
        echo "    [ --cidr-block {CIDR BLOCK} ] \\"
        echo "    --customer-state-prefix {CUSTOMER STATE PREFIX}  \\"
        echo "    --customer-tag {CUSTOMER TAG} \\"
        echo "    [ --desired-nodes {NUMBER OF NODES} ] \\"
        echo "    [ --desired-o11y-nodes {NUMBER OF NODES} ] \\"
        echo "    [--eks-cluster-dns-ip {DNS IP}] \\"
        echo "    [--eks-http-proxy {HTTP PROXY}] \\"
        echo "    [--eks-https-proxy {HTTPS PROXY}] \\"
        echo "    [--eks-no-proxy {NO PROXY}] \\"
        echo "    [--eks-user-script-pre-cloud-init-file {PATH}] \\"
        echo "    [--eks-user-script-post-cloud-init-file {PATH}] \\"
        echo "    --email {ADMIN EMAIL} \\"
        echo "    [ --enable-cache-registry ] \\"
        echo "    [ --environment {ENVIRONMENT NAME} ] \\"
        echo "    [ --internal ] \\"
        echo "    [ --jumphost-ip {EXISTING JUMPHOST IP ADDRESS} ] \\"
        echo "    [ --jumphost-ip-allow-list {COMMA SEPARATE IP SUBNET LIST} ] \\"
        echo "    [ --max-rds-acu {NUMBER OF ACU} ] \\"
        echo "    [ --max-nodes {NUMBER OF NODES} ] \\"
        echo "    [ --max-o11y-nodes {NUMBER OF NODES} ] \\"
        echo "    [ --min-rds-acu {NUMBER OF ACU} ] \\"
        echo "    [ --min-nodes {NUMBER OF NODES} ] \\"
        echo "    [ --min-o11y-nodes {NUMBER OF NODES} ] \\"
        echo "    [ --new-aws-account ] \\"
        echo "    [ --no-create-root-domain ] \\"
        echo "    [ --no-route53 ] \\"
        echo "    [ --node-type {Node type} ] \\"
        echo "    [ --num-rds-instance {Number of RDS instances} ] \\"
        echo "    [ --o11y-node-type {Observability node type} ] \\"
        echo "    [ --parent-domain {PARENT DOMAIN} ] \\"
        echo "    [ --profile {CLUSTER PROFILE TO DEPLOY} ] \\"
        echo "    [ --reduce-ns-ttl ] \\"
        echo "    --region {AWS REGION} \\"
        echo "    --reset-aws-account \\"
        echo "    --root-domain {ROOT DOMAIN} \\"
        echo "    [ --rs-token {RELEASE SERVER TOKEN NAME} ] \\"
        echo "    [ --skip-apply-cluster ] \\"
        echo "    [ --skip-apply-loadbalancer ]  \\"
        echo "    [ --skip-apply-route53 ] \\"
        echo "    [ --skip-apply-vpc ] \\"
        echo "    [ --skip-destroy-loadbalancer ]  \\"
        echo "    [ --skip-destroy-cluster ] \\"
        echo "    [ --skip-destroy-vpc ] \\"
        echo "    [ --skip-destroy-route53 ] \\"
        echo "    [ --skip-quota-checks ] \\"
        echo "    [ --sm-proxy-region ] \\"
        echo "    [ --socks-proxy {SOCKS PROXY} ] \\"
        echo "    [ --vpc-id ] {EXISTING VPC ID}"
        echo ""
        echo "Example:"
        echo "    To show the usage:"
        echo "        utils/provision.sh install --help  # Show the usage"
        echo "    To create the whole environment named customer1-env1:"
        echo "        utils/provision.sh install --aws-account '1234567890' --customer-state-prefix customer1-bucket --environment customer1-env1 --parent-domain mydomain.com --region us-east-1 --email myname@abc.mydomain"
        echo "    To create the whole environment named customer1-env1 with 1k edge node profile:"
        echo "        utils/provision.sh install --aws-account '1234567890' --customer-state-prefix customer1-bucket --environment customer1-env1 --parent-domain mydomain.com --region us-east-1 --email myname@abc.mydomain --profile 1kne"
        echo "    To initialize new AWS account:"
        echo "        utils/provision.sh account --new-aws-account --aws-account "1234567890" --customer-state-prefix customer1-bucket --region us-east-1 --azuread-token-endpoint https://registry-rs.edgeorchestration.intel.com/oauth/token --azuread-refresh-token 'XXXXXX'"
        echo "    To create the environment named customer1-env1 without route53 and certificate:"
        echo "        utils/provision.sh install --aws-account '1234567890' --customer-state-prefix customer1-bucket --no-route53 --environment customer1-env1 --region us-east-1"
        echo "    To destroy the environment named customer1-env1:"
        echo "        utils/provision.sh uninstall --aws-account '1234567890' --customer-state-prefix customer1-bucket --environment customer1-env1 --parent-domain mydomain.com --region us-west-2 --email myname@abc.mydomain"
        echo "    To destroy the environment named customer1-env1 which does not have any load balancer installed:"
        echo "        utils/provision.sh uninstall --aws-account '1234567890' --customer-state-prefix customer1-bucket --environment customer1-env1 --parent-domain mydomain.com --region us-west-2 --email myname@abc.mydomain --skip-destroy-loadbalancer"
}

parse_params() {
    OPTIONS_PARAM=$(IFS=,; echo "${OPTIONS_LIST[*]}")
    if ! options=$(getopt -o a:c:e:hMnp:r:s: -l "$OPTIONS_PARAM" -- "$@"); then
        # something went wrong, getopt will put out an error message for us
        usage
        exit 1
    fi

    set -- $options

    while [ $# -gt 0 ]
    do
        case $1 in
            --auto) export AUTO_APPROVE="true";;
            --auto-cert) export AUTO_CERT=true;;
            -a|--aws-account) AWS_ACCOUNT=$(eval echo $2); shift;;
            --aws-admin-roles) AWS_ADMIN_ROLES=$(eval echo $2); shift;;
            --azuread-clientid) AZUREAD_CLIENTID=$(eval echo $2); shift;;
            --azuread-pass) AZUREAD_PASS=$(eval echo $2); shift;;
            --azuread-refresh-token) AZUREAD_REFRESH_TOKEN=$(eval echo $2); shift;;
            --azuread-token-endpoint) AZUREAD_TOKEN_ENDPOINT=$(eval echo $2); shift;;
            --azuread-user) AZUREAD_USER=$(eval echo $2); shift;;
            --cidr-block) VPC_CIDR=$(eval echo $2); shift;;
            -c|--customer-state-prefix) CUSTOMER_STATE_PREFIX=$(eval echo $2); shift;;
            --customer-tag) CUSTOMER_TAG=$(eval echo $2); shift;;
            --desired-nodes) EKS_DESIRED_SIZE=$(eval echo $2); OVERRIDE_EKS_SIZE=true; shift;;
            --desired-o11y-nodes) EKS_O11Y_DESIRED_SIZE=$(eval echo $2); OVERRIDE_EKS_O11Y_SIZE=true; shift;;
            --eks-cluster-dns-ip) EKS_CLUSTER_DNS_IP=$(eval echo $2); shift;;
            --eks-http-proxy) EKS_HTTP_PROXY=$(eval echo $2); shift;;
            --eks-https-proxy) EKS_HTTPS_PROXY=$(eval echo $2); shift;;
            --eks-no-proxy) EKS_NO_PROXY=$(eval echo $2); shift;;
            --eks-user-script-pre-cloud-init-file) EKS_USER_SCRIPT_PRE_CLOUD_INIT=$(eval echo $2); shift;;
            --eks-user-script-post-cloud-init-file) EKS_USER_SCRIPT_POST_CLOUD_INIT=$(eval echo $2); shift;;
            --enable-cache-registry) ENABLE_CACHE_REGISTRY="true";;
            -e|--environment) ENV_NAME=$(eval echo $2); shift;;
            -h|--help) usage; exit;;
            -m|--email) EMAIL=$(eval echo $2); shift;;
            --internal) INTERNAL=true;;
            --jumphost-ip) JUMPHOST_IP=$(eval echo $2); shift;;
            --jumphost-ip-allow-list) JUMPHOST_IP_ALLOW_LIST=$(eval echo $2); shift;;
            --max-rds-acu) MAX_RDS_ACU=$(eval echo $2); OVERRIDE_RDS_ACU=true; shift;;
            --max-nodes) EKS_MAX_SIZE=$(eval echo $2); OVERRIDE_EKS_SIZE=true; shift;;
            --max-o11y-nodes) EKS_O11Y_MAX_SIZE=$(eval echo $2); OVERRIDE_EKS_O11Y_SIZE=true; shift;;
            --min-rds-acu) MIN_RDS_ACU=$(eval echo $2); OVERRIDE_RDS_ACU=true; shift;;
            --min-nodes) EKS_MIN_SIZE=$(eval echo $2); OVERRIDE_EKS_SIZE=true; shift;;
            --min-o11y-nodes) EKS_O11Y_MIN_SIZE=$(eval echo $2); OVERRIDE_EKS_O11Y_SIZE=true; shift;;
            -n|--new-aws-account) NEW_AWS_ACCOUNT=true;;
            --no-create-root-domain) CREATE_ROOT_DOMAIN=false;;
            --no-route53) ROUTE53=false;;
            --num-rds-instances) NUM_RDS_INSTANCES=$(eval echo $2); shift;;
            --node-type) EKS_NODE_TYPE=$(eval echo $2); OVERRIDE_EKS_NODE_TYPE=true; shift;;
            --o11y-node-type) EKS_O11Y_NODE_TYPE=$(eval echo $2); OVERRIDE_EKS_O11Y_NODE_TYPE=true; shift;;
            -p|--parent-domain) PARENT_DOMAIN=$(eval echo $2); shift;;
            --profile) CLUSTER_PROFILE=$(eval echo $2); shift;;
            -r|--region) AWS_REGION=$(eval echo $2); shift;;
            --reduce-ns-ttl) REDUCE_NS_TTL=true;;
            --reset-aws-account) RESET_AWS_ACCOUNT=true;;
            --root-domain) ROOT_DOMAIN=$(eval echo $2); shift;;
            --rs-token) RS_TOKEN=$(eval echo $2); shift;;
            --skip-apply-loadbalancer) SKIP_APPLY_LOADBALANCER=true;;
            --skip-apply-cluster) SKIP_APPLY_CLUSTER=true;;
            --skip-apply-vpc) SKIP_APPLY_VPC=true;;
            --skip-apply-route53) SKIP_APPLY_ROUTE53=true;;
            --skip-destroy-loadbalancer) SKIP_DESTROY_LOADBALANCER=true;;
            --skip-destroy-cluster) SKIP_DESTROY_CLUSTER=true;;
            --skip-destroy-vpc) SKIP_DESTROY_VPC=true;;
            --skip-destroy-route53) SKIP_DESTROY_ROUTE53=true;;
            --skip-quota-checks) SKIP_QUOTA_CHECKS=true;;
            --sm-proxy-region) SM_PROXY_REGION=$(eval echo $2); shift;;
            -s|--socks-proxy) SOCKS_PROXY=$(eval echo $2); shift;;
            --vpc-id) VPC_ID=$(eval echo $2); shift;;
            (--) shift; break;;
            (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
            (*) break;;
        esac
        shift
    done

    error_var_blank AWS_ACCOUNT
    export AWS_ACCOUNT=$AWS_ACCOUNT
    echo AWS_ACCOUNT=${AWS_ACCOUNT} > ~/.env
    echo AWS_REGION=${AWS_REGION} >> ~/.env
    echo CUSTOMER_STATE_PREFIX=${CUSTOMER_STATE_PREFIX} >> ~/.env

    if [[ -n "$JUMPHOST_IP" ]]; then
        echo JUMPHOST_IP=${JUMPHOST_IP} >> ~/.env
    fi
    if [[ -n "$VPC_CIDR" ]]; then
        echo VPC_CIDR=${VPC_CIDR} >> ~/.env
    fi

    case $COMMAND in
        account)
            if ( $NEW_AWS_ACCOUNT && $RESET_AWS_ACCOUNT ) || ( ! $NEW_AWS_ACCOUNT && ! $RESET_AWS_ACCOUNT ); then
                echo "Error: Either --new-aws-account or --reset-aws-account needs to be set and only one can be set."
                exit 1
            fi
            if $RESET_AWS_ACCOUNT && [[ -z "$RS_TOKEN" ]]; then
                echo "Error: missing some parameters since --reset-aws-account is set."
                usage
                exit 1
            fi
            ;;
        install|upgrade)
            if $SKIP_APPLY_VPC; then
                if [[ -z "$VPC_ID" ]] || [[ -z "$VPC_CIDR" ]] || [[ -z "$JUMPHOST_IP" ]]; then
                    echo "Error: --vpc_id, --cidr-block and --jumphost-ip must be specified since --skip-apply-vpc is specified."
                    exit 1
                fi
            fi
            ;;
        *)
            for v in CUSTOMER_STATE_PREFIX AWS_REGION ENV_NAME; do
                error_var_blank "$v"
            done

            if $NEW_AWS_ACCOUNT || $RESET_AWS_ACCOUNT; then
                echo "Error: The options of --new-aws-account and --reset-aws-account have been moved under the sub-command of 'account'. Please use the command '$0 account [ --new-aws-account | --reset-aws-account ] ...'."
                exit 1
            fi
            if ( [[ -z "$ROOT_DOMAIN" ]] && [[ -z "$PARENT_DOMAIN" ]] ) || ( [[ -n "$ROOT_DOMAIN" ]] && [[ -n "$PARENT_DOMAIN" ]] ); then
                echo "Error: Either --root-domain or --parent-domain must be set."
                exit 1
            fi
            if ( [[ -n "$PARENT_DOMAIN" ]] && [[ -n "$ROOT_DOMAIN" ]] ) || ( [[ -z "$PARENT_DOMAIN" ]] && [[ -z "$ROOT_DOMAIN" ]] ); then
                echo "Error: Either --parent-domain or --root-domain needs to be set and only one can be set."
                exit 1
            fi
            ;;
    esac

    if [[ $COMMAND == "account" ]]; then
        BUCKET_NAME="${AWS_ACCOUNT}-${ACCOUNT_BUCKET_PREFIX}"
        ENV_NAME="${AWS_ACCOUNT}"
    else
        BUCKET_NAME="${AWS_ACCOUNT}-${CUSTOMER_STATE_PREFIX}"
    fi
    export BUCKET_NAME=$BUCKET_NAME
    echo BUCKET_NAME=${BUCKET_NAME} >> ~/.env

    FULLCHAIN="fullchain-${AWS_ACCOUNT}-${ENV_NAME}.pem"
    CHAIN="chain-${AWS_ACCOUNT}-${ENV_NAME}.pem"
    PRIVKEY="privkey-${AWS_ACCOUNT}-${ENV_NAME}.pem"
    OUTPUT="output-${AWS_ACCOUNT}-${ENV_NAME}.json"
    VPCSTATE="${BUCKET_NAME}-${AWS_REGION}-vpc-${ENV_NAME}.json"
    KUBECONFIG="${SAVE_DIR}/kube-config-${AWS_ACCOUNT}-${ENV_NAME}"
    JUMPHOST_SSHKEY="jumphost_sshkey_${ENV_NAME}"
    VALUES="${AWS_ACCOUNT}-${ENV_NAME}-values.sh"
    VARIABLE_TFVAR="${AWS_ACCOUNT}-${ENV_NAME}-values.tfvar"
    PROFILE_TFVAR="${AWS_ACCOUNT}-${ENV_NAME}-profile.tfvar"
    VALUES_CHANGED=".${AWS_ACCOUNT}-${ENV_NAME}-valueschanged"
    SAVE_DIR_S3="${ENV_NAME}-SAVEME"
    S3_PENDING="${AWS_ACCOUNT}-${ENV_NAME}-pending"
    [[ -z "$ROOT_DOMAIN" ]] && ROOT_DOMAIN="${ENV_NAME}.${PARENT_DOMAIN}"

    if [[ -n "$ROOT_DOMAIN" ]]; then
        PARENT_DOMAIN="${ROOT_DOMAIN#*.}"
        ROOT_DOMAIN_HOST="${ROOT_DOMAIN%%.*}"
    else
    # --parent-domain
        ROOT_DOMAIN="${ENV_NAME}.${PARENT_DOMAIN}"
        ROOT_DOMAIN_HOST="${ENV_NAME}"
    fi

    echo CLUSTER_FQDN=${ROOT_DOMAIN} >> ~/.env
    echo ADMIN_EMAIL=${EMAIL} >> ~/.env
    export AWS_DEFAULT_REGION=$AWS_REGION

    if [[ $COMMAND != "account" ]]; then
        if [[ -z "$VPC_CIDR" ]]; then
            set_vpc_cidr
        else
            if check_vpc_exist ; then
                echo "Error: --cidr-block should not be specified because the VPC named $ENV_NAME has existed."
                exit 1
            fi
        fi
        echo "Info: Use CIDR block $VPC_CIDR."
        if ! $SKIP_APPLY_VPC && ! validate_cidr_block $VPC_CIDR; then
            echo "Error: The CIDR block setting is incorrect."
            exit 1
        fi
    fi

    if [[ -n "$VPC_ID" ]]; then
        if [[ ! -f "$SAVE_DIR/$JUMPHOST_SSHKEY" ]]; then
            echo "Error: Cannot find jumphost SSH private key file $SAVE_DIR/$JUMPHOST_SSHKEY. It must exist since --vpc-id is specify."
            exit 1
        fi
    fi

    if [[ -z "$AWS_ADMIN_ROLES" ]]; then
        AWS_ADMIN_ROLES=$(aws_admin_role $AWS_ACCOUNT)
        if [[ -z "$AWS_ADMIN_ROLES" ]]; then
            echo "Error: Missing required --aws-admin-roles value."
            exit 1
        fi
    else
        AWS_ADMIN_ROLES=\"$(echo $AWS_ADMIN_ROLES | sed -e 's/,/\",\"/g')\"
    fi

    SESSION_ACCOUNT=$(get_session_aws_account)
    if [[ "$AWS_ACCOUNT" != "$SESSION_ACCOUNT" ]]; then
        if [[ -z "$SESSION_ACCOUNT" ]]; then
            echo "Error: AWS credentials missing or expired. Please refresh AWS credentials to proceed."
        else
            echo "Error: Mismatched AWS session credentials. Current login session account doesn't match deployment account."
        fi
        exit 1
    fi

    if ! echo $NUM_RDS_INSTANCES | grep -q -P "^[0-9]+$" || [[ $NUM_RDS_INSTANCES -lt 1 || $NUM_RDS_INSTANCES -gt 15 ]]; then
        echo "Error: Number of RDS instances must be between 1 and 15."
        exit 1
    fi

    if [ -n "$EKS_USER_SCRIPT_PRE_CLOUD_INIT" ]; then
        if [ ! -f "$EKS_USER_SCRIPT_PRE_CLOUD_INIT" ]; then
            echo "Error: The file $EKS_USER_SCRIPT_PRE_CLOUD_INIT does not exist."
            exit 1
        fi
        EKS_USER_SCRIPT_PRE_CLOUD_INIT=$(cat $EKS_USER_SCRIPT_PRE_CLOUD_INIT)
    fi

    if [ -n "$EKS_USER_SCRIPT_POST_CLOUD_INIT" ]; then
        if [ ! -f "$EKS_USER_SCRIPT_POST_CLOUD_INIT" ]; then
            echo "Error: The file $EKS_USER_SCRIPT_POST_CLOUD_INIT does not exist."
            exit 1
        fi
        EKS_USER_SCRIPT_POST_CLOUD_INIT=$(cat $EKS_USER_SCRIPT_POST_CLOUD_INIT)
    fi

    check_proxy_settings
}

jumphost_ami() {
    aws ssm get-parameters --names /aws/service/canonical/ubuntu/server/23.04/stable/current/amd64/hvm/ebs-gp2/ami-id --region $AWS_REGION --query "Parameters[0].Value" --output text
}

eksnode126_ami() {
    aws ssm get-parameter --name /aws/service/eks/optimized-ami/1.26/amazon-linux-2/recommended/image_id --region $AWS_REGION --query "Parameter.Value" --output text
}

check_state_object() {
    module=$1
    env=$2

    state=$(aws s3 ls --region ${BUCKET_REGION} s3://${BUCKET_NAME}/${AWS_REGION}/${module}/${env})

    test -n "${state}"
}

account_backend() {
    cat <<EOF
bucket = "$BUCKET_NAME"
key    = "${AWS_REGION}/account/${AWS_ACCOUNT}"
region = "${BUCKET_REGION}" # region of the S3 bucket to store the TF state
EOF
}

account_variable() {
    cat <<EOF
region = "${AWS_REGION}"
EOF

    if [[ -n "$CUSTOMER_TAG" ]]; then
        echo "customer_tag = \"${CUSTOMER_TAG}\""
    fi
}

action_account() {
    action=$1

    echo "Info: $action account ${AWS_ACCOUNT}."

    dir="${ROOT_DIR}/account/environments/${AWS_ACCOUNT}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    backend="$(account_backend)" && echo "$backend" > "$dir/backend.tf"
    variable="$(account_variable)" && echo "$variable" > "$dir/variable.tfvar"

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    apply_terraform "${ROOT_DIR}/account" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR"

    rm -rf $dir
}

vpc_backend() {
    cat <<EOF
bucket = "$BUCKET_NAME"
key    = "${AWS_REGION}/vpc/${ENV_NAME}"
region = "${BUCKET_REGION}" # region of the S3 bucket to store the TF state
EOF
}

get_azs() {
    azs=$(aws ec2 describe-availability-zones --region $AWS_REGION --output json | jq -r '.AvailabilityZones[].ZoneName' | head -3 | sort)

    n=$(echo "$azs" | wc -l)
    if [[ $n != 3 ]]; then
        echo "Error: Cannot get three AWS available zone in the region $AWS_REGION. "
        exit 1
    fi

    echo "$azs"
}

net_increase() {
    base_net=$1
    step=$2
    IFS=. parts=($base_net)

    new_part4=$(( ${parts[3]} + $step ))
    if [[ $new_part4 -gt 255 ]]; then
        new_part3=$(( ${parts[2]} + ( ${new_part4} / 256 ) ))
        new_part4=$(( $new_part4 % 256 ))
    else
        new_part3=${parts[2]}
    fi

    echo "${parts[0]}.${parts[1]}.${new_part3}.${new_part4}"
}

vpc_variable() {
    IFS=/ s=($VPC_CIDR)
    network="${s[0]}"
    netbits=${s[1]}
    pub_subnetbits=$(( netbits + 4 ))
    priv_subnetbits=$(( netbits + 2 ))

    pub_step=$(( 2 ** ( 32 - $pub_subnetbits ) ))
    pub_subnet[0]=$network
    pub_subnet[1]=$(net_increase ${pub_subnet[0]} $pub_step)
    pub_subnet[2]=$(net_increase ${pub_subnet[1]} $pub_step)

    priv_step=$(( 2 ** ( 32 - $priv_subnetbits) ))
    priv_subnet[0]=$(net_increase ${pub_subnet[2]} $(( $pub_step * 2 )))
    priv_subnet[1]=$(net_increase ${priv_subnet[0]} $priv_step)
    priv_subnet[2]=$(net_increase ${priv_subnet[1]} $priv_step)

    [[ ! -f "${SAVE_DIR}/${JUMPHOST_SSHKEY}" ]] && ssh-keygen -b 4096 -t rsa -q -N "" -f "$SAVE_DIR/${JUMPHOST_SSHKEY}" -m pem -C "${JUMPHOST_SSHKEY}"
    jumphost_ssh_pub=$(cat "${SAVE_DIR}/${JUMPHOST_SSHKEY}.pub")

    s=$(get_azs)
    local IFS=$'\n'
    azs=($s)

    cat <<EOF
region                     = "${AWS_REGION}"
vpc_name                   = "${ENV_NAME}"
vpc_cidr_block             = "${VPC_CIDR}"
vpc_additional_cidr_blocks = []
private_subnets = {
  "subnet-a" : {
    az         = "${azs[0]}",
    cidr_block = "${priv_subnet[0]}/${priv_subnetbits}"
  },
  "subnet-b" : {
    az         = "${azs[1]}",
    cidr_block = "${priv_subnet[1]}/${priv_subnetbits}"
  },
  "subnet-c" : {
    az         = "${azs[2]}",
    cidr_block = "${priv_subnet[2]}/${priv_subnetbits}"
  }
}
public_subnets = {
  "subnet-a-pub" : {
    az         = "${azs[0]}"
    cidr_block = "${pub_subnet[0]}/${pub_subnetbits}"
  },
  "subnet-b-pub" : {
    az         = "${azs[1]}"
    cidr_block = "${pub_subnet[1]}/${pub_subnetbits}"
  },
  "subnet-c-pub" : {
    az         = "${azs[2]}"
    cidr_block = "${pub_subnet[2]}/${pub_subnetbits}"
  }
}
jumphost_instance_ssh_key_pub = "$jumphost_ssh_pub"
jumphost_subnet = {
  name       = "${ENV_NAME}-subnet-a-pub"
  az         = "${azs[0]}"
  cidr_block = "${pub_subnet[0]}/${pub_subnetbits}"
}
jumphost_ami_id = "$(get_jumphost_ami)"
jumphost_ip_allow_list = $(jumphost_ip_allow_list)
EOF

    if [[ -n "$CUSTOMER_TAG" ]]; then
        echo "customer_tag = \"${CUSTOMER_TAG}\""
    fi
}

jumphost_ip_allow_list() {
    if [ -z "$JUMPHOST_IP_ALLOW_LIST" ]; then
        echo "[]"
        return
    fi
    echo "[\"$(echo $JUMPHOST_IP_ALLOW_LIST | sed -e 's/,/","/g')\"]"
}

action_vpc() {
    action=$1

    echo "Info: $action VPC ${ENV_NAME}."

    dir="${ROOT_DIR}/${ORCH_DIR}/vpc/environments/${ENV_NAME}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    backend="$(vpc_backend)" && echo "$backend" > "$dir/backend.tf"
    variable="$(vpc_variable)" && echo "$variable" > "$dir/variable.tfvar"

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    apply_terraform "${ROOT_DIR}/${ORCH_DIR}/vpc" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR"

    rm -rf $dir
}

create_vpc() {
    action_vpc apply

    local_public_ip_https=$(curl -s 'https://ipinfo.io' 2>/dev/null | jq -r '.ip')
    if [[ -z "$local_public_ip_https" ]]; then
        echo "Warning: Not able to get https public IP address."
    fi
    add_cidr_to_jumphost "${local_public_ip_https}/32"

    if [[ -n "$SOCKS_PROXY" ]]; then
        cmd="http_proxy=\"\" https_proxy=\"\" HTTPS_PROXY=\"\" HTTP_PROXY=\"\" curl --socks5 $SOCKS_PROXY https://ipinfo.io"
        local_public_ip_socks=$(curl -s 'https://ipinfo.io' 2>/dev/null | jq -r '.ip')
        if [[ -z "$local_public_ip_socks" ]]; then
            echo "Warning: Not able to get socks public IP address."
        fi
        if [[ "$local_public_ip_https" != "$local_public_ip_socks" ]]; then
            add_cidr_to_jumphost "${local_public_ip_socks}/32"
        fi
    fi

    upload_savedir_file $VPCSTATE
    upload_savedir_file $JUMPHOST_SSHKEY
    upload_savedir_file ${JUMPHOST_SSHKEY}.pub
}

get_vpc_id() {
    if [[ -n "$VPC_ID" ]]; then
        echo $VPC_ID
        return
    fi

    aws ec2 describe-vpcs --region ${AWS_REGION} --filter Name=tag:Name,Values=$ENV_NAME --query Vpcs[].VpcId --output text
}

orch_route53_backend() {
    cat <<EOF
bucket = "$BUCKET_NAME"
key    = "${AWS_REGION}/orch-route53/${ENV_NAME}"
region = "$BUCKET_REGION"
EOF
}

orch_route53_variable() {
    cat <<EOF
parent_zone                     = "$PARENT_DOMAIN"
create_root_domain              = "$CREATE_ROOT_DOMAIN"
orch_name                       = "$ENV_NAME"
vpc_id                          = "$(get_vpc_id)"
vpc_region                      = "${AWS_REGION}"
lb_created                      = $1
enable_pull_through_cache_proxy = $ENABLE_CACHE_REGISTRY
EOF

    if [[ -n "$CUSTOMER_TAG" ]]; then
        echo "customer_tag = \"${CUSTOMER_TAG}\""
    fi
}

action_orch_route53_wo_lb() {
    action=$1

    echo "Info: $action orch-route53 ${ENV_NAME}.${PARENT_DOMAIN}."

    dir="${ROOT_DIR}/${ORCH_DIR}/orch-route53/environments/${ENV_NAME}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    backend="$(orch_route53_backend)" && echo "$backend" > $dir/backend.tf
    variable="$(orch_route53_variable false)" && echo "$variable" > $dir/variable.tfvar

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    apply_terraform "${ROOT_DIR}/${ORCH_DIR}/orch-route53" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR"
    if $REDUCE_NS_TTL && [[ $action == "apply" ]]; then
        reduce_ns_ttl
    fi
}

action_orch_route53_wi_lb() {
    action=$1

    echo "Info: $action orch-route53 with the load balancer FQDNs."

    dir="${ROOT_DIR}/${ORCH_DIR}/orch-route53/environments/${ENV_NAME}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    backend="$(orch_route53_backend)" && echo "$backend" > $dir/backend.tf
    variable="$(orch_route53_variable true)" && echo "$variable" > $dir/variable.tfvar

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    apply_terraform "${ROOT_DIR}/${ORCH_DIR}/orch-route53" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR"
}

reduce_ns_ttl() {
    for i in $(seq 1 3); do
        if public_hostedzone_id=$(aws route53 list-hosted-zones-by-name | jq --arg name "${ROOT_DOMAIN}." -r '.HostedZones[] | select(.Name==$name and (.Config.PrivateZone|not)) | .Id'); then
            break
        fi
        sleep 60
    done
    if [[ -z "${public_hostedzone_id:-}" ]]; then
        echo "Error: Failed to call list-hosted-zones-by-name."
        exit 1
    fi

    for i in $(seq 1 3); do
        if values=$(aws route53 list-resource-record-sets --hosted-zone-id ${public_hostedzone_id} --query 'ResourceRecordSets[?Type==`NS`]'| jq -r '.[0].ResourceRecords'); then
            break
        fi
        sleep 60
    done
    if [[ -z "${values:-}" ]]; then
        echo "Error: Failed to call list-resource-record-sets."
        exit 1
    fi

    cat > upsert.json << EOF
{
    "Changes": [
        {
          "Action": "UPSERT",
          "ResourceRecordSet": {
                "Name": "${ROOT_DOMAIN}.",
                "Type": "NS",
                "TTL": 900,
                "ResourceRecords": ${values}
          }
        }
    ]
}
EOF
    aws route53 change-resource-record-sets --hosted-zone-id ${public_hostedzone_id} --change-batch file://upsert.json
    rm upsert.json
}

get_letsencrypt_cert() {
    diff=-1

    if sudo test -f ${SAVE_DIR}/${FULLCHAIN} ; then
        expiry_date=$(date -d $(sudo openssl x509 -noout -enddate -in ${SAVE_DIR}/${FULLCHAIN} | cut -d= -f2|awk '{print $1"-"$2"-"$4}') +'%s')
        today=$(date +'%s')
        diff=$(echo $(( ( expiry_date - today )/(60*60*24) )) )
    fi
    # Let's Encrypt does not allow renewing too many times in 168 hours.
    if [[ $diff -lt 15 ]]; then
        echo "Info: The certificate ${SAVE_DIR}/${FULLCHAIN} does not exist or will expire in $diff days. Renewing it..."
        if [[ -n "$AUTO_APPROVE" ]]; then
            sudo -E certbot certonly --dns-route53 --preferred-chain "ISRG Root X1" --agree-tos -m $EMAIL -d *.${ROOT_DOMAIN} -d ${ROOT_DOMAIN} --key-type ecdsa --elliptic-curve=secp384r1 --non-interactive
        else
            sudo -E certbot certonly --dns-route53 --preferred-chain "ISRG Root X1" --agree-tos -m $EMAIL -d *.${ROOT_DOMAIN} -d ${ROOT_DOMAIN} --key-type ecdsa --elliptic-curve=secp384r1
        fi
        if ! sudo test -f /etc/letsencrypt/live/${ROOT_DOMAIN}/fullchain.pem; then
            echo "Error: No /etc/letsencrypt/live/${ROOT_DOMAIN}/fullchain.pem is generated."
            exit 1
        fi
        echo "${LETSENCRYPT_isrgrootx1}" | base64 -d > ${SAVE_DIR}/isrgrootx1.pem
        sudo cat /etc/letsencrypt/live/${ROOT_DOMAIN}/fullchain.pem ${SAVE_DIR}/isrgrootx1.pem > ${SAVE_DIR}/${FULLCHAIN}
        sudo cp /etc/letsencrypt/live/${ROOT_DOMAIN}/privkey.pem ${SAVE_DIR}/${PRIVKEY}
        sudo cat /etc/letsencrypt/live/${ROOT_DOMAIN}/chain.pem ${SAVE_DIR}/isrgrootx1.pem > ${SAVE_DIR}/${CHAIN}
        sudo chown $USER ${SAVE_DIR}/{${FULLCHAIN},${PRIVKEY},${CHAIN}}
    else
        echo "Info: Reuse the certificate ${SAVE_DIR}/${FULLCHAIN} which will expire in $diff days."
    fi
}

get_eks_node_ami() {
    ami=""
    if check_eks_exist; then
        ami=$(aws eks describe-nodegroup --region ${AWS_REGION} --cluster-name ${ENV_NAME} --nodegroup-name nodegroup-${ENV_NAME} | jq -r '.nodegroup.releaseVersion')
    fi

    if [[ -z "$ami" ]] || [[ "$ami" == "null" ]]; then
        ami="$(get_eksnode_ami $EKS_VERSION)"
    fi

    echo $ami
}

get_jumphost_ami() {
    ami=""
    if check_state_object ${ORCH_DIR}/vpc $ENV_NAME; then
        outfile="${SAVE_DIR}/${VPCSTATE}"
        if aws s3api get-object --region "${BUCKET_REGION}" --bucket ${BUCKET_NAME} --key ${AWS_REGION}/vpc/${ENV_NAME} $outfile > /dev/null; then
            ami="$(jq -r '.resources[] | select((.module == "module.jumphost") and (.type == "aws_instance")) | .instances[0].attributes.ami' $outfile)"
        fi
    fi

    if [[ -z "$ami" ]] || [[ "$ami" == "null" ]]; then
        ami=$(jumphost_ami ${AWS_REGION})
    fi

    echo $ami
}

cluster_backend() {
    cat <<EOF
bucket = "$BUCKET_NAME"
key    = "${AWS_REGION}/cluster/${ENV_NAME}"
region = "${BUCKET_REGION}" # region of the S3 bucket to store the TF state
EOF
}

get_aurora_ins_azs() {
    aurora_azs=($1)

    aurora_ins_azs="["
    n=$(( $NUM_RDS_INSTANCES - 1 ))
    i_az=0
    for i in $(seq 0 $n); do
        if [[ $i -gt 0 ]]; then
            aurora_ins_azs="${aurora_ins_azs},"
        fi
        aurora_ins_azs="${aurora_ins_azs}\"${aurora_azs[$i_az]}\""

        (( i_az ++ )) || true
        [[ $i_az -ge 3 ]] && i_az=0
    done
    aurora_ins_azs="${aurora_ins_azs}]"

    echo $aurora_ins_azs
}

cluster_variable() {
    s=$(get_azs)
    azs=($s)
    aurora_ins_azs="$(get_aurora_ins_azs "$s")"
    VPC_TERRAFORM_BACKEND_KEY="${AWS_REGION}/vpc/${ENV_NAME}"

    # We will not create VPC if the --skip-apply-vpc is set
    # and the VPC_ID is not empty.
    if [[ -n "$VPC_ID" ]] && $SKIP_APPLY_VPC; then
        VPC_TERRAFORM_BACKEND_KEY="${AWS_REGION}/vpc/${VPC_ID}"
    fi

    cat <<EOF
vpc_terraform_backend_bucket       = "$BUCKET_NAME"
vpc_terraform_backend_key          = "${VPC_TERRAFORM_BACKEND_KEY}"
vpc_terraform_backend_region       = "${BUCKET_REGION}" # region of the S3 bucket to store the TF state
eks_cluster_name                   = "$ENV_NAME"
aws_account_number                 = "$AWS_ACCOUNT"
eks_volume_size                    = 128
eks_desired_size                   = $EKS_DESIRED_SIZE
eks_min_size                       = $EKS_MIN_SIZE
eks_max_size                       = $EKS_MAX_SIZE
eks_node_ami_id                    = "$(get_eks_node_ami)"
eks_volume_type                    = "gp3"
aws_region                         = "${AWS_REGION}"
aurora_availability_zones          = ["${azs[0]}", "${azs[1]}", "${azs[2]}"]
aurora_instance_availability_zones = ${aurora_ins_azs}
aurora_dev_mode                    = false
public_cloud                       = true
efs_throughput_mode                = "elastic"
cluster_fqdn                       = "${ROOT_DOMAIN}"
enable_cache_registry              = ${ENABLE_CACHE_REGISTRY}
enable_pull_through_cache_proxy    = ${ENABLE_CACHE_REGISTRY}
cache_registry                     = "https://docker-cache.${ROOT_DOMAIN}"

# specific to IAC shippable version
enable_eks_auth                    = true
aws_roles                          = [${AWS_ADMIN_ROLES}]
release_service_refresh_token      =  "$AZUREAD_REFRESH_TOKEN"
eks_additional_iam_policies        = []
auto_cert                          = "${AUTO_CERT}"
eks_user_script_post_cloud_init        = <<CIEOF
$EKS_USER_SCRIPT_POST_CLOUD_INIT
CIEOF

eks_user_script_pre_cloud_init         = <<CIEOF
$EKS_USER_SCRIPT_PRE_CLOUD_INIT
CIEOF

eks_http_proxy                         = "$EKS_HTTP_PROXY"
eks_https_proxy                        = "$EKS_HTTPS_PROXY"
eks_no_proxy                           = "$EKS_NO_PROXY"
eks_cluster_dns_ip                 = "$EKS_CLUSTER_DNS_IP"
EOF

    if [[ -n "$CUSTOMER_TAG" ]]; then
        echo "customer_tag = \"${CUSTOMER_TAG}\""
    fi
}

add_cidr_to_jumphost() {
    cidr="$1"

    outfile="${SAVE_DIR}/${VPCSTATE}"
    aws s3api get-object --region "${BUCKET_REGION}" --bucket ${BUCKET_NAME} --key ${AWS_REGION}/vpc/${ENV_NAME} $outfile > /dev/null
    jumphost_sg=$(jq -r '.resources[] | select((.module == "module.jumphost") and (.type == "aws_instance")) | .instances[0].attributes.vpc_security_group_ids[0]' $outfile)
    ALLOWED_IPS=$(aws ec2 describe-security-groups --group-id "${jumphost_sg}" | jq '.SecurityGroups[].IpPermissions[].IpRanges[].CidrIp' -r)
    if echo "$ALLOWED_IPS" | grep -q "$cidr"; then
        echo "Info: CIDR $cidr is already allowed in the security group $jumphost_sg."
        return
    fi
    aws ec2 authorize-security-group-ingress --group-id "${jumphost_sg}" --protocol tcp --port 22 --cidr "$cidr" --region "${AWS_REGION}"
}

verify_var_blank() {
    v=$1
    test -n "${!v:-""}"
}

error_var_blank() {
    v=$1
    if ! verify_var_blank "$v"; then
        echo "Error: Variable $v is not set."
        exit 1
    fi
}

action_cluster() {
    action=$1

    echo "Info: $action cluster ${ENV_NAME}."

    dir="${ROOT_DIR}/${ORCH_DIR}/cluster/environments/${ENV_NAME}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    # vvvvvvvv Keep this part for backward compatibility vvvvvvvv
    # Will deprecate in 25.02 release
    if $AUTO_CERT; then
        export TF_VAR_tls_key="$(cat ${SAVE_DIR}/${PRIVKEY})"
        export TF_VAR_tls_cert="$(cat ${SAVE_DIR}/${FULLCHAIN})"
        export TF_VAR_ca_cert="$(cat ${SAVE_DIR}/${CHAIN})"
    fi
    export TF_VAR_auto_cert=${AUTO_CERT}
    export TF_VAR_webhook_github_netrc=""

    for v in "TF_VAR_tls_key" "TF_VAR_tls_cert" "TF_VAR_ca_cert"; do
        error_var_blank $v
    done
    # ^^^^^^^^ Keep this part for backward compatibility ^^^^^^^^

    local tfvar_override=$(mktemp)
    if $AUTO_CERT; then
        echo "tls_key = <<-EOF" > $tfvar_override
        cat "${SAVE_DIR}/${PRIVKEY}" >> $tfvar_override
        echo "EOF" >> $tfvar_override
        echo "tls_cert = <<-EOF" >> $tfvar_override
        cat "${SAVE_DIR}/${FULLCHAIN}" >> $tfvar_override
        echo "EOF" >> $tfvar_override
        echo "ca_cert = <<-EOF" >> $tfvar_override
        cat "${SAVE_DIR}/${CHAIN}" >> $tfvar_override
        echo "EOF" >> $tfvar_override
    fi
    echo "auto_cert=${AUTO_CERT}" >> $tfvar_override
    echo "webhook_github_netrc=\"\"" >> $tfvar_override

    if [[ "$OVERRIDE_EKS_SIZE" == "true" ]]; then
        echo "eks_min_size=${EKS_MIN_SIZE}" >> $tfvar_override
        echo "eks_max_size=${EKS_MAX_SIZE}" >> $tfvar_override
        echo "eks_desired_size=${EKS_DESIRED_SIZE}" >> $tfvar_override
    fi

    if [[ "$OVERRIDE_EKS_NODE_TYPE" == "true" ]]; then
        echo "eks_node_instance_type=\"${EKS_NODE_TYPE}\"" >> $tfvar_override
    fi

    if [[ "$OVERRIDE_EKS_O11Y_SIZE" == "true" ]] || [[ "$OVERRIDE_EKS_O11Y_NODE_TYPE" == "true" ]]; then
        cat <<EOF >> $tfvar_override
eks_additional_node_groups={
    "observability": {
        desired_size = $EKS_O11Y_DESIRED_SIZE
        min_size = $EKS_O11Y_MIN_SIZE
        max_size = $EKS_O11Y_MAX_SIZE
        labels = {
            "node.kubernetes.io/custom-rule": "observability"
        }
        taints = {
        "node.kubernetes.io/custom-rule": {
            value = "observability"
            effect = "NO_SCHEDULE"
        }
        }
        instance_type = "$EKS_O11Y_NODE_TYPE"
        volume_size = 20
        volume_type = "gp3"
    }
}
EOF
    fi

    if [[ "$OVERRIDE_RDS_ACU" == "true" ]]; then
        cat <<EOF >> $tfvar_override
aurora_min_acus = $MIN_RDS_ACU
aurora_max_acus = $MAX_RDS_ACU
EOF
    fi

    if [[ "$AZUREAD_REFRESH_TOKEN" != "" ]]; then
        echo "release_service_refresh_token = \"$AZUREAD_REFRESH_TOKEN\"" >> $tfvar_override
    fi

    if [[ -n "$CLUSTER_PROFILE" ]]; then
        echo "# Profile: \"$CLUSTER_PROFILE\"" > "$SAVE_DIR/$PROFILE_TFVAR"
        profile_variables "$CLUSTER_PROFILE" >> "$SAVE_DIR/$PROFILE_TFVAR"
    else
        rm -f "$SAVE_DIR/$PROFILE_TFVAR" || true
        touch "$SAVE_DIR/$PROFILE_TFVAR"
    fi

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    backend="$(cluster_backend)" && echo "$backend" > $dir/backend.tf
    variable="$(cluster_variable)" && echo "$variable" > $dir/variable.tfvar
    module="${ROOT_DIR}/${ORCH_DIR}/cluster"
    apply_terraform "$module" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$PROFILE_TFVAR" "$SAVE_DIR/$VARIABLE_TFVAR" "$tfvar_override"

    if [[ "$action" = "apply" ]]; then
        pushd $module
        gitea_argo_user="\"argocd\""
        gitea_argo_token=$(terraform show -json | jq '.values.outputs.gitea_user_passwords.value.argocd')
        gitea_co_user="\"clusterorch\""
        gitea_co_token=$(terraform show -json | jq '.values.outputs.gitea_user_passwords.value.clusterorch')
        popd

        jq  -n ". += {\"gitea_argo_user\":${gitea_argo_user}}" | \
        jq  ". += {\"gitea_argo_token\":${gitea_argo_token}}" | \
        jq  ". += {\"gitea_co_user\":${gitea_co_user}}" | \
        jq  ". += {\"gitea_co_token\":${gitea_co_token}}" > \
        ${SAVE_DIR}/${OUTPUT}

        upload_savedir_file ${PRIVKEY}
        upload_savedir_file ${FULLCHAIN}
        upload_savedir_file ${CHAIN}
        upload_savedir_file ${OUTPUT}
        upload_savedir_file ${PROFILE_TFVAR}
        connect_cluster
        kubectl --kubeconfig "${KUBECONFIG}" patch storageclass gp2 -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'
    fi

    rm -rf $dir
}

destroy_cluster() {
    aws rds modify-db-cluster --db-cluster-identifier "${ENV_NAME}-aurora-postgresql" --no-deletion-protection --region "${AWS_REGION}" --no-cli-pager &>/dev/null || true
    aws rds delete-db-cluster-snapshot --db-cluster-snapshot-identifier ${ENV_NAME}-aurora-postgresql-final-snapshot --region "${AWS_REGION}" --no-cli-pager &>/dev/null || true
    action_cluster destroy
}

orch_loadbalancer_backend() {
    cat <<EOF
bucket = "$BUCKET_NAME"
key    = "${AWS_REGION}/orch-load-balancer/${ENV_NAME}"
region = "${BUCKET_REGION}" # region of the S3 bucket to store the TF state
EOF
}

orch_loadbalancer_variable() {
    cat <<EOF
vpc_terraform_backend_bucket = "${BUCKET_NAME}"
vpc_terraform_backend_key    = "${AWS_REGION}/vpc/${ENV_NAME}"
vpc_terraform_backend_region = "${BUCKET_REGION}"

cluster_terraform_backend_bucket = "${BUCKET_NAME}"
cluster_terraform_backend_key    = "${AWS_REGION}/cluster/${ENV_NAME}"
cluster_terraform_backend_region = "${BUCKET_REGION}"

cluster_name = "${ENV_NAME}"
ip_allow_list = ["0.0.0.0/0"]
create_target_group_attachment = true
root_domain = "${ROOT_DOMAIN}"

internal = false
EOF

    if [[ -n "$CUSTOMER_TAG" ]]; then
        echo "customer_tag = \"${CUSTOMER_TAG}\""
    fi
}

orch_loadbalancer_variable_internal_default() {
    cat <<EOF
vpc_terraform_backend_bucket = "${BUCKET_NAME}"
vpc_terraform_backend_key    = "${AWS_REGION}/vpc/${VPC_ID}"
vpc_terraform_backend_region = "${BUCKET_REGION}"

cluster_terraform_backend_bucket = "${BUCKET_NAME}"
cluster_terraform_backend_key    = "${AWS_REGION}/cluster/${ENV_NAME}"
cluster_terraform_backend_region = "${BUCKET_REGION}"

cluster_name = "${ENV_NAME}"
ip_allow_list = ["0.0.0.0/0"]
create_target_group_attachment = true
root_domain = "${ROOT_DOMAIN}"

internal = true
EOF

    if [[ -n "$CUSTOMER_TAG" ]]; then
        echo "customer_tag = \"${CUSTOMER_TAG}\""
    fi
}

action_orch_loadbalancer() {
    action=$1

    echo "Info: $1 orch load balancer ${ENV_NAME} ."

    dir="${ROOT_DIR}/${ORCH_DIR}/orch-load-balancer/environments/${ENV_NAME}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    # vvvvvvvv Keep this part for backward compatibility vvvvvvvv
    # Will deprecate in 25.02 release
    # The application load balancer module has its own certificate variables.
    if $AUTO_CERT; then
        export TF_VAR_tls_key="$(cat ${SAVE_DIR}/${PRIVKEY})"
        export TF_VAR_tls_cert_chain="$(cat ${SAVE_DIR}/${CHAIN})"
        export TF_VAR_tls_cert_body="$(get_end_cert "$(cat ${SAVE_DIR}/${FULLCHAIN})")"
        export TF_VAR_auto_cert=${AUTO_CERT}
    else
        load_values
        export TF_VAR_tls_cert_chain="$TF_VAR_tls_cert"
        export TF_VAR_tls_cert_body=$(get_end_cert "$TF_VAR_tls_cert")
    fi

    # Skip destroying the aws-lb-target-group-binding Kubernetes resources to avoid errors for old clusters which don't have the Kubernetes CRD installed
    if [[ $action == "destroy" ]]; then
        export TF_VAR_create_target_group_binding=false
    fi
    # ^^^^^^^^ Keep this part for backward compatibility ^^^^^^^^

    # The application load balancer module has its own certificate variables.
    local variable_override=$(mktemp)
    if $AUTO_CERT; then
        echo "tls_key = <<-EOF" > $variable_override
        cat "${SAVE_DIR}/${PRIVKEY}" >> $variable_override
        echo "EOF" >> $variable_override

        echo "tls_cert_chain = <<-EOF" >> $variable_override
        cat "${SAVE_DIR}/${CHAIN}" >> $variable_override
        echo "EOF" >> $variable_override

        tls_cert_body="$(get_end_cert "$(cat ${SAVE_DIR}/${FULLCHAIN})")"
        echo "tls_cert_body = <<-EOF" >> $variable_override
        echo "$tls_cert_body" >> $variable_override
        echo "EOF" >> $variable_override

        echo "auto_cert = ${AUTO_CERT}"
    else
        # TODO: remove "load_values" once we remove values.sh in 25.02 release
        # We should use a single tfvar file to store all the variables, including certificate variables
        load_values

        echo "tls_key = <<-EOF" > $variable_override
        cat "${TF_VAR_tls_key}" >> $variable_override
        echo "EOF" >> $variable_override

        echo "tls_cert_chain = <<-EOF" >> $variable_override
        cat "${TF_VAR_tls_cert}" >> $variable_override
        echo "EOF" >> $variable_override

        tls_cert_body="$(get_end_cert "${TF_VAR_tls_cert}")"
        echo "tls_cert_body = <<-EOF" >> $variable_override
        echo "$tls_cert_body" >> $variable_override
        echo "EOF" >> $variable_override
    fi

    # Skip destroying the aws-lb-target-group-binding Kubernetes resources to avoid errors for old clusters which don't have the Kubernetes CRD installed
    if [[ $action == "destroy" ]]; then
        echo "create_target_group_binding = false" >> $variable_override
    fi

    module=""
    backend="$(orch_loadbalancer_backend)" && echo "$backend" > $dir/backend.tf
    module="${ROOT_DIR}/${ORCH_DIR}/orch-load-balancer"
    if ! $INTERNAL; then
        variable="$(orch_loadbalancer_variable)" && echo "$variable" > $dir/variable.tfvar
    else
        variable="$(orch_loadbalancer_variable_internal_default)" && echo "$variable" > $dir/variable.tfvar
    fi

    if [[ $action == "destroy" ]] && ! disable_lb_protect; then
        echo "Warning: It seems there is no load balancers created. Skipping destroying load balancers."
        return
    fi

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    logs_file=$(mktemp)
    exec_result=0
    touch $dir/variable.tfvar
    apply_terraform "$module" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR" "$variable_override" 2>&1 | tee "$logs_file" || exec_result=$?
    # Retry if log contains "WAFUnavailableEntityException" string, maximum retry is 20 times
    for i in {1..20}; do
        if [[ $exec_result -ne 0 && $(grep -c "WAFUnavailableEntityException" "$logs_file") -gt 0 ]]; then
            echo "Got WAFUnavailableEntityException error, retrying...(retry $i)"
            exec_result=0
            apply_terraform "$module" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR" "$variable_override" 2>&1 | tee "$logs_file" || exec_result=$?
            sleep 15
        elif [[ $exec_result -ne 0 ]]; then
            exit 1
        else
            break
        fi
    done
    if [[ $exec_result -ne 0 ]]; then
        echo "Error: Failed to $action orch load balancer ${ENV_NAME}."
        if grep -q "WAFUnavailableEntityException" "$logs_file"; then
            echo "Error: WAFUnavailableEntityException error still exists after retrying."
            echo "Please check the AWS WAF service status and try again."
            echo "Reference: https://docs.aws.amazon.com/waf/latest/APIReference/API_AssociateWebACL.html#API_AssociateWebACL_Errors"
        fi
        exit 1
    fi
    rm -rf "$logs_file"

    rm -rf $dir
}

ptcp_backend() {
    cat <<EOF
bucket = "$BUCKET_NAME"
key    = "${AWS_REGION}/pull-through-cache-proxy/${ENV_NAME}"
region = "${BUCKET_REGION}" # region of the S3 bucket to store the TF state
EOF
}

ptcp_variable() {
    # We will not create VPC if the --skip-apply-vpc is set
    # and the VPC_ID is not empty.
    if [[ -n "$VPC_ID" ]] && $SKIP_APPLY_VPC; then
        VPC_TERRAFORM_BACKEND_KEY="${AWS_REGION}/vpc/${VPC_ID}"
    else
        VPC_TERRAFORM_BACKEND_KEY="${AWS_REGION}/vpc/${ENV_NAME}"
    fi
    cat <<EOF
vpc_terraform_backend_bucket = "${BUCKET_NAME}"
vpc_terraform_backend_key    = "${VPC_TERRAFORM_BACKEND_KEY}"
vpc_terraform_backend_region = "${BUCKET_REGION}"

name       = "$ENV_NAME"
aws_region = "${AWS_REGION}"

# reusing the EKS cluster proxy setting since we are deploying the PTCP in the same VPC
http_proxy                         = "$EKS_HTTP_PROXY"
https_proxy                        = "$EKS_HTTPS_PROXY"
no_proxy                           = "$EKS_NO_PROXY"

customer_tag = "${CUSTOMER_TAG}"
route53_zone_name = "${ROOT_DOMAIN}"

EOF

if $INTERNAL; then
    echo "with_public_ip = false"
else
    echo "with_public_ip = true"
fi
}

action_ptcp() {
    action=$1

    echo "Info: $action pull-through cache proxy ${ENV_NAME}."

    dir="${ROOT_DIR}/${ORCH_DIR}/pull-through-cache-proxy/environments/${ENV_NAME}"
    [[ ! -d ${dir} ]] && mkdir -p "${dir}"

    backend="$(ptcp_backend)" && echo "$backend" > $dir/backend.tf
    variable="$(ptcp_variable)" && echo "$variable" > $dir/variable.tfvar

    if [[ ! -f "$SAVE_DIR/$VARIABLE_TFVAR" ]]; then
        # In case the variable file is not created, create an empty one
        touch "$SAVE_DIR/$VARIABLE_TFVAR"
    fi

    local tfvar_override=$(mktemp)
    if $AUTO_CERT; then
        echo "tls_key = <<-EOF" > $tfvar_override
        cat "${SAVE_DIR}/${PRIVKEY}" >> $tfvar_override
        echo "EOF" >> $tfvar_override
        echo "tls_cert = <<-EOF" >> $tfvar_override
        cat "${SAVE_DIR}/${FULLCHAIN}" >> $tfvar_override
        echo "EOF" >> $tfvar_override
    fi

    module="${ROOT_DIR}/${ORCH_DIR}/pull-through-cache-proxy"
    apply_terraform "$module" "$action" "$dir/backend.tf" "$dir/variable.tfvar" "$SAVE_DIR/$VARIABLE_TFVAR" "$tfvar_override"

    rm -rf $dir
}

disable_lb_protect() {
    local tf_state_region=$BUCKET_REGION
    local tf_state_bucket=$BUCKET_NAME
    local tf_state_key=${AWS_REGION}/orch-load-balancer/${ENV_NAME}

    outfile="${SAVE_DIR}/${BUCKET_NAME}-${AWS_REGION}-orch-load-balancer-${ENV_NAME}.json"
    # Use bucket information from backend file if it is internal cluster

    if ! aws s3api get-object --region "${tf_state_region}" --bucket "${tf_state_bucket}" --key "${tf_state_key}" "$outfile" > /dev/null; then
        echo "Info: Not able to download load balancer state file. Have the load balancers been created? Skip destroying load balancers."
        return
    fi

    lb_created=false

    echo "Info: Disable load balancer protection."
    traefik_lb_arn=$(jq -r '.resources[] | select((.module == "module.traefik_load_balancer") and (.type == "aws_lb")) | .instances[0].attributes.arn' $outfile)
    if [[ -n "$traefik_lb_arn" ]]; then
        aws elbv2 modify-load-balancer-attributes --load-balancer-arn $traefik_lb_arn --attributes "Key=deletion_protection.enabled,Value=false" --region "${AWS_REGION}" > /dev/null
        lb_created=true
    fi
    traefik2_lb_arn=$(jq -r '.resources[] | select((.module == "module.traefik2_load_balancer[0]") and (.type == "aws_lb")) | .instances[0].attributes.arn' $outfile)
    if [[ -n "$traefik2_lb_arn" ]]; then
        aws elbv2 modify-load-balancer-attributes --load-balancer-arn $traefik2_lb_arn --attributes "Key=deletion_protection.enabled,Value=false" --region "${AWS_REGION}" > /dev/null
        lb_created=true
    fi
    traefik3_lb_arn=$(jq -r '.resources[] | select((.module == "module.traefik3_load_balancer[0]") and (.type == "aws_lb")) | .instances[0].attributes.arn' $outfile)
    if [[ -n "$traefik3_lb_arn" ]]; then
        aws elbv2 modify-load-balancer-attributes --load-balancer-arn $traefik3_lb_arn --attributes "Key=deletion_protection.enabled,Value=false" --region "${AWS_REGION}" > /dev/null
        lb_created=true
    fi
    argocd_lb_arn=$(jq -r '.resources[] | select((.module == "module.argocd_load_balancer[0]") and (.type == "aws_lb")) | .instances[0].attributes.arn' $outfile)
    if [[ -n "$argocd_lb_arn" ]]; then
        aws elbv2 modify-load-balancer-attributes --load-balancer-arn $argocd_lb_arn --attributes "Key=deletion_protection.enabled,Value=false" --region "${AWS_REGION}" > /dev/null
        lb_created=true
    fi

    $lb_created
}

destroy_orch_loadbalancer() {
    action_orch_loadbalancer destroy
}

check_tool_version() {
    curr=$1
    required=$2

    l=$(echo -e "${curr}\n${required}" | sort -V | tail -1)
    test $l = $curr
}

check_prerequisites() {
    for i in aws terraform jq psql kubectl pip3 make ssh-keygen sshuttle; do
        if ! which $i &>/dev/null; then
            echo "Error: Please install $i and try again."
            exit 1
        fi
    done

    if [[ -n "$AUTO_CERT" ]]; then
        if ! which certbot &>/dev/null || (! pip3 list | grep certbot-dns-route53 &>/dev/null && ! snap list | grep certbot-dns-route53 &>/dev/null); then
            echo "Error: Please install certbot with route53 plugin and try again."
            exit 1
        fi

        v=$(certbot --version | awk '{print $2}')
        if ! check_tool_version $v "1.7.100"; then
            echo "Error: The version of certbot must be higher than 1.7. Please reinstall it and try again. This could work: sudo snap install certbot-dns-route53."
            exit 1
        fi
    fi

    v=$(aws --version|cut -d '/' -f2 |cut -d ' '  -f1)
    if ! check_tool_version $v "2.15.19"; then
        echo "Error: The version of aws must be equal or higher than 2.15.19. Please reinstall it and try again. You may want to follow the instruction in https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html."
        exit 1
    fi
}

check_quotas() {
    if [[ "$SKIP_QUOTA_CHECKS" == "true" ]]; then
        return
    fi

    # Elastic IPs (EIPs)
    EIP_REQ=7
    EIP_QUOTA=$(aws service-quotas get-service-quota --service-code ec2 --quota-code L-0263D0A3 |  jq .Quota.Value)
    ec=0

    if ! $INTERNAL && (( EIP_QUOTA < EIP_REQ )); then
        echo "Error: EIP quota is insufficient. You need at least $((EIP_REQ - EIP_QUOTA)) more EIPs."
        ec=1
    fi

    # Network Load Balancers (NLBs)
    NLB_REQ=1
    NLB_QUOTA=$(aws service-quotas get-service-quota --service-code elasticloadbalancing --quota-code L-69A177A2 |  jq .Quota.Value)
    NLB_USED=$(aws elbv2 describe-load-balancers --query 'LoadBalancers[?Type==`network`].LoadBalancerArn' | jq '. | length')

    if (( NLB_QUOTA < NLB_USED + NLB_REQ )); then
        echo "Error: Network Load Balancer (NLB) quota is insufficient. You need at least $((NLB_USED + NLB_REQ - NLB_QUOTA)) more NLBs."
        ec=1
    fi

    # Application Load Balancers (ALBs)
    ALB_REQ=2
    ALB_QUOTA=$(aws service-quotas get-service-quota --service-code elasticloadbalancing --quota-code L-53DA6B97 | jq .Quota.Value)
    ALB_USED=$(aws elbv2 describe-load-balancers --query 'LoadBalancers[?Type==`application`].LoadBalancerArn' | jq '. | length')

    if (( ALB_QUOTA < ALB_USED + ALB_REQ )); then
        echo "Error: Application Load Balancer (ALB) quota is insufficient. You need at least $((ALB_USED + ALB_REQ - ALB_QUOTA)) more ALBs."
        ec=1
    fi

    # VPCs
    VPC_REQ=1
    VPC_QUOTA=$(aws service-quotas get-service-quota --service-code vpc --quota-code L-F678F1CE |  jq .Quota.Value)
    VPC_USED=$(aws ec2 describe-vpcs --query 'Vpcs[*].VpcId' | jq '. | length')

    if (( VPC_QUOTA < VPC_USED + VPC_REQ )); then
        echo "Warning VPC quota is insufficient. You need at least $((VPC_USED + VPC_REQ - VPC_QUOTA)) more VPCs."
    fi

    # NAT Gateways
    NATG_REQ=1
    NATG_QUOTA=$(aws service-quotas get-service-quota --service-code vpc --quota-code L-FE5A380F |  jq .Quota.Value)
    NATG_USED=$(aws ec2 describe-nat-gateways --query 'NatGateways[*].NatGatewayId' | jq '. | length')

    if (( NATG_QUOTA < NATG_USED + NATG_REQ )); then
        echo "Warning: NAT Gateway quota is insufficient. You need at least $((NATG_USED + NATG_REQ - NATG_QUOTA)) more NAT Gateways."
    fi

    # RDS DB instances
    RDSI_REQ=3
    RDSI_QUOTA=$(aws service-quotas get-service-quota --service-code rds --quota-code L-78E853F4 |  jq .Quota.Value)
    RDSI_USED=$(aws rds describe-db-instances --query 'DBInstances[*].DBInstanceIdentifier' | jq '. | length')

    if (( RDSI_QUOTA < RDSI_USED + RDSI_REQ )); then
        echo "Warning: RDS DB Instance quota is insufficient. You need at least $((RDSI_USED + RDSI_REQ - RDSI_QUOTA)) more RDS Instances."
    fi

    if [[ $ec -ne 0 ]]; then
        exit 1
    fi
 }

check_vpc_exist() {
    vpc_cidr=$(aws ec2 describe-vpcs --region ${AWS_REGION} --filter Name=tag:Name,Values=$ENV_NAME --query Vpcs[].CidrBlock --output text)
    test -n "$vpc_cidr"
}

check_eks_exist() {
    if aws eks describe-cluster --region ${AWS_REGION} --name ${ENV_NAME} &> /dev/null; then
        return 0
    else
        return 1
    fi
}

validate_cidr_block() {
    if ! echo $VPC_CIDR | grep -q -P '^[0-9]{1,3}.[0-9]{1,3}+.[0-9]{1,3}+.[0-9]{1,3}/[0-9]{1,2}$'; then
        echo "Error: Wrong CIDR format. It should be something like 192.168.248.0/21."
        exit 1
    fi

    local IFS=/
    s=($VPC_CIDR)
    network="${s[0]}"
    netbits=${s[1]}

    if [[ $netbits -gt 22 ]] && [[ $netbits -lt 16 ]]; then
        echo "Error: The size of the VPC CIDR /$netbits is not supported. The range must be between /16 and /22. The recommendaton is /21"
        exit 1
    fi

    local IFS=.
    s=($network)
    ip1=${s[0]}
    ip2=${s[1]}
    ip3=${s[2]}
    ip4=${s[3]}

    if [[ "$ip4" != "0" ]]; then
        echo "Error: Wrong CIDR block value. The last number of of ${ip4} of the network address must be 0 for the bit number /${netbits}."
        exit 1
    fi

    step=$(( 2 ** ( 24 - $netbits ) ))
    mod=$(( $ip3 % $step ))
    if [[ $mod -ne 0 ]]; then
        echo "Error: Wrong CIDR block value. the third number of ${ip3} of the network address does not match the bit number /${netbits}."
        exit 1
    fi
}

get_new_vpc_cidr() {
    used_cidrs=$(aws ec2 describe-vpcs --region ${AWS_REGION} --filter Name=cidrBlock,Values='192.168.*.0/21' --query Vpcs[].CidrBlock | grep "$VPC_DEFAULT_CIDR_PREFIX")
    for i in $(seq 248 -8 16); do
        cidr="${VPC_DEFAULT_CIDR_PREFIX}${i}.0/21"
        if ! echo $used_cidrs | grep -q $cidr; then
            echo $cidr
            return
        fi
    done

    echo "192.168.248.0/21"
}

set_vpc_cidr() {
    vpc_cidr=$(aws ec2 describe-vpcs --region ${AWS_REGION} --filter Name=tag:Name,Values=$ENV_NAME --query Vpcs[].CidrBlock --output text)

    if [[ -n "$vpc_cidr" ]]; then
        VPC_CIDR=$vpc_cidr
    else
        VPC_CIDR=$(get_new_vpc_cidr)
    fi
}

install() {
    parse_params "$@"
    check_prerequisites
    check_quotas

    values_changed="${SAVE_DIR}/${VALUES_CHANGED}"
    echo "Info: Checking data file..."
    if ! check_s3_savedir_empty && ! [[ -f ${values_changed} ]]; then
        download_savedir
        echo "Info: Pulled S3 ${SAVE_DIR}."
    fi

    if [[ ! -f ${SAVE_DIR}/${VARIABLE_TFVAR} ]]; then
        set_values "install"
    fi

    load_values
    check_running_sshuttle
    check_bucket

    if $NEW_AWS_ACCOUNT; then
        # Check if the IAM policy aws_load_balancer_controller has been created. For some accounts, this policy may have been manually created.
        s=$(aws iam list-policies | jq -r '.Policies[] | select (.PolicyName == "aws_load_balancer_controller")')
        if [[ -z "$s" ]]; then
            action_account apply
        else
            echo "Info: Policy aws_load_balancer_controller has existed. Skip applying account."
        fi
    fi

    if ! $SKIP_APPLY_VPC; then
        create_vpc
    fi

    if $ROUTE53 && ! $SKIP_APPLY_ROUTE53; then
        action_orch_route53_wo_lb apply
    fi
    if $ROUTE53 && $AUTO_CERT; then
        get_letsencrypt_cert
    fi

    refresh_sshuttle

    if $ENABLE_CACHE_REGISTRY; then
        action_ptcp apply
    fi

    if ! $SKIP_APPLY_CLUSTER; then
        action_cluster apply
    fi

    if ! $SKIP_APPLY_LOADBALANCER; then
        action_orch_loadbalancer apply
        if $ROUTE53  && ! $SKIP_APPLY_ROUTE53; then
            action_orch_route53_wi_lb apply
        fi
    fi

    echo "Info: Upload ${SAVE_DIR}."

    upload_savedir
    rm -f ${values_changed} || true

    terminate_sshuttle

    if ! $SKIP_APPLY_CLUSTER; then
        wait_for_gitea
    fi

    echo "Info: Installation completed successfully. Please back up the files in ${SAVE_DIR} directory."
}

uninstall() {
    parse_params "$@"
    check_prerequisites

    values_changed="${SAVE_DIR}/${VALUES_CHANGED}"
    echo "Info: Checking data file..."
    if ! check_s3_savedir_empty && ! [[ -f ${values_changed} ]]; then
        download_savedir
        echo "Info: Pulled S3 ${SAVE_DIR}."
    else
        echo "Warning: Failed to download data files from S3. Make sure the value specified for the --customer-state-prefix parameter is correct. Type \"yes\" to continue, others to quit: "; read s

        s=$(echo $s | tr '[:upper:]' '[:lower:]')
        if [[ "$s" != "yes" ]]; then
            exit 1
        fi
    fi

    load_values
    if ! $SKIP_DESTROY_CLUSTER && ! $SKIP_DESTROY_LOADBALANCER; then
        check_running_sshuttle
        refresh_sshuttle
    fi

    # need to destroy cache registry first since it depends on route53 zone
    if $ENABLE_CACHE_REGISTRY; then
        action_ptcp destroy
    fi

    if $ROUTE53 && ! $SKIP_DESTROY_ROUTE53; then
        if $SKIP_DESTROY_LOADBALANCER; then
            action_orch_route53_wo_lb destroy
        else
            action_orch_route53_wi_lb destroy
        fi
    fi
    if ! $SKIP_DESTROY_LOADBALANCER; then
        destroy_orch_loadbalancer
    fi
    $SKIP_DESTROY_CLUSTER || destroy_cluster
    if has_s3_pending; then
        s3_pending=true
    else
        s3_pending=false
    fi
    if ! $s3_pending ; then
        $SKIP_DESTROY_VPC || action_vpc destroy
        if $RESET_AWS_ACCOUNT; then
            check_state_object account $ENV_NAME && action_account destroy
        fi

        if ! check_eks_exist && ! check_vpc_exist; then
            remove_s3_data
        fi
    else
        upload_savedir_file $S3_PENDING
    fi

    if ! $SKIP_DESTROY_CLUSTER && ! $SKIP_DESTROY_LOADBALANCER; then
        terminate_sshuttle
    fi

    if ! $s3_pending ; then
        echo "Info: Uninstallation completed successfully."
    else
        msg="Info: Uninstallation completed partly."
        msg="${msg} Because some AWS S3 buckets contain large amount of data,"
        msg="${msg} they are not able to be removed immediately for now."
        msg="${msg} A lifecycle configuration rule has been set in each of them for AWS to start to empty them in 24 hours."
        msg="${msg} Depending on the sizes of the contents, the empty process could take hours to days to complete."
        msg="${msg} Please retry the same uninstall command later to complete the process."
        echo "${msg}"
    fi
}

pull-savedir() {
    # pull-savedir is a command
    parse_params "$@"

    if check_local_savedir_empty && ! check_s3_savedir_empty; then
        download_savedir
        echo "Info: files have been pulled in the ${SAVE_DIR} directory."
    else
        echo "Error: Some files have existed. No file is pulled. If you really want to pull, remove the files related to ${ENV_NAME} from the ${SAVE_DIR} directory and try again."
        exit 1
    fi
}

push-savedir() {
    # push-savedir is a command
    parse_params "$@"

    upload_savedir
}

set_values() {
    action="$1"

    TF_VAR_smtp_user=""
    TF_VAR_smtp_pass=""
    TF_VAR_smtp_url=""
    TF_VAR_smtp_port=587
    TF_VAR_smtp_from=""
    TF_VAR_tls_key=""
    TF_VAR_tls_cert=""
    TF_VAR_ca_cert=""

    values_changed="${SAVE_DIR}/${VALUES_CHANGED}"
    rm -f ${values_changed} || true

    # Generate tfvar file for deployment
    update="${SAVE_DIR}/.update-${AWS_ACCOUNT}-${ENV_NAME}-values.tfvar"
    if [[ -f "${SAVE_DIR}/${VARIABLE_TFVAR}" ]]; then
        cp "${SAVE_DIR}/${VARIABLE_TFVAR}" "$update"
        cp ${SAVE_DIR}/${VARIABLE_TFVAR} ${SAVE_DIR}/${VARIABLE_TFVAR}.bak

        # On an upgrade or edit of settings, the auto_cert value may need to be updated
        if grep -q "auto_cert" "$update"; then
            if $AUTO_CERT; then
                sed -i 's/auto_cert=false/auto_cert=true/' $update
            else
                sed -i 's/auto_cert=true/auto_cert=false/' $update
            fi
        else
            if $AUTO_CERT; then
                echo >> $update
                echo "# The Automatic Certificate Management flag." >> $update
                echo 'auto_cert=true' >> $update
            fi
        fi
    else
        cat <<EOF >$update
# Modify the following variables with the right values. Values can be multiple lines."
# Note:
#   Escape double quotation marks in the value by replacing " with \", for example:
#       var = "\"key\":\"name\""
#   Or alternatively, use heredoc
#       var = <<-EOF
#         ... your text ...
#       EOF

# The SRE settings, leave empty if disabled.
sre_basic_auth_username=""
sre_basic_auth_password=""
sre_destination_secret_url=""
sre_destination_ca_secret=""

# The email configuration information if it is needed.
smtp_user=""
smtp_pass=""
smtp_url=""
smtp_port=587
smtp_from=""

# The TLS certificate and key if it is needed.
tls_key=""
tls_cert=""
ca_cert=""
EOF

        if $AUTO_CERT; then
            cat <<EOF >> $update
# The Automatic Certificate Management flag.
auto_cert=true
EOF
        fi
    fi

    if [[ $action == "config" ]]; then
        # Pop up the editor only when the config command is used
        editor $update
        if [[ ! -f $update ]]; then
            echo "Info: ${SAVE_DIR}/${VARIABLE_TFVAR} is not created."
            exit 0
        fi

        if [[ ! -f ${SAVE_DIR}/${VARIABLE_TFVAR} ]] || ! diff $update "${SAVE_DIR}/${VARIABLE_TFVAR}" &>/dev/null; then
            echo -n "The configuration values have been modified. Do you want to save it? Type \"yes\" to save it and proceed, others to quit: "; read s

            s=$(echo $s | tr '[:upper:]' '[:lower:]')
            if [[ "$s" == "yes" ]]; then
                mv "$update" ${SAVE_DIR}/${VARIABLE_TFVAR}
                touch "$values_changed"
                check_bucket  # Create the bucket if not existing
                upload_savedir_file "$VARIABLE_TFVAR"
            else
                rm -f $update || true
                echo "Info: Quitting. User selected to quit without saving the file."
                exit
            fi
        else
            rm -f $update || true
            echo "Info: ${SAVE_DIR}/${VARIABLE_TFVAR} is not changed."
        fi
    elif [[ "$action" == "install" ]]; then
        # If the install command is used
        echo "Info: Generate default values."
        mv "$update" ${SAVE_DIR}/${VARIABLE_TFVAR}
        touch "$values_changed"
        check_bucket  # Create the bucket if not existing
        upload_savedir_file "$VARIABLE_TFVAR"
    fi
}

config() {
    parse_params "$@"

    values_changed="${SAVE_DIR}/${VALUES_CHANGED}"
    echo "Info: Checking data file..."
    if ! check_s3_savedir_empty && ! [[ -f ${values_changed} ]]; then
        download_savedir
        echo "Info: Pulled S3 ${SAVE_DIR}."
    fi

    set_values "config"

    echo "Info: Values are saved. Execute the config command to update them if it is needed."
}

get_upgrade_files() {
    version=$1

    file_pattern="upgrade-to"
    dir="${ROOT_DIR}/utils/upgrades"
    reg="${dir}/[0-9]+-${file_pattern}-[0-9]+.[0-9]+[0-9]+.*\\.sh"
    files=$(find ${dir}/ -maxdepth 1 -type f -regex "$reg")
    fix_versions="$(IFS=$'\n' echo "${files}" | sed -ne "s|^${dir}/[0-9]\+-${file_pattern}-\([0-9]\+\.[0-9]\+\.[0-9]\+\).*|\1|p" | sort -V)"

    # Get the upgrade candidate version numbers
    include_version=false
    if echo "${fix_versions[*]}" | grep -q $version; then
        include_version=true
    fi

    if ! $include_version; then
        all="$(echo -e "${fix_versions}\n${version}" | sort -V)"
    else
        all="$fix_versions"
    fi

    # Get the files which have version numbers higher than the the current deployed version number.
    i=0
    upgrade_files=()
    reach_version=false
    for v in ${all}; do
        if [[ $v == $version ]]; then
            reach_version=true

            continue
        fi

        if $reach_version; then
            reg="${dir}/[0-9]+-${file_pattern}-${v}+.*\\.sh"
            f="$(find ${dir}/ -maxdepth 1 -type f -regex "$reg")"
            upgrade_files[i]=$f
            (( i++ ))
        fi
    done

    # Order the upgrade files based on the prefix numbers
    i=0
    prefix_numbers="$(printf "%s\n" ${upgrade_files[*]} | sed -ne "s|^${dir}/\([0-9]\+\)-upgrade-to-.*\.sh|\1|p" | sort -n)"
    upgrade_files=()
    for n in $prefix_numbers; do
        reg="${dir}/${n}-${file_pattern}-[0-9]+.[0-9]+[0-9]+.*\\.sh"
        f="$(find ${dir}/ -maxdepth 1 -type f -regex "$reg")"
        upgrade_files[i]=$f
        (( i++ ))
    done

    if [[ -n "${upgrade_files[*]}" ]]; then
        echo ${upgrade_files[*]}
    else
        current_upgrade=$(ls -d $dir/*-${file_pattern}-${version}.sh)
        echo "$current_upgrade"
    fi
}

upgrade() {
    parse_params "$@"
    echo "Info: Checking deployed version..."

    download_savedir
    load_values
    check_running_sshuttle
    refresh_sshuttle

    deployed_version=$(get_deployed_version)
    echo "Info: The deployed version is $deployed_version."

    fs=$(get_upgrade_files $deployed_version)
    echo "Info: The following upgrades will be applied:"
    for f in $fs; do
        echo "    $f"
    done
    echo -n "Enter 'yes' to start the upgrades. Enter others to exit: "; read s
    s=$(echo $s | tr '[:upper:]' '[:lower:]')
    if [[ "$s" != "yes" ]]; then
        exit
    fi

    for f in $fs; do
        if [[ -f $f ]]; then
            echo "Info: Executing $f."
            source $f
        else
            echo "Warning: File $f does not exist."
        fi
    done

    echo "Info: The upgrade completed successfully."
}

account() {
    parse_params "$@"
    check_bucket

    if $NEW_AWS_ACCOUNT; then
        # Check if the IAM policy aws_load_balancer_controller has been created. For some accounts, this policy may have been manually created.
        s=$(aws iam list-policies | jq -r '.Policies[] | select (.PolicyName == "aws_load_balancer_controller")')
        if [[ -z "$s" ]]; then
            action_account apply
        else
            echo "Info: Policy aws_load_balancer_controller has existed. Skip applying account."
        fi
        echo "Info: The AWS account is initialized."
    fi

    if $RESET_AWS_ACCOUNT; then
        check_state_object account $ENV_NAME && action_account destroy
        echo "Info: The AWS account is reset."
    fi
}

update-cluster-setting() {
    parse_params "$@"
    # Get and set profile for the existing deployment
    # Get the profile name from the savedir

    if ! check_s3_savedir_empty; then
        download_savedir
        echo "Info: Pulled S3 ${SAVE_DIR}."
    fi

    local profile_file="$SAVE_DIR/$PROFILE_TFVAR"

    # Error if no `--profile` parameter is provided
    if [[ -z "$CLUSTER_PROFILE" ]]; then
        echo "Error: The profile parameter should be specified."
        exit 1
    fi

    echo "Info: Updating cluster."
    load_values
    check_running_sshuttle
    refresh_sshuttle
    connect_cluster
    action_cluster apply
    echo "Info: The cluster is updated successfully."
    terminate_sshuttle
}

# Main:

if [[ $# == 0 ]]; then
    echo "Error: No command is found."
    usage
    exit 1
fi

if [[ $1 == '-h' ]] || [[ $1 == '--help' ]]; then
    usage
    exit
fi

COMMAND=$1

if [[ "$COMMAND" =~ ^("install"|"uninstall"|"pull-savedir"|"push-savedir"|"config"|"upgrade"|"account"|"update-cluster-setting")$ ]]; then
    check_savedir

    if [[ -f "${SAVE_DIR}/${VALUES}" ]]; then
        export $(grep -P -v "^\s*#" ${SAVE_DIR}/${VALUES} | grep -P -v  '^\s*$' | xargs)
    fi
    # shellcheck disable=SC2068
    $@
else
    echo "Error: Unrecognized command of $1."
    usage
    exit 1
fi
