#!/usr/bin/env bash

# shellcheck disable=SC1091

set -e
set -o pipefail

# version consts
DOCKER_CE_VERSION_DEBIAN=19.03.5
DOCKER_CE_VERSION_CENTOS=19.03.5
DOCKER_CE_VERSION_SLES=17.09.1
GLUSTER_VERSION_DEBIAN=7
GLUSTER_VERSION_CENTOS=6
IMDS_VERSION=2019-04-30

# consts
DOCKER_CE_PACKAGE_DEBIAN="5:${DOCKER_CE_VERSION_DEBIAN}~3-0~"
DOCKER_CE_PACKAGE_CENTOS="${DOCKER_CE_VERSION_CENTOS}-3.el7"
DOCKER_CE_PACKAGE_SLES="docker-${DOCKER_CE_VERSION_SLES}_ce-257.3"
VMBUS_DEVICES_PATH=/sys/bus/vmbus/devices
GEN1_VMBUS_DEVICE_PREFIX="{00000000-0001-"
GEN2_VMBUS_DEVICE_ID="{f8b3781a-1e82-4818-a1c3-63d806ec15bb}"
IB_SM_DEVICE=/dev/infiniband/issm0
IB_UVERBS_DEVICE=/dev/infiniband/uverbs0
MOUNTS_PATH=$AZ_BATCH_NODE_ROOT_DIR/mounts
VOLATILE_PATH=$AZ_BATCH_NODE_ROOT_DIR/volatile
IB_PKEY_FILE=$AZ_BATCH_TASK_WORKING_DIR/IB_PKEY
UCX_IB_PKEY_FILE=$AZ_BATCH_TASK_WORKING_DIR/UCX_IB_PKEY
MCR_REPO=mcr.microsoft.com
BLOBXFER_IMAGE_PREFIX=${MCR_REPO}/blobxfer
SHIPYARD_IMAGE_PREFIX=${MCR_REPO}/azure-batch/shipyard

# status file consts
lisinstalled=${VOLATILE_PATH}/.batch_shipyard_lis_installed
nodeprepfinished=${VOLATILE_PATH}/.batch_shipyard_node_prep_finished
cascadefailed=${VOLATILE_PATH}/.batch_shipyard_cascade_failed

log() {
    local level=$1
    shift
    echo "$(date -u -Ins) - $level - $*"
}

# dump uname immediately
uname -ar

# try to get os release vars
if [ -e /etc/os-release ]; then
    . /etc/os-release
    DISTRIB_ID=$ID
    DISTRIB_RELEASE=$VERSION_ID
    DISTRIB_CODENAME=$VERSION_CODENAME
    if [ -z "$DISTRIB_CODENAME" ]; then
        if [ "$DISTRIB_ID" == "debian" ] && [ "$DISTRIB_RELEASE" == "9" ]; then
            DISTRIB_CODENAME=stretch
        fi
    fi
else
    if [ -e /etc/lsb-release ]; then
        . /etc/lsb-release
    fi
fi
if [ -z "${DISTRIB_ID}" ] || [ -z "${DISTRIB_RELEASE}" ]; then
    log ERROR "Unknown DISTRIB_ID or DISTRIB_RELEASE."
    exit 1
fi
if [ -z "${DISTRIB_CODENAME}" ]; then
    log WARNING "Unknown DISTRIB_CODENAME."
fi
DISTRIB_ID=${DISTRIB_ID,,}
DISTRIB_RELEASE=${DISTRIB_RELEASE,,}
DISTRIB_CODENAME=${DISTRIB_CODENAME,,}

# set distribution specific vars
PACKAGER=
SYSTEMD_PATH=/lib/systemd/system
if [ "$DISTRIB_ID" == "ubuntu" ]; then
    PACKAGER=apt
elif [ "$DISTRIB_ID" == "debian" ]; then
    PACKAGER=apt
elif [[ $DISTRIB_ID == centos* ]] || [ "$DISTRIB_ID" == "rhel" ]; then
    PACKAGER=yum
else
    PACKAGER=zypper
    SYSTEMD_PATH=/usr/lib/systemd/system
fi
if [ "$PACKAGER" == "apt" ]; then
    export DEBIAN_FRONTEND=noninteractive
fi

# globals
azureblob=0
azurefile=0
batch_insights=0
beeond=0
blobxferversion=latest
block=
cascadecontainer=0
custom_image=0
default_container_runtime=
delay_preload=0
docker_group=
encrypted=
EPHEMERAL_DISK=
EPHEMERAL_DISK_PARTITION=
EPHEMERAL_DEVICE=
EPHEMERAL_ENCRYPTED=
fallback_registry=
gluster_on_compute=0
gpu=
hpnssh=0
kata=0
lis=
networkopt=0
native_mode=0
concurrent_source_downloads=
prefix=
sc_args=
shipyardversion=
singularity_basedir=
singularityversion=
USER_MOUNTPOINT=
vm_size=
vm_rdma_type=0

# process command line options
while getopts "h?abcde:fg:i:jkl:m:no:p:qrs:tuv:wx:yz:" opt; do
    case "$opt" in
        h|\?)
            echo "shipyard_nodeprep.sh parameters"
            echo ""
            echo "-a mount azurefile shares"
            echo "-b block until resources loaded"
            echo "-c mount azureblob containers"
            echo "-d use docker container for cascade"
            echo "-e [thumbprint] encrypted credentials with cert"
            echo "-f set up glusterfs on compute"
            echo "-g [nv-series:driver file:nvidia docker pkg] gpu support"
            echo "-i [version] install singularity container runtime"
            echo "-j delay docker image preload"
            echo "-k install kata containers runtime"
            echo "-l [lis pkg] LIS package install"
            echo "-m [type:scid] mount storage cluster"
            echo "-n native mode"
            echo "-o [fallback registry] fallback registry"
            echo "-p [prefix] storage container prefix"
            echo "-q enable batch insights"
            echo "-r enable azure batch docker group"
            echo "-s [concurrent source downloads] concurrent source downloads"
            echo "-t optimize network TCP settings"
            echo "-u custom image"
            echo "-v [version] batch-shipyard version"
            echo "-w install openssh-hpn"
            echo "-x [version] blobxfer version"
            echo "-y install beegfs beeond for autoscratch"
            echo "-z [runtime] default container runtime"
            echo ""
            exit 1
            ;;
        a)
            azurefile=1
            ;;
        b)
            block=${SHIPYARD_CONTAINER_IMAGES_PRELOAD}
            ;;
        c)
            azureblob=1
            ;;
        d)
            cascadecontainer=1
            ;;
        e)
            encrypted=${OPTARG,,}
            ;;
        f)
            gluster_on_compute=1
            ;;
        g)
            gpu=$OPTARG
            ;;
        i)
            singularityversion=$OPTARG
            ;;
        j)
            delay_preload=1
            ;;
        k)
            kata=1
            ;;
        l)
            lis=$OPTARG
            ;;
        m)
            OLD_IFS=$IFS
            IFS=',' read -ra sc_args <<< "${OPTARG,,}"
            IFS=$OLD_IFS
            ;;
        n)
            native_mode=1
            ;;
        o)
            fallback_registry=$OPTARG
            ;;
        p)
            prefix=$OPTARG
            ;;
        q)
            batch_insights=1
            ;;
        r)
            docker_group="\"group\": \"_azbatchsudogrp\","
            ;;
        s)
            concurrent_source_downloads=$OPTARG
            ;;
        t)
            networkopt=1
            ;;
        u)
            custom_image=1
            ;;
        v)
            shipyardversion=$OPTARG
            ;;
        w)
            hpnssh=1
            ;;
        x)
            blobxferversion=$OPTARG
            ;;
        y)
            beeond=1
            ;;
        z)
            default_container_runtime=${OPTARG,,}
            ;;
    esac
done
shift $((OPTIND-1))
[ "$1" = "--" ] && shift
# check versions
if [ -z "$shipyardversion" ]; then
    log ERROR "batch-shipyard version not specified"
    exit 1
fi
if [ -z "$blobxferversion" ]; then
    log ERROR "blobxfer version not specified"
    exit 1
fi

save_startup_to_volatile() {
    set +e
    touch "${VOLATILE_PATH}"/startup/.save
    set -e
}

get_ephemeral_device() {
    if [ -n "$EPHEMERAL_DISK_PARTITION" ]; then
        return
    fi
    local OLD_IFS=$IFS
    local devices
    local devicepath
    local deviceid
    local device
    local dev
    local blockpaths
    local blockpath
    mapfile -t devices < <(ls "${VMBUS_DEVICES_PATH}")
    for devicepath in "${devices[@]}"; do
        deviceid=$(<"${VMBUS_DEVICES_PATH}/${devicepath}/device_id")
        if [[ "$deviceid" == ${GEN1_VMBUS_DEVICE_PREFIX}* ]] || [ "$deviceid" == "${GEN2_VMBUS_DEVICE_ID}" ]; then
            mapfile -t blockpaths < <(find "${VMBUS_DEVICES_PATH}/${devicepath}/" -type d -name block)
            for blockpath in "${blockpaths[@]}"; do
                if [ -z "$blockpath" ]; then
                    continue
                fi
                # gen2 VMs have all disks presented in this device, so we need
                # to read the LUN. The LUN mappings are:
                # 0: OS disk
                # 1: Resource/ephemeral disk
                # 2: CD-ROM
                if [ "$deviceid" == "${GEN2_VMBUS_DEVICE_ID}" ]; then
                    local lunsplit
                    local lun
                    IFS='/'
                    read -ra lunsplit <<< "$blockpath"
                    IFS=':'
                    read -ra lun <<< "${lunsplit[-2]}"
                    IFS=$OLD_IFS
                    if [ "${lun[-1]}" != "1" ]; then
                        continue
                    fi
                fi
                device=$(find "${blockpath}/" -type d -name "sd*" | head -1)
                IFS='/'
                read -ra dev <<< "$device"
                EPHEMERAL_DISK="/dev/${dev[-1]}"
                EPHEMERAL_DISK_PARTITION="${EPHEMERAL_DISK}1"
                log INFO "Ephemeral disk discovered as $EPHEMERAL_DISK"
                break
            done
        fi
        if [ -n "$EPHEMERAL_DISK_PARTITION" ]; then
            break
        fi
    done
    IFS=$OLD_IFS
}

check_for_buggy_ntfs_mount() {
    # Check to ensure sdb1 mount is not mounted as ntfs
    set +e
    mount | grep /dev/sdb1 | grep fuseblk
    local rc=$?
    set -e
    if [ $rc -eq 0 ]; then
        log ERROR "/dev/sdb1 temp disk is mounted as fuseblk/ntfs"
        exit 1
    fi
}

set_user_mountpoint() {
    local lsblk
    local cryptout
    lsblk=$(lsblk -npr "$EPHEMERAL_DISK_PARTITION")
    log DEBUG "lsblk: $lsblk"
    set +e
    cryptout=$(echo "$lsblk" | grep crypt)
    # check for Azure Disk Encryption
    if [ "$(echo "$cryptout" | cut -d' ' -f 6)" == "crypt" ]; then
        EPHEMERAL_ENCRYPTED=1
        EPHEMERAL_DEVICE=$(echo "$cryptout" | cut -d' ' -f 1)
    else
        EPHEMERAL_ENCRYPTED=0
        EPHEMERAL_DEVICE="$EPHEMERAL_DISK_PARTITION"
    fi
    USER_MOUNTPOINT=$(lsblk -nr -o MOUNTPOINT "$EPHEMERAL_DEVICE")
    log INFO "ephemeral: $EPHEMERAL_DEVICE (encrypted=$EPHEMERAL_ENCRYPTED user=$USER_MOUNTPOINT)"
    if [ "$USER_MOUNTPOINT" != "/mnt" ] && [ "$USER_MOUNTPOINT" != "/mnt/resource" ]; then
        log ERROR "User mountpoint is not in a supported location: $USER_MOUNTPOINT"
        exit 1
    fi
    set -e
}

optimize_tcp_network_settings() {
    # optimize network TCP settings
    if [ "$1" -eq 1 ]; then
        local sysctlfile=/etc/sysctl.d/60-azure-batch-shipyard.conf
        if [ ! -e $sysctlfile ] || [ ! -s $sysctlfile ]; then
cat > $sysctlfile << EOF
net.core.rmem_default=16777216
net.core.wmem_default=16777216
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.core.netdev_max_backlog=30000
net.ipv4.tcp_max_syn_backlog=80960
net.ipv4.tcp_mem=16777216 16777216 16777216
net.ipv4.tcp_rmem=4096 87380 16777216
net.ipv4.tcp_wmem=4096 65536 16777216
net.ipv4.tcp_slow_start_after_idle=0
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_abort_on_overflow=1
net.ipv4.route.flush=1
EOF
        fi
        if [ "$PACKAGER" == "apt" ]; then
            service procps reload
        else
            sysctl -p
        fi
    fi
    # set up hpn-ssh
    if [ "$2" -eq 1 ]; then
        ./shipyard_hpnssh.sh "$DISTRIB_ID" "$DISTRIB_RELEASE"
    fi
}

get_vm_size_from_imds() {
    if [ -n "$vm_size" ]; then
        return
    fi
    curl -fSsL -H Metadata:true "http://169.254.169.254/metadata/instance?api-version=${IMDS_VERSION}" > imd.json
    vm_size=$(python -c "import json;f=open('imd.json','r');a=json.load(f);print(a['compute']['vmSize']).lower()")
    if [[ "$vm_size" =~ ^standard_((hb|hc)[0-9]+m?rs?(_v[1-9])?)$ ]]; then
        # SR-IOV RDMA
        vm_rdma_type=1
    elif [[ "$vm_size" =~ ^standard_(nc[0-9]+rs_v3)$ ]]; then
        # SR-IOV RDMA (transition)
        vm_rdma_type=1
    elif [[ "$vm_size" =~ ^standard_((a8|a9)|((h|nc|nd)[0-9]+m?rs?(_v[1-3])?))$ ]]; then
        # network direct RDMA
        vm_rdma_type=2
    fi
    # perform device-based overrides
    if [ -e "$IB_SM_DEVICE" ]; then
        if [ $vm_rdma_type -ne 1 ]; then
            log INFO "Changing RDMA type from $vm_rdma_type to SR-IOV for VM size $vm_size as $IB_SM_DEVICE was found."
        fi
        vm_rdma_type=1
    elif [ -e "$IB_UVERBS_DEVICE" ]; then
        if [ $vm_rdma_type -ne 2 ]; then
            log INFO "Changing RDMA type from $vm_rdma_type to Network Direct for VM size $vm_size as $IB_UVERBS_DEVICE was found."
        fi
        vm_rdma_type=2
    fi
    # validate setting
    if [ $vm_rdma_type -eq 1 ]; then
        if [ ! -e "$IB_SM_DEVICE" ]; then
            log ERROR "Expected IB device not found for VM size $vm_size: $IB_SM_DEVICE"
            exit 1
        fi
    fi
    if [ $vm_rdma_type -ne 0 ]; then
        if [ ! -e "$IB_UVERBS_DEVICE" ]; then
            log ERROR "Expected IB device not found for VM size $vm_size: $IB_UVERBS_DEVICE"
            exit 1
        fi
    fi
    log INFO "VmSize=$vm_size RDMA=$vm_rdma_type"
}

set_default_pool_user_privileges() {
    if [ -n "$docker_group" ]; then
        # Modify the default pool user to have sudo privileges, which in turn
        # will give access to the Docker socket. This is required to allow
        # input/output phases via Docker containers as the default pool user.
        usermod -aG _azbatchsudogrp _azbatch
    else
        usermod -aG docker _azbatch
    fi
}

download_file_as() {
    log INFO "Downloading: $1 as $2"
    local retries=10
    set +e
    while [ $retries -gt 0 ]; do
        if [ "$DISTRIB_ID" == "debian" ]; then
            if wget -O "$2" "$1"; then
                break
            fi
        else
            if curl -fSsL -o "$2" "$1"; then
                break
            fi
        fi
        retries=$((retries-1))
        if [ $retries -eq 0 ]; then
            log ERROR "Could not download: $1"
            exit 1
        fi
        sleep 1
    done
    set -e
}

add_repo() {
    local url=$1
    set +e
    local retries=120
    local rc
    while [ $retries -gt 0 ]; do
        if [ "$PACKAGER" == "apt" ]; then
            curl -fSsL "$url" | apt-key add -
            rc=$?
        elif [ "$PACKAGER" == "yum" ]; then
            yum-config-manager --add-repo "$url"
            rc=$?
        elif [ "$PACKAGER" == "zypper" ]; then
            zypper addrepo "$url"
            rc=$?
        fi
        if [ $rc -eq 0 ]; then
            break
        fi
        retries=$((retries-1))
        if [ $retries -eq 0 ]; then
            log ERROR "Could not add repo: $url"
            exit 1
        fi
        sleep 1
    done
    set -e
}

refresh_package_index() {
    set +e
    local retries=120
    local rc
    while [ $retries -gt 0 ]; do
        if [ "$PACKAGER" == "apt" ]; then
            apt-get update
            rc=$?
        elif [ "$PACKAGER" == "yum" ]; then
            yum makecache -y fast
            rc=$?
        elif [ "$PACKAGER" == "zypper" ]; then
            zypper -n --gpg-auto-import-keys ref
            rc=$?
        fi
        if [ $rc -eq 0 ]; then
            break
        fi
        retries=$((retries-1))
        if [ $retries -eq 0 ]; then
            log ERROR "Could not update package index"
            exit 1
        fi
        sleep 1
    done
    set -e
}

install_packages() {
    set +e
    local retries=120
    local rc
    while [ $retries -gt 0 ]; do
        if [ "$PACKAGER" == "apt" ]; then
            apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends "$@"
            rc=$?
        elif [ "$PACKAGER" == "yum" ]; then
            yum install -y "$@"
            rc=$?
        elif [ "$PACKAGER" == "zypper" ]; then
            zypper -n in "$@"
            rc=$?
        fi
        if [ $rc -eq 0 ]; then
            break
        fi
        retries=$((retries-1))
        if [ $retries -eq 0 ]; then
            log ERROR "Could not install packages ($PACKAGER): $*"
            exit 1
        else
            # extend retries (by sleeping longer in-between) in the case of packager lock contention
            if [ "$PACKAGER" == "apt" ]; then
                if lsof /var/lib/dpkg/lock-frontend; then
                    sleep 2
                fi
            fi
        fi
        sleep 1
    done
    set -e
}

install_local_packages() {
    set +e
    local retries=120
    local rc
    while [ $retries -gt 0 ]; do
        if [ "$PACKAGER" == "apt" ]; then
            dpkg -i "$@"
            rc=$?
        else
            rpm -Uvh --nodeps "$@"
            rc=$?
        fi
        if [ $rc -eq 0 ]; then
            break
        fi
        retries=$((retries-1))
        if [ $retries -eq 0 ]; then
            log ERROR "Could not install local packages ($PACKAGER): $*"
            exit 1
        else
            # extend retries (by sleeping longer in-between) in the case of packager lock contention
            if [ "$PACKAGER" == "apt" ]; then
                if lsof /var/lib/dpkg/lock-frontend; then
                    sleep 2
                fi
            fi
        fi
        sleep 1
    done
    set -e
}

execute_command_with_retry() {
    set +e
    local retries=$1
    shift
    local fatal=$1
    shift
    local rc
    while [ "$retries" -gt 0 ]; do
        "$@"
        rc=$?
        if [ "$rc" -eq 0 ]; then
            break
        fi
        retries=$((retries-1))
        if [ $retries -eq 0 ]; then
            log ERROR "Could not execute command: $*"
            if [ "$fatal" -eq 1 ]; then
                exit $rc
            else
                break
            fi
        fi
        sleep 1
    done
    set -e
}

# TODO remove this once native images are fixed
blacklist_kernel_upgrade() {
    if [ "$DISTRIB_ID" != "ubuntu" ]; then
        log DEBUG "No kernel upgrade blacklist required on $DISTRIB_ID $DISTRIB_RELEASE"
        return
    fi
    set +e
    grep linux-azure /etc/apt/apt.conf.d/50unattended-upgrades
    local rc=$?
    set -e
    if [ $rc -ne 0 ]; then
        sed -i "/^Unattended-Upgrade::Package-Blacklist {/a\"linux-azure\";\\n\"linux-cloud-tools-azure\";\\n\"linux-headers-azure\";\\n\"linux-image-azure\";\\n\"linux-tools-azure\";" /etc/apt/apt.conf.d/50unattended-upgrades
        log INFO "Added linux-azure to package blacklist for unattended upgrades"
    fi
}

check_for_nvidia_container_runtime() {
    set +e
    if ! command -v nvidia-container-cli > /dev/null 2>&1; then
        log ERROR "nvidia container runtime not installed"
        exit 1
    fi
    set -e
}

check_for_nvidia_driver_on_custom_or_native() {
    set +e
    local out
    out=$(lsmod)
    echo "$out" | grep -i nvidia > /dev/null
    local rc=$?
    set -e
    echo "$out"
    if [ $rc -ne 0 ]; then
        log ERROR "No Nvidia drivers detected!"
        exit 1
    else
        check_for_nvidia_container_runtime
    fi
}

enable_nvidia_persistence_mode() {
    execute_command_with_retry 5 0 nvidia-persistenced --user root
    nvidia-smi -pm 1
}

query_nvidia_card() {
    if [ "$SHIPYARD_GPU_IGNORE_WARNINGS" -eq 1 ]; then
        set +e
    fi
    nvidia-smi -q
    nvidia-smi
    if [ "$SHIPYARD_GPU_IGNORE_WARNINGS" -eq 1 ]; then
        set -e
    fi
}

check_for_nvidia_on_custom_or_native() {
    log INFO "Checking for Nvidia Hardware"
    # first check for card
    set +e
    local out
    out=$(lspci)
    echo "$out" | grep -i nvidia > /dev/null
    local rc=$?
    set -e
    echo "$out"
    if [ $rc -ne 0 ]; then
        log INFO "No Nvidia card(s) detected!"
    else
        check_for_nvidia_driver_on_custom_or_native
        # prevent kernel upgrades from breaking driver
        blacklist_kernel_upgrade
        enable_nvidia_persistence_mode
        query_nvidia_card
    fi
}

ensure_nvidia_driver_installed() {
    check_for_nvidia_card
    # ensure that nvidia drivers are loaded
    set +e
    local out
    out=$(lsmod)
    echo "$out" | grep -i nvidia > /dev/null
    local rc=$?
    set -e
    echo "$out"
    if [ $rc -ne 0 ]; then
        log WARNING "Nvidia driver not present!"
        install_nvidia_software
    else
        log INFO "Nvidia driver detected"
        enable_nvidia_persistence_mode
        query_nvidia_card
    fi
}

check_for_nvidia_card() {
    set +e
    local out
    out=$(lspci)
    echo "$out" | grep -i nvidia > /dev/null
    local rc=$?
    set -e
    echo "$out"
    if [ $rc -ne 0 ]; then
        log ERROR "No Nvidia card(s) detected!"
        exit 1
    fi
}

install_lis() {
    if [ -z "$lis" ]; then
        log INFO "LIS installation not required"
        return
    fi
    if [ -f "$lisinstalled" ]; then
        log INFO "Assuming LIS installed with file presence"
        return
    fi
    # lis install is controlled by variable presence driven from fleet
    log DEBUG "Installing LIS"
    tar zxpf "$lis"
    pushd LISISO
    ./install.sh
    popd
    touch "$lisinstalled"
    rm -rf LISISO "$lis"
    log INFO "LIS installed, rebooting"
    reboot
}

install_kernel_devel_package() {
    if [[ $DISTRIB_ID == centos* ]]; then
        local kernel_devel_package
        local installed=0
        kernel_devel_package="kernel-devel-$(uname -r)"
        set +e
        if ! yum list installed "${kernel_devel_package}"; then
            set -e
            local centos_ver
            centos_ver=$(cut -d' ' -f 4 /etc/centos-release)
            if [ "$vm_rdma_type" -ne 0 ]; then
                # HPC distros have pinned repos
                install_packages "${kernel_devel_package}"
                installed=1
            fi
            if [ "$installed" -eq 0 ]; then
                if [[ "$centos_ver" == 7.3.* ]] || [[ "$centos_ver" == 7.4.* ]] || [[ "$centos_ver" == 7.5.* ]] || [[ "$centos_ver" == 7.6.* ]]; then
                    local pkg
                    pkg="${kernel_devel_package}.rpm"
                    download_file_as "http://vault.centos.org/${centos_ver}/updates/x86_64/Packages/${pkg}" "$pkg"
                    install_local_packages "$pkg"
                else
                    install_packages "${kernel_devel_package}"
                fi
            fi
        fi
        set -e
    fi
}

install_nvidia_software() {
    log INFO "Installing Nvidia Software"
    # check for nvidia card
    check_for_nvidia_card
    # split arg into two
    local OLD_IFS=$IFS
    IFS=':' read -ra GPUARGS <<< "$gpu"
    IFS=$OLD_IFS
    local is_viz=${GPUARGS[0]}
    local nvdriver=${GPUARGS[1]}
    # remove nouveau
    set +e
    rmmod nouveau
    # purge nouveau off system
    if [ "$DISTRIB_ID" == "ubuntu" ]; then
        apt-get --purge remove xserver-xorg-video-nouveau "xserver-xorg-video-nouveau-hwe-${DISTRIB_RELEASE}"
    elif [[ $DISTRIB_ID == centos* ]]; then
        yum erase -y xorg-x11-drv-nouveau
    else
        log ERROR "unsupported distribution for nvidia/GPU: $DISTRIB_ID $DISTRIB_RELEASE"
        exit 1
    fi
    set -e
    # blacklist nouveau from being loaded if rebooted
    local blfile
    if [ "$DISTRIB_ID" == "ubuntu" ]; then
        blfile=/etc/modprobe.d/blacklist-nouveau.conf
    elif [[ $DISTRIB_ID == centos* ]]; then
        blfile=/etc/modprobe.d/blacklist.conf
    fi
cat >> "$blfile" << EOF
blacklist nouveau
blacklist lbm-nouveau
options nouveau modeset=0
alias nouveau off
alias lbm-nouveau off
EOF
    # get development essentials for nvidia driver
    if [ "$DISTRIB_ID" == "ubuntu" ]; then
        install_packages build-essential dkms
    elif [[ $DISTRIB_ID == centos* ]]; then
        dracut /boot/initramfs-"$(uname -r)".img "$(uname -r)" --force
        install_kernel_devel_package
        install_packages gcc binutils make epel-release
        install_packages dkms
    fi
    # get additional dependency if NV-series VMs
    if [ "$is_viz" == "True" ]; then
        if [ "$DISTRIB_ID" == "ubuntu" ]; then
            install_packages xserver-xorg-dev
        elif [[ $DISTRIB_ID == centos* ]]; then
            install_packages xorg-x11-server-devel
        fi
    fi
    # install driver
    ./"${nvdriver}" -s --dkms
    # add flag to config for GRID driver
    if [ "$is_viz" == "True" ]; then
        cp /etc/nvidia/gridd.conf.template /etc/nvidia/gridd.conf
        echo "IgnoreSP=TRUE" >> /etc/nvidia/gridd.conf
    fi
    # enable persistence daemon (and mode)
    enable_nvidia_persistence_mode
    # install nvidia-docker
    if [ "$DISTRIB_ID" == "ubuntu" ]; then
        add_repo https://nvidia.github.io/nvidia-docker/gpgkey
        curl -fSsL "https://nvidia.github.io/nvidia-docker/ubuntu${DISTRIB_RELEASE}/nvidia-docker.list" | \
            tee /etc/apt/sources.list.d/nvidia-docker.list
    elif [[ $DISTRIB_ID == centos* ]]; then
        add_repo "https://nvidia.github.io/nvidia-docker/centos${DISTRIB_RELEASE}/nvidia-docker.repo"
    fi
    refresh_package_index
    install_packages nvidia-container-toolkit
    # merge daemon configs if necessary
    set +e
    grep \"data-root\" /etc/docker/daemon.json
    local rc=$?
    set -e
    if [ $rc -ne 0 ]; then
        systemctl stop docker.service
        log DEBUG "data-root not detected in Docker daemon.json"
        if [ "$DISTRIB_ID" == "ubuntu" ]; then
            python -c "import json;a=json.load(open('/etc/docker/daemon.json.dpkg-old'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
            rm -f /etc/docker/daemon.json.dpkg-old
        elif [[ $DISTRIB_ID == centos* ]]; then
            echo "{ $docker_group \"data-root\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json.merge
            python -c "import json;a=json.load(open('/etc/docker/daemon.json.merge'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
            rm -f /etc/docker/daemon.json.merge
        fi
        # ensure no options are specified after dockerd
        sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' "${SYSTEMD_PATH}"/docker.service
        systemctl daemon-reload
        systemctl start docker.service
    else
        systemctl restart docker.service
    fi
    systemctl --no-pager status docker.service
    check_for_nvidia_container_runtime
    check_docker_root_dir
    query_nvidia_card
}

mount_azurefile_share() {
    log INFO "Mounting Azure File Shares"
    chmod 700 azurefile-mount.sh
    chown root:root azurefile-mount.sh
    ./azurefile-mount.sh
    rm azurefile-mount.sh
}

mount_azureblob_container() {
    log INFO "Mounting Azure Blob Containers"
    chmod 700 azureblob-mount.sh
    chown root:root azureblob-mount.sh
    # check if blobfuse is installed
    local blobfuse_installed=0
    set +e
    if [ "$PACKAGER" == "apt" ]; then
        if dpkg -s blobfuse | grep "^Status:.* installed$"; then
            blobfuse_installed=1
        fi
    elif [ "$PACKAGER" == "yum" ]; then
        if yum list installed blobfuse; then
            blobfuse_installed=1
        fi
    elif [ "$PACKAGER" == "zypper" ]; then
        if rpm -q blobfuse; then
            blobfuse_installed=1
        fi
    fi
    set -e
    if [ "$blobfuse_installed" -eq 1 ]; then
        log DEBUG "blobfuse is installed"
    else
        log DEBUG "blobfuse is not installed"
        local mspkg
        if [ "$PACKAGER" == "apt" ]; then
            mspkg=packages-microsoft-prod.deb
            if [ "$DISTRIB_ID" == "ubuntu" ]; then
                download_file_as "https://packages.microsoft.com/config/${DISTRIB_ID}/${DISTRIB_RELEASE}/${mspkg}" "$mspkg"
            elif [ "$DISTRIB_ID" == "debian" ]; then
                install_packages apt-transport-https
                if [ "$DISTRIB_RELEASE" == "9" ]; then
                    download_file_as "https://packages.microsoft.com/config/ubuntu/16.04/${mspkg}" "$mspkg"
                fi
            fi
        elif [ "$PACKAGER" == "yum" ]; then
            mspkg=packages-microsoft-prod.rpm
            download_file_as "https://packages.microsoft.com/config/rhel/${DISTRIB_RELEASE}/${mspkg}" "$mspkg"
        elif [ "$PACKAGER" == "zypper" ]; then
            mspkg=packages-microsoft-prod.rpm
            download_file_as "https://packages.microsoft.com/config/sles/${DISTRIB_RELEASE}/${mspkg}" "$mspkg"
        fi
        if [ ! -f ${mspkg} ]; then
            echo "ERROR: unsupported distribution for Azure blob: $DISTRIB_ID $DISTRIB_RELEASE"
            exit 1
        fi
        install_local_packages ${mspkg}
        refresh_package_index
        install_packages blobfuse
    fi
    ./azureblob-mount.sh
    rm azureblob-mount.sh
}

docker_pull_image_fallback() {
    local image="${fallback_registry}/${1}"
    log DEBUG "Pulling Docker Image through fallback registry: $image"
    set +e
    local retries=25
    while [ $retries -gt 0 ]; do
        local pull_out
        pull_out=$(docker pull "$image" 2>&1)
        local rc=$?
        if [ $rc -eq 0 ]; then
            echo "$pull_out"
            break
        fi
        # non-zero exit code: check if pull output has toomanyrequests,
        # connection resets, or image config error
        local tmr
        tmr=$(grep -i 'toomanyrequests' <<<"$pull_out")
        local crbp
        crbp=$(grep -i 'connection reset by peer' <<<"$pull_out")
        local epic
        epic=$(grep -i 'error pulling image configuration' <<<"$pull_out")
        local erb
        erb=$(grep -i 'error parsing HTTP 404 response body' <<<"$pull_out")
        local uhs
        uhs=$(grep -i 'received unexpected HTTP status' <<<"$pull_out")
        local tht
        tht=$(grep -i 'TLS handshake timeout' <<<"$pull_out")
        if [[ -n "$tmr" ]] || [[ -n "$crbp" ]] || [[ -n "$epic" ]] || [[ -n "$erb" ]] || [[ -n "$uhs" ]] || [[ -n "$tht" ]]; then
            log WARNING "will retry: $pull_out"
        else
            log ERROR "$pull_out"
            exit $rc
        fi
        retries=$((retries-1))
        if [ $retries -le 0 ]; then
            log ERROR "Could not pull docker image through fallback registry: $image"
            exit $rc
        fi
        sleep $((RANDOM % 5 + 1))s
    done
    set -e
    # re-tag image
    docker tag "$image" "$1"
    docker rmi "$image"
}

docker_pull_image() {
    local image=$1
    local try_fallback=0
    if [ -n "$fallback_registry" ]; then
        try_fallback=1
    fi
    local rc
    log DEBUG "Pulling Docker Image: $image (fallback: $try_fallback)"
    set +e
    local retries=25
    while [ $retries -gt 0 ]; do
        local pull_out
        pull_out=$(docker pull "$image" 2>&1)
        rc=$?
        if [ $rc -eq 0 ]; then
            echo "$pull_out"
            break
        fi
        # non-zero exit code: check if pull output has toomanyrequests,
        # connection resets, or image config error
        local tmr
        tmr=$(grep -i 'toomanyrequests' <<<"$pull_out")
        local crbp
        crbp=$(grep -i 'connection reset by peer' <<<"$pull_out")
        local epic
        epic=$(grep -i 'error pulling image configuration' <<<"$pull_out")
        local erb
        erb=$(grep -i 'error parsing HTTP 404 response body' <<<"$pull_out")
        local uhs
        uhs=$(grep -i 'received unexpected HTTP status' <<<"$pull_out")
        local tht
        tht=$(grep -i 'TLS handshake timeout' <<<"$pull_out")
        if [[ -n "$tmr" ]] || [[ -n "$crbp" ]] || [[ -n "$epic" ]] || [[ -n "$erb" ]] || [[ -n "$uhs" ]] || [[ -n "$tht" ]]; then
            log WARNING "will retry: $pull_out"
        else
            log ERROR "$pull_out"
            break
        fi
        retries=$((retries-1))
        if [ $retries -le 0 ]; then
            log ERROR "Could not pull docker image: $image"
            break
        fi
        sleep $((RANDOM % 5 + 1))s
    done
    set -e
    if [ $rc -ne 0 ]; then
        if [ $try_fallback -eq 1 ]; then
            docker_pull_image_fallback "$image"
        else
            log ERROR "No fallback registry specified, terminating"
            exit $rc
        fi
    fi
}

install_singularity() {
    if [ $native_mode -eq 1 ]; then
        log DEBUG "Skipping Singularity install on native"
        return
    fi
    if [ -z "$singularityversion" ]; then
        log DEBUG "Singularity not flagged for install"
        return
    fi
    local disuffix
    singularity_basedir="${USER_MOUNTPOINT}/singularity"
    if [ "${USER_MOUNTPOINT}" == "/mnt" ]; then
        disuffix=singularity-mnt
    else
        disuffix=singularity-mnt-resource
    fi
    log DEBUG "Setting up Singularity for $DISTRIB_ID $DISTRIB_RELEASE"
    # install prereqs
    local cryptpkg
    if [ "$PACKAGER" == "apt" ]; then
        cryptpkg=cryptsetup-bin
    elif [ "$PACKAGER" == "yum" ]; then
        cryptpkg=cryptsetup
    fi
    install_packages squashfs-tools $cryptpkg
    # fetch docker image for singularity bits
    local di="${SHIPYARD_IMAGE_PREFIX}:${singularityversion}-${disuffix}"
    log DEBUG "Image=${di} basedir=${singularity_basedir}"
    docker_pull_image "$di"
    mkdir -p /opt/singularity
    docker run --runtime runc --rm -v /opt/singularity:/singularity "$di" \
        /bin/sh -c 'cp -r /opt/singularity/ /'
    # symlink for global exec
    ln -sf /opt/singularity/bin/singularity /usr/bin/singularity
    # fix perms
    chown -R root:root /opt/singularity
    chmod 4755 /opt/singularity/libexec/singularity/bin/*-suid
    # prep singularity root/container dir
    mkdir -p "$singularity_basedir/mnt/container"
    mkdir -p "$singularity_basedir/mnt/final"
    mkdir -p "$singularity_basedir/mnt/overlay"
    mkdir -p "$singularity_basedir/mnt/session"
    chmod 755 "$singularity_basedir"
    chmod 755 "$singularity_basedir/mnt"
    chmod 755 "$singularity_basedir/mnt/container"
    chmod 755 "$singularity_basedir/mnt/final"
    chmod 755 "$singularity_basedir/mnt/overlay"
    chmod 755 "$singularity_basedir/mnt/session"
    # create singularity tmp/cache/sypgp paths
    mkdir -p "$singularity_basedir/tmp"
    mkdir -p "$singularity_basedir/cache"
    mkdir -p "$singularity_basedir/sypgp"
    chmod 775 "$singularity_basedir/tmp"
    chmod 775 "$singularity_basedir/cache"
    chmod 700 "$singularity_basedir/sypgp"
    # set proper ownership
    chown -R _azbatch:_azbatchgrp "$singularity_basedir/tmp"
    chown -R _azbatch:_azbatchgrp "$singularity_basedir/cache"
    chown -R _azbatch:_azbatchgrp "$singularity_basedir/sypgp"
    # ensure it runs
    singularity version
    # remove docker image
    docker rmi "$di"
}

install_kata_containers() {
    if [ $kata -eq 0 ]; then
        log DEBUG "Kata containers not flagged for install"
        return
    fi
    if [ $custom_image -eq 1 ]; then
        log DEBUG "Not installing Kata containers due to custom image"
        return
    fi
    logger DEBUG "Installing Kata containers"
    local ARCH
    local repo_url
    ARCH=$(arch)
    if [ "$DISTRIB_ID" == "ubuntu" ]; then
        repo_url="http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/master/xUbuntu_$(lsb_release -rs)/"
        echo "deb $repo_url /" > /etc/apt/sources.list.d/kata-containers.list
        repo_url="$repo_url/Release.key"
    elif [[ "$DISTRIB_ID" == "rhel" ]] || [[ $DISTRIB_ID == centos* ]]; then
        repo_url="http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/master/CentOS_${DISTRIB_RELEASE}/home:katacontainers:releases:${ARCH}:master.repo"
    else
        log WARNING "Kata containers not supported on $DISTRIB_ID $DISTRIB_RELEASE"
        return
    fi
    add_repo "$repo_url"
    refresh_package_index
    install_packages kata-runtime kata-proxy kata-shim
    # add to available runtimes
    local dcr
    if [ "$default_container_runtime" == "kata_containers" ]; then
        dcr="a['default-runtime']='kata-runtime';"
    fi
    python -c "import json;a=json.load(open('/etc/docker/daemon.json'));${dcr}b=a.get('runtimes',{});b['kata-runtime']={'path':'/usr/bin/kata-runtime'};a['runtimes']=b;f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
    # restart docker to pickup changes
    systemctl restart docker.service
    logger INFO "Kata containers installed"
}

process_fstab_entry() {
    local desc=$1
    local fstab_entry=$2
    local OLD_IFS=$IFS
    IFS=' ' read -ra fs <<< "$fstab_entry"
    IFS=$OLD_IFS
    local mountpoint="${fs[1]}"
    log INFO "Creating host directory for $desc at $mountpoint"
    mkdir -p "$mountpoint"
    chmod 777 "$mountpoint"
    echo "INFO: Adding $mountpoint to fstab"
    echo "$fstab_entry" >> /etc/fstab
    tail -n1 /etc/fstab
    echo "INFO: Mounting $mountpoint"
    local START
    START=$(date -u +"%s")
    set +e
    while :
    do
        if mount "$mountpoint"; then
            break
        else
            local NOW
            NOW=$(date -u +"%s")
            local DIFF=$(((NOW-START)/60))
            # fail after 5 minutes of attempts
            if [ $DIFF -ge 5 ]; then
                echo "ERROR: Could not mount $desc on $mountpoint"
                exit 1
            fi
            sleep 1
        fi
    done
    set -e
    log INFO "$mountpoint mounted."
}

mount_storage_clusters() {
    if [ -n "$SHIPYARD_STORAGE_CLUSTER_FSTAB" ]; then
        log DEBUG "Mounting storage clusters"
        local fstab_mounts
        local OLD_IFS=$IFS
        IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_STORAGE_CLUSTER_FSTAB"
        for fstab in "${fstab_mounts[@]}"; do
            # eval and split fstab var to expand vars
            fstab_entry=$(eval echo "$fstab")
            IFS=' ' read -ra parts <<< "$fstab_entry"
            mount "${parts[1]}"
        done
        IFS=$OLD_IFS
        log INFO "Storage clusters mounted"
    fi
}

process_storage_clusters() {
    if [ -n "${sc_args[0]}" ]; then
        log DEBUG "Processing storage clusters"
        # eval and split fstab var to expand vars (this is ok since it is set by shipyard)
        local fstab_mounts
        local OLD_IFS=$IFS
        fstab_mounts=$(eval echo "$SHIPYARD_STORAGE_CLUSTER_FSTAB")
        IFS='#' read -ra fstabs <<< "$fstab_mounts"
        i=0
        for sc_arg in "${sc_args[@]}"; do
            IFS=':' read -ra sc <<< "$sc_arg"
            fstab_entry="${fstabs[$i]}"
            process_fstab_entry "$sc_arg" "$fstab_entry"
            i=$((i + 1))
        done
        IFS=$OLD_IFS
        log INFO "Storage clusters processed"
    fi
}

mount_custom_fstab() {
    if [ -n "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" ]; then
        log DEBUG "Mounting custom mounts via fstab"
        local fstab_mounts
        local OLD_IFS=$IFS
        IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_CUSTOM_MOUNTS_FSTAB"
        for fstab in "${fstab_mounts[@]}"; do
            # eval and split fstab var to expand vars
            fstab_entry=$(eval echo "$fstab")
            IFS=' ' read -ra parts <<< "$fstab_entry"
            mount "${parts[1]}"
        done
        IFS=$OLD_IFS
        log INFO "Custom mounts via fstab mounted"
    fi
}

process_custom_fstab() {
    if [ -n "$SHIPYARD_CUSTOM_MOUNTS_FSTAB" ]; then
        log DEBUG "Processing custom mounts via fstab"
        local fstab_mounts
        local OLD_IFS=$IFS
        IFS='#' read -ra fstab_mounts <<< "$SHIPYARD_CUSTOM_MOUNTS_FSTAB"
        for fstab in "${fstab_mounts[@]}"; do
            # eval and split fstab var to expand vars
            fstab_entry=$(eval echo "$fstab")
            IFS=' ' read -ra parts <<< "$fstab_entry"
            process_fstab_entry "${parts[2]}" "$fstab_entry"
        done
        IFS=$OLD_IFS
        log INFO "Custom mounts via fstab processed"
    fi
}

convert_singularity_certificates() {
    # converts pfx certs to PKCS1 (RSA private) pem certs for singularity
    if [ -z "$SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES" ]; then
        log INFO "No singularity decryption certificates defined"
        return
    fi
    log INFO "Processing Singularity decryption certificates: $SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES"
    local OLD_IFS=$IFS
    IFS=',' read -ra cert_tps <<< "${SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES,,}"
    IFS=$OLD_IFS
    local pfx
    local pw
    local pkcs1
    pushd "$AZ_BATCH_CERTIFICATES_DIR"
    for tp in "${cert_tps[@]}"; do
        pfx="sha1-${tp}.pfx"
        pw="${pfx}.pw"
        if [ ! -f "$pfx" ] || [ ! -f "$pw" ]; then
            log ERROR "Certificate needed for Singularity image decryption not found: $pfx or $pw"
            exit 1
        fi
        pkcs1="sha1-${tp}-rsa.pem"
        openssl pkcs12 -in "$pfx" -nodes -password "file:${pw}" | openssl rsa -out "$pkcs1"
        chmod 640 "$pkcs1"
        chown _azbatch:_azbatchsudogrp "$pkcs1"
        # remove the pfx/pw files bound to start task
        rm -f "$pfx" "$pw"
    done
    popd
}

decrypt_encrypted_credentials() {
    # convert pfx to pem
    pfxfile=$AZ_BATCH_CERTIFICATES_DIR/sha1-$encrypted.pfx
    privatekey=$AZ_BATCH_CERTIFICATES_DIR/shipyard-enckey.pem
    openssl pkcs12 -in "$pfxfile" -out "$privatekey" -nodes -password "file:${pfxfile}.pw"
    # remove pfx-related files
    rm -f "$pfxfile" "${pfxfile}.pw"
    # decrypt creds
    SHIPYARD_STORAGE_ENV=$(echo "$SHIPYARD_STORAGE_ENV" | base64 -d | openssl rsautl -decrypt -inkey "$privatekey")
    if [[ -n ${DOCKER_LOGIN_USERNAME+x} ]]; then
        DOCKER_LOGIN_PASSWORD=$(echo "$DOCKER_LOGIN_PASSWORD" | base64 -d | openssl rsautl -decrypt -inkey "$privatekey")
    fi
}

check_for_docker_host_engine() {
    set +e
    # start docker service
    systemctl start docker.service
    systemctl --no-pager status docker.service
    if ! docker version; then
        log ERROR "Docker not installed"
        exit 1
    fi
    set -e
    docker info
}

check_docker_root_dir() {
    set +e
    local rootdir
    rootdir=$(awk -F' ' '{print $NF}' <<< "$(docker info | grep 'Docker Root Dir')")
    if echo "$rootdir" | grep "^$USER_MOUNTPOINT" > /dev/null; then
        log INFO "Docker root dir within ephemeral temp disk: $rootdir"
    else
        log WARNING "Docker root dir not within $USER_MOUNTPOINT: $rootdir"
    fi
    set -e
}

install_docker_host_engine() {
    log DEBUG "Installing Docker Host Engine"
    # set vars
    local srvstart="systemctl start docker.service"
    local srvstop="systemctl stop docker.service"
    local srvdisable="systemctl disable docker.service"
    local srvstatus="systemctl --no-pager status docker.service"
    if [ "$PACKAGER" == "apt" ]; then
        local repo="https://download.docker.com/linux/${DISTRIB_ID}"
        local gpgkey="${repo}/gpg"
        local docker_ce_pkg="docker-ce=${DOCKER_CE_PACKAGE_DEBIAN}${DISTRIB_ID}-${DISTRIB_CODENAME}"
        local docker_cli_pkg="docker-ce-cli=${DOCKER_CE_PACKAGE_DEBIAN}${DISTRIB_ID}-${DISTRIB_CODENAME}"
        local prereq_pkgs="apt-transport-https ca-certificates curl gnupg2 software-properties-common"
    elif [ "$PACKAGER" == "yum" ]; then
        local repo=https://download.docker.com/linux/centos/docker-ce.repo
        local docker_ce_pkg="docker-ce-${DOCKER_CE_PACKAGE_CENTOS}"
        local docker_cli_pkg="docker-ce-cli-${DOCKER_CE_PACKAGE_CENTOS}"
        local prereq_pkgs="yum-utils device-mapper-persistent-data lvm2"
    elif [ "$PACKAGER" == "zypper" ]; then
        if [[ "$DISTRIB_RELEASE" == 12-sp3* ]]; then
            local repodir=SLE_12_SP3
        fi
        local repo="http://download.opensuse.org/repositories/Virtualization:containers/${repodir}/Virtualization:containers.repo"
        local docker_ce_pkg="${DOCKER_CE_PACKAGE_SLES}"
        local docker_cli_pkg
    fi
    # refresh package index
    refresh_package_index
    # install required software first
    # shellcheck disable=SC2086
    install_packages $prereq_pkgs
    if [ "$PACKAGER" == "apt" ]; then
        # add gpgkey for repo
        add_repo "$gpgkey"
        # add repo
        add-apt-repository "deb [arch=amd64] $repo $(lsb_release -cs) stable"
    else
        add_repo "$repo"
    fi
    # refresh index
    refresh_package_index
    # install docker engine
    install_packages "$docker_ce_pkg" "$docker_cli_pkg"
    # disable docker from auto-start due to temp disk issues
    $srvstop
    $srvdisable
    # ensure docker daemon modifications are idempotent
    rm -rf /var/lib/docker
    mkdir -p /etc/docker
    if [ "$PACKAGER" == "apt" ]; then
        echo "{ $docker_group \"data-root\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
    else
        echo "{ $docker_group \"data-root\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
    fi
    # ensure no options are specified after dockerd
    sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' "${SYSTEMD_PATH}"/docker.service
    systemctl daemon-reload
    $srvstart
    $srvstatus
    docker info
    log INFO "Docker Host Engine installed"
}

check_for_glusterfs_on_compute() {
    set +e
    if ! gluster; then
        log ERROR "gluster client not installed"
        exit 1
    fi
    if ! glusterfs -V; then
        log ERROR "gluster server not installed"
        exit 1
    fi
    set -e
}

install_beeond() {
    if [ $beeond -eq 0 ]; then
        log DEBUG "BeeGFS BeeOND not flagged for install"
        return
    fi
    if [ $custom_image -eq 1 ]; then
        log DEBUG "Not installing BeeGFS BeeOND due to custom image"
        return
    fi
    logger DEBUG "Installing BeeGFS BeeOND"
    local led
    local pkgnum
    if [ "$PACKAGER" == "apt" ]; then
        if { [ "$DISTRIB_ID" == "debian" ] && [ "$DISTRIB_RELEASE" == "9" ]; } || { [ "$DISTRIB_ID" == "ubuntu" ] && [ "$DISTRIB_RELEASE" == "16.04" ]; } then
            pkgnum=9
        elif [ "$DISTRIB_ID" == "ubuntu" ] && [ "$DISTRIB_RELEASE" == "18.04" ]; then
            logger ERROR "BeeGFS BeeOND is not supported on Ubuntu 18.04"
            exit 1
        fi
        download_file_as "https://www.beegfs.io/release/latest-stable/dists/beegfs-deb${pkgnum}.list" "/etc/apt/sources.list.d/beegfs-deb${pkgnum}.list"
        add_repo "https://www.beegfs.io/release/latest-stable/gpg/DEB-GPG-KEY-beegfs"
        led=libelf-dev
    elif [ "$PACKAGER" == "yum" ]; then
        if [[ "$DISTRIB_RELEASE" == 7* ]]; then
            pkgnum=7
        elif [[ "$DISTRIB_RELEASE" == 8* ]]; then
            pkgnum=8
        fi
        download_file_as "https://www.beegfs.io/release/latest-stable/dists/beegfs-rhel${pkgnum}.repo" "/etc/yum.repos.d/beegfs-rhel${pkgnum}.repo"
        rpm --import "https://www.beegfs.io/release/latest-stable/gpg/RPM-GPG-KEY-beegfs"
        led=elfutils-libelf-devel
        install_kernel_devel_package
        install_packages epel-release
    fi
    refresh_package_index
    install_packages dkms beeond $led
    logger INFO "BeeGFS BeeOND installed"
}

install_glusterfs_on_compute() {
    log DEBUG "Installing GlusterFS on compute"
    local gfsstart="systemctl start glusterd"
    local gfsenable="systemctl enable glusterd"
    if [ "$PACKAGER" == "zypper" ]; then
        if [[ "$DISTRIB_RELEASE" == 12-sp3* ]]; then
            local repodir=SLE_12_SP3
        fi
        local repo="http://download.opensuse.org/repositories/filesystems/${repodir}/filesystems.repo"
    fi
    if [ "$PACKAGER" == "apt" ]; then
        if [ "$DISTRIB_ID" == "debian" ]; then
            add_repo "http://download.gluster.org/pub/gluster/glusterfs/${GLUSTER_VERSION_DEBIAN}/rsa.pub"
        else
            add-apt-repository ppa:gluster/glusterfs-${GLUSTER_VERSION_DEBIAN}
        fi
        install_packages glusterfs-server
    elif [ "$PACKAGER" == "yum" ]; then
        install_packages centos-release-gluster${GLUSTER_VERSION_CENTOS}
        install_packages glusterfs-server acl
    elif [ "$PACKAGER" == "zypper" ]; then
        add_repo "$repo"
        "$PACKAGER" -n --gpg-auto-import-keys ref
        install_packages glusterfs
    fi
    $gfsenable
    $gfsstart
    # create brick directory
    mkdir -p "${USER_MOUNTPOINT}"/gluster
    log INFO "GlusterFS on compute installed"
}

check_for_storage_cluster_software() {
    if [ -z "${sc_args[0]}" ]; then
        return
    fi
    local rc
    for sc_arg in "${sc_args[@]}"; do
        local OLD_IFS=$IFS
        IFS=':' read -ra sc <<< "$sc_arg"
        IFS=$OLD_IFS
        local server_type=${sc[0]}
        if [ "$server_type" == "nfs" ]; then
            set +e
            mount.nfs4 -V
            rc=$?
            set -e
        elif [ "$server_type" == "glusterfs" ]; then
            set +e
            glusterfs -V
            rc=$?
            set -e
        else
            log ERROR "Unknown file server type ${sc[0]} for ${sc[1]}"
            exit 1
        fi
        if [ $rc -ne 0 ]; then
            log ERROR "Required storage cluster software to mount $server_type not installed"
            exit 1
        fi
    done
}

install_storage_cluster_dependencies() {
    if [ -z "${sc_args[0]}" ]; then
        return
    fi
    log DEBUG "Installing storage cluster dependencies"
    if [ "$PACKAGER" == "zypper" ]; then
        if [[ "$DISTRIB_RELEASE" == 12-sp3* ]]; then
            local repodir=SLE_12_SP3
        fi
        local repo="http://download.opensuse.org/repositories/filesystems/${repodir}/filesystems.repo"
    fi
    for sc_arg in "${sc_args[@]}"; do
        local OLD_IFS=$IFS
        IFS=':' read -ra sc <<< "$sc_arg"
        IFS=$OLD_IFS
        server_type=${sc[0]}
        if [ "$server_type" == "nfs" ]; then
            if [ "$PACKAGER" == "apt" ]; then
                install_packages nfs-common nfs4-acl-tools
            elif [ "$PACKAGER" == "yum" ] ; then
                install_packages nfs-utils nfs4-acl-tools
                systemctl enable rpcbind
                systemctl start rpcbind
            elif [ "$PACKAGER" == "zypper" ]; then
                install_packages nfs-client nfs4-acl-tools
                systemctl enable rpcbind
                systemctl start rpcbind
            fi
        elif [ "$server_type" == "glusterfs" ]; then
            if [ "$PACKAGER" == "apt" ]; then
                if [ "$DISTRIB_ID" == "debian" ]; then
                    add_repo "http://download.gluster.org/pub/gluster/glusterfs/${GLUSTER_VERSION_DEBIAN}/rsa.pub"
                else
                    add-apt-repository ppa:gluster/glusterfs-${GLUSTER_VERSION_DEBIAN}
                fi
                install_packages glusterfs-client acl
            elif [ "$PACKAGER" == "yum" ] ; then
                install_packages centos-release-gluster${GLUSTER_VERSION_CENTOS}
                install_packages glusterfs-server acl
            elif [ "$PACKAGER" == "zypper" ]; then
                add_repo "$repo"
                "$PACKAGER" -n --gpg-auto-import-keys ref
                install_packages glusterfs acl
            fi
        else
            log ERROR "Unknown file server type ${sc[0]} for ${sc[1]}"
            exit 1
        fi
    done
    log INFO "Storage cluster dependencies installed"
}

install_cascade_dependencies() {
    if [ $cascadecontainer -ne 0 ]; then
        return
    fi
    log DEBUG "Installing dependencies for cascade on host"
    # install azure storage python dependency
    install_packages build-essential libssl-dev libffi-dev libpython3-dev python3-dev
    download_file_as https://bootstrap.pypa.io/get-pip.py get-pip.py
    python3 get-pip.py
    rm -f get-pip.py
    pip3 install --no-cache-dir --upgrade wheel setuptools
    pip3 install --no-cache-dir -r requirements.txt
    log INFO "Cascade on host dependencies installed"
}

install_intel_mpi() {
    if [ $custom_image -eq 1 ]; then
        log DEBUG "Not installing Intel MPI due to custom image"
        return
    fi
    if [ "$vm_rdma_type" -eq 2 ]; then
        if [ ! -d /opt/intel/compilers_and_libraries/linux/mpi ]; then
            log DEBUG "Installing Intel MPI"
            if [[ "$DISTRIB_ID" == sles* ]]; then
                install_packages lsb
                install_local_packages /opt/intelMPI/intel_mpi_packages/*.rpm
                mkdir -p /opt/intel/compilers_and_libraries/linux
                ln -sf /opt/intel/impi/5.0.3.048 /opt/intel/compilers_and_libraries/linux/mpi
            else
                if [ ! -f "${AZ_BATCH_TASK_WORKING_DIR}/intel_mpi_rt.tar.gz" ]; then
                    log ERROR "VM size $vm_size is RDMA capable and Intel MPI Runtime installer is missing"
                    exit 1
                fi
                pushd "$AZ_BATCH_TASK_WORKING_DIR"
                mkdir tmp
                cd tmp
                tar zxpf ../intel_mpi_rt.tar.gz
                cd l_mpi-rt*
                sed -i -e 's/^ACCEPT_EULA=decline/ACCEPT_EULA=accept/g' silent.cfg
                sed -i -e 's,^PSET_INSTALL_DIR=.*,PSET_INSTALL_DIR=/opt/intel,g' silent.cfg
                ./install.sh -s silent.cfg
                cd ../..
                rm -rf tmp
                popd
            fi
            log INFO "Intel MPI installed"
        fi
        # check for intel mpi
        if [ -f /opt/intel/compilers_and_libraries/linux/mpi/bin64/mpivars.sh ]; then
            log INFO "Intel MPI found"
        else
            log ERROR "Intel MPI not found"
            exit 1
        fi
    else
        log INFO "Not installing Intel MPI for RDMA type: $vm_rdma_type"
    fi
}

export_ib_pkey()
{
    key0=$(cat /sys/class/infiniband/mlx5_0/ports/1/pkeys/0)
    key1=$(cat /sys/class/infiniband/mlx5_0/ports/1/pkeys/1)

    if [ $((key0 - key1)) -gt 0 ]; then
        IB_PKEY=$key0
    else
        IB_PKEY=$key1
    fi
cat > "$IB_PKEY_FILE" << EOF
IB_PKEY=$IB_PKEY
EOF
    UCX_IB_PKEY=$(printf '0x%04x' "$((IB_PKEY & 0x0FFF))")
cat > "$UCX_IB_PKEY_FILE" << EOF
UCX_IB_PKEY=$UCX_IB_PKEY
EOF
}

check_for_mellanox_card() {
    if [ "$vm_rdma_type" -eq 0 ]; then
        log DEBUG "Not an RDMA capable VM size, skipping IB detection/setup"
        return
    fi
    # check for mellanox card
    set +e
    local out
    out=$(lspci)
    echo "$out" | grep -i mellanox > /dev/null
    local rc=$?
    set -e
    echo "$out"
    if [ $rc -ne 0 ]; then
        if [ "$vm_rdma_type" -eq 1 ]; then
            log ERROR "Expected Mellanox IB card not detected"
            exit 1
        fi
    fi
    if [ "$vm_rdma_type" -eq 1 ]; then
        # extract IB PKEY
        export_ib_pkey
        # get ofed info
        ofed_info
        # run ib tools
        ibstatus
        ibv_devinfo
    elif [ "$vm_rdma_type" -eq 2 ]; then
        # network direct rmda
        install_intel_mpi
    fi
}

process_ib_rdma() {
    if [ "$vm_rdma_type" -eq 0 ]; then
        log DEBUG "VM size $vm_size does not have IB RDMA"
    fi
    check_for_mellanox_card
    ip addr
}

setup_cascade() {
    envfile=$AZ_BATCH_NODE_STARTUP_DIR/wd/.cascade_envfile
    if [ $cascadecontainer -eq 1 ]; then
        # store shipyard docker pull start
        if command -v python3 > /dev/null 2>&1; then
            drpstart=$(python3 -c 'import datetime;print(datetime.datetime.utcnow().timestamp())')
        else
            drpstart=$(python -c 'import datetime;import time;print(time.mktime(datetime.datetime.utcnow().timetuple()))')
        fi
        log DEBUG "Pulling $cascade_docker_image"
        docker_pull_image "$cascade_docker_image"
        if [ -n "$singularity_basedir" ]; then
            log DEBUG "Pulling $cascade_singularity_image"
            docker_pull_image "$cascade_singularity_image"
        fi
        # store shipyard pull end
        if command -v python3 > /dev/null 2>&1; then
            drpend=$(python3 -c 'import datetime;print(datetime.datetime.utcnow().timestamp())')
        else
            drpend=$(python -c 'import datetime;import time;print(time.mktime(datetime.datetime.utcnow().timetuple()))')
        fi
        # create env file
cat > "$envfile" << EOF
PYTHONASYNCIODEBUG=1
prefix=$prefix
offer=$DISTRIB_ID
sku=$DISTRIB_RELEASE
npstart=$npstart
npend=$npend
drpstart=$drpstart
drpend=$drpend
concurrent_source_downloads=$concurrent_source_downloads
$(env | grep SHIPYARD_)
$(env | grep AZ_BATCH_)
$(env | grep DOCKER_LOGIN_)
$(env | grep SINGULARITY_)
EOF
    else
        # add timings
        if [[ -n ${SHIPYARD_TIMING+x} ]]; then
            # backfill node prep start
            # shellcheck disable=SC2086
            ./perf.py nodeprep start --prefix "$prefix" --ts "$npstart" --message "offer=$DISTRIB_ID,sku=$DISTRIB_RELEASE"
            # mark node prep finished
            # shellcheck disable=SC2086
            ./perf.py nodeprep end --prefix "$prefix" --ts "$npend"
        fi
        # create env file
cat > "$envfile" << EOF
concurrent_source_downloads=$concurrent_source_downloads
$(env | grep SHIPYARD_)
$(env | grep AZ_BATCH_)
$(env | grep DOCKER_LOGIN_)
$(env | grep SINGULARITY_)
EOF
    fi
    chmod 600 "$envfile"
}

run_cascade() {
    local d
    if [ $cascadecontainer -ne 0 ]; then d="-d"; else d=""; fi
    log_directory=$(pwd)
    set +e
    "${AZ_BATCH_TASK_WORKING_DIR}"/shipyard_cascade.sh \
        -b "$block" \
        -c "$concurrent_source_downloads" \
        $d \
        -e "$envfile" \
        -i "$cascade_docker_image" \
        -j "$cascade_singularity_image" \
        -l "$log_directory" \
        -p "$prefix" \
        -s "$singularity_basedir" \
        -t
    rc=$?
    if [ $rc -ne 0 ]; then
        rm -f "$nodeprepfinished"
        exit $rc
    fi
    set -e
}

install_and_start_node_exporter() {
    if [ -z "${PROM_NODE_EXPORTER_PORT}" ]; then
        log INFO "Prometheus node exporter disabled."
        return
    else
        log DEBUG "Installing Prometheus node exporter"
    fi
    # install
    tar zxpf node_exporter.tar.gz
    mv node_exporter-*.linux-amd64/node_exporter .
    rm -rf node_exporter-*.linux-amd64 node_exporter.tar.gz
    chmod +x node_exporter
    # start
    local ib
    local nfs
    nfs="--no-collector.nfs"
    if [ $vm_rdma_type -eq 1 ]; then
        ib="--collector.infiniband"
    else
        ib="--no-collector.infiniband"
    fi
    local OLD_IFS=$IFS
    if [ -n "${sc_args[0]}" ]; then
        for sc_arg in "${sc_args[@]}"; do
            IFS=':' read -ra sc <<< "$sc_arg"
            if [ "${sc[0]}" == "nfs" ]; then
                nfs="--collector.nfs --collector.mountstats"
                break
            fi
        done
        IFS=$OLD_IFS
    fi
    local pneo
    OLD_IFS=$IFS
    IFS=',' read -ra pneo <<< "$PROM_NODE_EXPORTER_OPTIONS"
    IFS=$OLD_IFS
    # shellcheck disable=SC2086
    "${AZ_BATCH_TASK_WORKING_DIR}"/node_exporter \
        "$ib" $nfs \
        --no-collector.textfile \
        --no-collector.mdadm \
        --no-collector.wifi \
        --no-collector.xfs \
        --no-collector.zfs \
        --web.listen-address=":${PROM_NODE_EXPORTER_PORT}" \
        --collector.filesystem.ignored-mount-points="${USER_MOUNTPOINT}/docker" \
        "${pneo[@]}" &
    log INFO "Prometheus node exporter enabled."
}

install_and_start_cadvisor() {
    if [ -z "${PROM_CADVISOR_PORT}" ]; then
        log INFO "Prometheus cAdvisor disabled."
        return
    else
        log INFO "Installing Prometheus cAdvisor"
    fi
    # install
    gunzip -f cadvisor.gz
    chmod +x cadvisor
    # start
    local pcao
    if [ -n "${PROM_CADVISOR_OPTIONS}" ]; then
        local OLD_IFS=$IFS
        IFS=',' read -ra pcao <<< "$PROM_CADVISOR_OPTIONS"
        IFS=$OLD_IFS
    else
        pcao=()
    fi
    "${AZ_BATCH_TASK_WORKING_DIR}"/cadvisor \
        -port "${PROM_CADVISOR_PORT}" \
        "${pcao[@]}" &
    log INFO "Prometheus cAdvisor enabled."
}

install_and_start_batch_insights() {
    if [ $batch_insights -eq 0 ]; then
        log INFO "Batch Insights disabled."
        return
    else
        log INFO "Installing Batch Insights"
    fi
    chmod +x batch-insights
    ./batch-insights > node-stats.log &
    log INFO "Batch Insights enabled."
}

log INFO "Prep start"
echo "Configuration:"
echo "--------------"
echo "Custom image: $custom_image"
echo "Native mode: $native_mode"
echo "OS Distribution: $DISTRIB_ID $DISTRIB_RELEASE"
echo "Batch Shipyard version: $shipyardversion"
echo "Blobxfer version: $blobxferversion"
echo "Singularity version: $singularityversion"
echo "User mountpoint: $USER_MOUNTPOINT"
echo "Mount path: $MOUNTS_PATH"
echo "Batch Insights: $batch_insights"
echo "Prometheus: NE=$PROM_NODE_EXPORTER_PORT,$PROM_NODE_EXPORTER_OPTIONS CA=$PROM_CADVISOR_PORT,$PROM_CADVISOR_OPTIONS"
echo "Network optimization: $networkopt"
echo "Encryption cert thumbprint: $encrypted"
echo "Install Kata Containers: $kata"
echo "Default container runtime: $default_container_runtime"
echo "Install BeeGFS BeeOND: $beeond"
echo "Storage cluster mounts (${#sc_args[@]}): ${sc_args[*]}"
echo "Custom mount: $SHIPYARD_CUSTOM_MOUNTS_FSTAB"
echo "Install LIS: $lis"
echo "GPU: $gpu"
echo "GPU ignore warnings: $SHIPYARD_GPU_IGNORE_WARNINGS"
echo "Azure Blob: $azureblob"
echo "Azure File: $azurefile"
echo "GlusterFS on compute: $gluster_on_compute"
echo "HPN-SSH: $hpnssh"
echo "Enable Azure Batch group for Docker access: $docker_group"
echo "Fallback registry: $fallback_registry"
echo "Docker image preload delay: $delay_preload"
echo "Cascade via container: $cascadecontainer"
echo "Concurrent source downloads: $concurrent_source_downloads"
echo "Block on images: $block"
echo "Singularity decryption certs: $SHIPYARD_SINGULARITY_DECRYPTION_CERTIFICATES"
echo ""

# set python env vars
export LC_ALL=en_US.UTF-8

# store node prep start
if command -v python3 > /dev/null 2>&1; then
    npstart=$(python3 -c 'import datetime;print(datetime.datetime.utcnow().timestamp())')
else
    npstart=$(python -c 'import datetime;import time;print(time.mktime(datetime.datetime.utcnow().timetuple()))')
fi

# get ephemeral device/disk
get_ephemeral_device

# check ephemeral mount
check_for_buggy_ntfs_mount

# set ephemeral device/user mountpoint
set_user_mountpoint

# save startup stderr/stdout
save_startup_to_volatile

# get vm size
get_vm_size_from_imds

# install LIS if required first (lspci won't work on certain distros without it)
install_lis

# decrypt encrypted creds
if [ -n "$encrypted" ]; then
    decrypt_encrypted_credentials
fi

# convert certs to pkcs1 for singularity encrypted container support
convert_singularity_certificates

# create shared mount points
mkdir -p "$MOUNTS_PATH"

# custom or native mode should have docker/nvidia installed
if [ $custom_image -eq 1 ] || [ $native_mode -eq 1 ]; then
    check_for_docker_host_engine
    check_docker_root_dir
    # check for nvidia card/driver/docker
    check_for_nvidia_on_custom_or_native
fi

# mount azure resources (this must be done every boot)
if [ $azurefile -eq 1 ]; then
    mount_azurefile_share
fi
if [ $azureblob -eq 1 ]; then
    mount_azureblob_container
fi

# prep task execute helper script in non-native mode
if [ "$native_mode" -eq 0 ]; then
    chmod 755 shipyard_task_runner.sh
fi

# check if we're coming up from a reboot
if [ -f "$cascadefailed" ]; then
    log ERROR "$cascadefailed file exists, assuming cascade failure during node prep"
    exit 1
elif [ -f "$nodeprepfinished" ]; then
    # start prometheus collectors
    install_and_start_node_exporter
    install_and_start_cadvisor
    # mount any storage clusters
    mount_storage_clusters
    # mount any custom mounts
    mount_custom_fstab
    # mount glusterfs on compute volumes
    if [ $gluster_on_compute -eq 1 ]; then
        if [ $custom_image -eq 1 ]; then
            mount "${MOUNTS_PATH}/gluster_on_compute/gv0"
        fi
    fi
    # non-native mode checks
    if [ "$custom_image" -eq 0 ] && [ "$native_mode" -eq 0 ]; then
        # start docker engine
        check_for_docker_host_engine
        # ensure nvidia software has been installed
        if [ -n "$gpu" ]; then
            ensure_nvidia_driver_installed
        fi
    fi
    envfile=$AZ_BATCH_NODE_STARTUP_DIR/wd/.cascade_envfile
    cat > "$envfile" << EOF
concurrent_source_downloads=$concurrent_source_downloads
$(env | grep SHIPYARD_)
$(env | grep AZ_BATCH_)
$(env | grep DOCKER_LOGIN_)
$(env | grep SINGULARITY_)
EOF
    chmod 600 "$envfile"
    log INFO "$nodeprepfinished file exists, assuming successful completion of node prep"
    exit 0
fi

# network setup
set +e
optimize_tcp_network_settings $networkopt $hpnssh
set -e

# set sudoers to not require tty
sed -i 's/^Defaults[ ]*requiretty/# Defaults requiretty/g' /etc/sudoers

# install docker host engine on non-native
if [ $custom_image -eq 0 ] && [ $native_mode -eq 0 ]; then
    install_docker_host_engine
fi

# set default pool user privileges
if [ $native_mode -eq 0 ]; then
    set_default_pool_user_privileges
fi

# login to registry servers (do not specify -e as creds have been decrypted)
./registry_login.sh

# install gpu related items
if [ -n "$gpu" ] && { [ "$DISTRIB_ID" == "ubuntu" ] || [ "$DISTRIB_ID" == "centos" ]; }; then
    if [ $custom_image -eq 0 ] && [ $native_mode -eq 0 ]; then
        install_nvidia_software
    fi
fi

# check or set up glusterfs on compute
if [ $gluster_on_compute -eq 1 ]; then
    if [ $custom_image -eq 1 ]; then
        check_for_glusterfs_on_compute
    else
        install_glusterfs_on_compute
    fi
fi

# check or install dependencies for storage cluster mount
if [ -n "${sc_args[0]}" ]; then
    if [ $custom_image -eq 1 ]; then
        check_for_storage_cluster_software
    else
        install_storage_cluster_dependencies
    fi
fi

# setup ib/rdma
process_ib_rdma

# start batch insights
install_and_start_batch_insights

# install prometheus collectors
install_and_start_node_exporter
install_and_start_cadvisor

# install dependencies if not using cascade container
if [ $native_mode -eq 0 ] || [ $delay_preload -eq 1 ]; then
    install_cascade_dependencies
fi

# retrieve required docker images
docker_pull_image "${BLOBXFER_IMAGE_PREFIX}:${blobxferversion}"
docker_pull_image "${SHIPYARD_IMAGE_PREFIX}:${shipyardversion}-cargo"

# install container runtimes
install_singularity
install_kata_containers

# install autoscratch
install_beeond

# process and mount any storage clusters
process_storage_clusters

# process and mount any custom mounts
process_custom_fstab

# store node prep end
if command -v python3 > /dev/null 2>&1; then
    npend=$(python3 -c 'import datetime;print(datetime.datetime.utcnow().timestamp())')
else
    npend=$(python -c 'import datetime;import time;print(time.mktime(datetime.datetime.utcnow().timetuple()))')
fi

# touch node prep finished file to preserve idempotency
touch "$nodeprepfinished"

cascade_docker_image="${SHIPYARD_IMAGE_PREFIX}:${shipyardversion}-cascade-docker"
cascade_singularity_image="${SHIPYARD_IMAGE_PREFIX}:${shipyardversion}-cascade-singularity"

# execute cascade
if [ $native_mode -eq 0 ] || [ $delay_preload -eq 1 ]; then
    # touch cascade failed file, this will be removed once cascade is successful
    if [ $cascadecontainer -ne 0 ]; then
        touch "$cascadefailed"
    fi
    setup_cascade
    run_cascade
    # remove cascade failed file
    rm -f "$cascadefailed"
fi

log INFO "Prep completed"
