#!/bin/bash

ciao_bin="$HOME/local"
ciao_gobin="$GOPATH"/bin
event_counter=0

#Utility functions
function exitOnError {
	local exit_code="$1"
	local error_string="$2"

	if [ $1 -ne 0 ]
	then
		echo "FATAL ERROR exiting: " "$error_string" "$exit_code"
		exit 1
	fi
}

#Checks that no network artifacts are left behind
function checkForNetworkArtifacts() {
	local retry=0

	#Verify that there are no ciao related artifacts left behind
	until [ $retry -ge 6 ]
	do
		ciao_networks=`sudo docker network ls --filter driver=ciao -q | wc -l`

		if [ $ciao_networks -eq 0 ]
		then
			break
		fi

		let retry=retry+1
		sleep 5
	done

	if [ $ciao_networks -ne 0 ]
	then
		echo "FATAL ERROR: ciao docker networks not cleaned up"
		sudo docker network ls --filter driver=ciao
		exit 1
	fi

	let retry=0

	#The only ciao interfaces left behind should be CNCI VNICs
	#Once we can delete tenants we should not even have them around
	until [ $retry -ge 6 ]
	do
		cnci_vnics=`ip -d link | grep alias | grep cnci | wc -l`
		ciao_vnics=`ip -d link | grep alias | wc -l`

		if [ $cnci_vnics -eq $ciao_vnics ]
		then
			break
		fi

		let retry=retry+1
		sleep 5
	done

	if [ $cnci_vnics -ne $ciao_vnics ]
	then
		echo "FATAL ERROR: ciao network interfaces not cleaned up"
		ip -d link | grep alias
		exit 1
	fi
}

function rebootCNCI {
	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" cloud-admin@"$ssh_ip" <<-EOF
	sudo reboot now
	EOF

	#Now wait for it to come back up
	ping -w 120 -c 3 $ssh_ip
	exitOnError $?  "Unable to ping CNCI after restart"

	#Dump the tables for visual verification
	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" cloud-admin@"$ssh_ip" <<-EOF
	sudo iptables-save
	sudo ip l
	sudo ip a
	EOF

	echo "Rebooted the CNCI"
}

function checkExtIPConnectivity {
    #We checked the event before calling this, so the mapping should exist
    testip=`"$ciao_gobin"/ciao list external-ips -f '{{with index . 0}}{{.ExternalIP}}{{end}}'`
    test_instance=$1

    sudo ip route add 203.0.113.0/24 dev ciaovlan
    ping -w 90 -c 3 $testip
    ping_result=$?
    #Make sure we are able to reach the VM
    test_hostname=`ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" demouser@"$testip" hostname -s`
    sudo ip route del 203.0.113.0/24 dev ciaovlan

    exitOnError $ping_result "Unable to ping external IP"

    if [ "$test_hostname" == "$test_instance" ]
    then
	    echo "SSH connectivity using external IP verified"
    else
	    echo "FATAL ERROR: Unable to ssh via external IP"
	    exit 1
    fi
}

#There are too many failsafes in the CNCI. Hence just disable iptables utility to trigger failure
#This also ensures that the CNCI is always left in a consistent state (sans the permission)
# this function to be run on the CNCI
function triggerIPTablesFailure {
	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" cloud-admin@"$ssh_ip" <<-EOF
	sudo chmod -x /usr/bin/iptables
	EOF
}

#Restore the iptables so that the cluster is usable
# this function to be run on the CNCI
function restoreIPTables {
	ssh -T -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "$CIAO_SSH_KEY" cloud-admin@"$ssh_ip" <<-EOF
	sudo chmod +x /usr/bin/iptables
	EOF
}

function clearAllEvents() {
	#Clear out all prior events. Currently this is an admin only operation.

	ciao_client_cert_file=$CIAO_CLIENT_CERT_FILE
	export CIAO_CLIENT_CERT_FILE=$CIAO_ADMIN_CLIENT_CERT_FILE

	"$ciao_gobin"/ciao delete events

	#Wait for the event count to drop to 0
	retry=0
	ciao_events=0

	until [ $retry -ge 6 ]
	do
		ciao_events=`"$ciao_gobin"/ciao list events -f '{{len .}}'`

		if [ $ciao_events -eq 0 ]
		then
			break
		fi
		let retry=retry+1
		sleep 1
	done

	export CIAO_CLIENT_CERT_FILE=$ciao_client_cert_file

	exitOnError $ciao_events "ciao events not deleted properly"
}

function checkEventStatus {
	local event_index="$1"
	local event_code="$2"
	local retry=0
	local ciao_events=0
	local total_events=0
	local code=""

	#We only need to wait for as many events as the index
	total_events=$((event_index + 1))

	until [ $retry -ge 10 ]
	do
		ciao_events=`"$ciao_gobin"/ciao list events -f '{{len .}}'`

		if [ $ciao_events -eq $total_events ]
		then
			break
		fi

		let retry=retry+1
		sleep 2
	done

	if [ $ciao_events -ne $total_events ]
	then
		echo "FATAL ERROR: ciao event not reported. Events seen =" $ciao_events
		"$ciao_gobin"/ciao list events
		exit 1
	fi

	code=$("$ciao_gobin"/ciao list events -f "{{(index . ${event_index}).Message}}" | cut -d ' ' -f 1)

	if [ "$event_code" != "$code" ]
	then
		echo "FATAL ERROR: Unknown event $code. Looking for $event_code"
		"$ciao_gobin"/ciao list events
		exit 1
	fi

	"$ciao_gobin"/ciao list events
}

function createExternalIPPool() {
	# first create a new external IP pool and add a subnet to it.
	# this is an admin only operation, so make sure our env variables
	# are set accordingly. Since user admin might belong to more than one
	# tenant, make sure to specify that we are logging in as part of the
	# "admin" tenant/project.
	ciao_client_cert_file=$CIAO_CLIENT_CERT_FILE
	export CIAO_CLIENT_CERT_FILE=$CIAO_ADMIN_CLIENT_CERT_FILE
	
	"$ciao_gobin"/ciao create pool test
	"$ciao_gobin"/ciao add external-ip test 203.0.113.0/24
	
	export CIAO_CLIENT_CERT_FILE=$ciao_client_cert_file
}

function deleteExternalIPPool() {
	#Cleanup the pool
	ciao_client_cert_file=$CIAO_CLIENT_CERT_FILE
	export CIAO_CLIENT_CERT_FILE=$CIAO_ADMIN_CLIENT_CERT_FILE

	"$ciao_gobin"/ciao delete pool test
	exitOnError $?  "Unable to delete pool"

	export CIAO_CLIENT_CERT_FILE=$ciao_client_cert_file
}

# Read cluster env variables
. $ciao_bin/demo.sh

for item in `ciao list workloads -f '{{select . "ID"}} '`
do
    type=`ciao show workload $item -f '{{.VMType}}'`;
    if [ $type == "qemu" ]
    then
	vm_wlid=$item
	break
    fi
done

if [ -z $vm_wlid ]
then
    echo "FATAL ERROR exiting: No VM workloads defined"
    exit 1
fi

"$ciao_gobin"/ciao create instance $vm_wlid --instances=2
exitOnError $?  "Unable to launch VMs"

"$ciao_gobin"/ciao list instances
exitOnError $? "Unable to list instances"

#Launch containers
#Pre-cache the image to reduce the start latency
sudo docker pull debian
debian_wlid=$("$ciao_gobin"/ciao list workloads  -f='{{$x := filter . "Name" "Debian latest test container"}}{{if gt (len $x) 0}}{{(index $x 0).ID}}{{end}}')
echo "Starting workload $debian_wlid"
"$ciao_gobin"/ciao create instance $debian_wlid --instances=1
exitOnError $? "Unable to launch containers"

sleep 5

"$ciao_gobin"/ciao list instances
exitOnError $? "Unable to list instances"

container_1=`sudo docker ps -q -l`
container_1_ip=`sudo docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_1`

"$ciao_gobin"/ciao create instance $debian_wlid --instances=1
exitOnError $?  "Unable to launch containers"
sleep 5

"$ciao_gobin"/ciao list instances
exitOnError $?  "Unable to list instances"

container_2=`sudo docker ps -q -l`

#Check SSH connectivity
"$ciao_gobin"/ciao list instances

#The VM takes time to boot as you are running on two
#layers of virtualization. Hence wait a bit
retry=0
until [ $retry -ge 6 ]
do
	ssh_ip=""
	read -r ssh_ip ssh_port <<<$("$ciao_gobin"/ciao list instances $vm_wlid -f='{{if gt (len .) 0}}{{with (index . 0)}}{{println .SSHIP}}{{println .SSHPort}}{{end}}{{end}}')

	if [ "$ssh_ip" == "" ] 
	then
		echo "Waiting for instance to boot"
		let retry=retry+1
		sleep 30
		continue
	fi

	ssh_check=$(head -1 < /dev/tcp/"$ssh_ip"/"$ssh_port")

	echo "Attempting to ssh to: $ssh_ip:$ssh_port"

	if [[ "$ssh_check" == *SSH-2.0-OpenSSH* ]]
	then
		echo "SSH connectivity verified"
		break
	else
		let retry=retry+1
		echo "Retrying ssh connection $retry"
	fi
	sleep 30
done

if [ $retry -ge 6 ]
then
	echo "Unable check ssh connectivity into VM"
	exit 1
fi

#Check docker networking
echo "Checking Docker Networking"
sudo docker exec $container_2 /bin/ping -w 90 -c 3 $container_1_ip

exitOnError $?  "Unable to ping across containers"
echo "Container connectivity verified"

#Clear out all prior events
clearAllEvents

#Test External IP Assignment support
#Pick the first instance which is a VM, as we can even SSH into it
#We have already checked that the VM is up.
createExternalIPPool

testinstance=`"$ciao_gobin"/ciao list instances $vm_wlid -f '{{with index . 0}}{{.ID}}{{end}}'`
"$ciao_gobin"/ciao attach external-ip test $testinstance 

#Wait for the CNCI to report successful map
checkEventStatus $event_counter "Mapped"

"$ciao_gobin"/ciao list events
"$ciao_gobin"/ciao list external-ips

checkExtIPConnectivity $testinstance

#Check that the CNCI retains state after reboot
#If state has been restored, the Ext IP should be reachable
rebootCNCI
checkExtIPConnectivity $testinstance

"$ciao_gobin"/ciao detach external-ip $testip

#Wait for the CNCI to report successful unmap
event_counter=$((event_counter+1))
checkEventStatus $event_counter "Unmapped"

"$ciao_gobin"/ciao list external-ips

#Test for External IP Failures

#Map failure
triggerIPTablesFailure
"$ciao_gobin"/ciao attach external-ip test $testinstance 
#Wait for the CNCI to report unsuccessful map
event_counter=$((event_counter+1))
checkEventStatus $event_counter "Failed"
restoreIPTables

#Unmap failure
"$ciao_gobin"/ciao attach external-ip test $testinstance
event_counter=$((event_counter+1))
checkEventStatus $event_counter "Mapped"

triggerIPTablesFailure 
"$ciao_gobin"/ciao detach external-ip $testip
event_counter=$((event_counter+1))
checkEventStatus $event_counter "Failed"
restoreIPTables

#Cleanup
"$ciao_gobin"/ciao detach external-ip $testip
event_counter=$((event_counter+1))
checkEventStatus $event_counter "Unmapped"

#Cleanup pools
deleteExternalIPPool

to_delete=`"$ciao_gobin"/ciao list instances -f '{{len .}}'`
#Now delete all instances
"$ciao_gobin"/ciao delete instance --all
exitOnError $?  "Unable to delete instances"

"$ciao_gobin"/ciao list instances

#Wait for all the instance deletions to be reported back
event_counter=$((event_counter+$to_delete))
checkEventStatus $event_counter "Deleted"

#Verify that there are no ciao related artifacts left behind
checkForNetworkArtifacts

echo "###########################################"
echo "-----------All checks passed!--------------"
echo "###########################################"
