#!/bin/bash
# Derived from https://github.com/ray-project/ray/blob/master/doc/source/cluster/doc_code/slurm-basic.sh.
# Env vars required by this script (should be set by SLURM):
# SLURM_JOB_NODELIST: list of nodes in the job.
# SLURM_CPUS_PER_TASK: number of CPUs per node.
# SLURM_GPUS_ON_NODE: number of GPUs per node.

set -e

nodes=$(scontrol show hostnames "$SLURM_JOB_NODELIST")
nodes_array=($nodes)
head_node=${nodes_array[0]}
head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address)

# if we detect a space character in the head node IP, we'll
# convert it to an ipv4 address. This step is optional.
if [[ "$head_node_ip" == *" "* ]]; then
IFS=' ' read -ra ADDR <<<"$head_node_ip"
if [[ ${#ADDR[0]} -gt 16 ]]; then
head_node_ip=${ADDR[1]}
else
head_node_ip=${ADDR[0]}
fi
echo "IPV6 address detected. We split the IPV4 address as $head_node_ip"
fi

port=6379
ip_head=$head_node_ip:$port
export ip_head

echo "Starting HEAD at $head_node"
srun --nodes=1 --ntasks=1 -w "$head_node" \
    ray start --head --node-ip-address="$head_node_ip" --port=$port \
    --num-cpus "$SLURM_CPUS_PER_TASK" --num-gpus "$SLURM_GPUS_ON_NODE" \
    --block &

# Ray claims this is optional for Ray >= 1.0.
# However, for single-node jobs without this, they seem to try
# to connect to the cluster before it's up and fail.
sleep 10

# number of nodes other than the head node
worker_num=$((SLURM_JOB_NUM_NODES - 1))

for ((i = 1; i <= worker_num; i++)); do
    node_i=${nodes_array[$i]}
    echo "Starting WORKER $i at $node_i"
    srun --nodes=1 --ntasks=1 -w "$node_i" \
        ray start --address "$ip_head" \
        --num-cpus "$SLURM_CPUS_PER_TASK" --num-gpus "$SLURM_GPUS_ON_NODE" \
        --block &
    sleep 5
done

# Print cluster status to confirm setup.
echo "Ray cluster status:"
ray status

# The Ray dashboard is run on the head node. To view it on our local machine, we need to
# set up the following SSH tunnel and keep it alive:
# localhost:8265 -> Slurm gateway -> Slurm head node -> localhost:8265 (on head node)
echo "-----------------------------------------------------------------------------------------"
echo "To view the Ray dashboard, run the following command (keep the connection alive)"
echo "and open http://localhost:8265 on your local machine:"
echo "ssh -L 8265:127.0.0.1:8265 -J \$OUMI_SLURM_CONNECTIONS $USER@$head_node_ip"
echo "-----------------------------------------------------------------------------------------"
