shell_scripts/job_starter.sh
#!/bin/bash
#wrapper for cluster_runner_....sh which copies job-specific, frequently changing files (e.g. configs.py) before the actual sbatch job 
#is submitted since the job might pend in queue before execution --> hazard of job-specific files being unintentionally changed during queue wait time. 
#positonal
# -arg #1 identifies the folder name of the dataset-related code (e.g. >toy_exp< or >lidc_exp<) within the code source directory
# -arg #2 is the experiment and first part of the job name,
# optional args and flags:
# -c / --create: (flag) whether to create the exp, i.e., if this is a new start of the exp with configs etc from source dir.
# -f / --folds FOLDS: (option) fold(s) to run on (FOLDS needs to be only one int or string of multiple ints separated by space), default None (-->set to all in config)
# -m / --mode MODE: (option) string, one of "train", "train_test", "test", defaults to "train_test"
# -p / --exp_parent_dir: (option) name of parent_dir rel to dataset folder on cluster. exp_dir is exp_parent_dir/exp_name, if not given defaults to "experiments"
# -q / --queue: (option) which queue (-q parameter for bsub) to send job to. default: gputest. others: gputest-short (max 5h jobs). 
# -w / --which: (option) same as argument -m to bsub; host or host list (string separated by space) to send the job to.
# 		use nodenameXX where XX==nr of node or nodenameXX,nodenameYY,... or nodename[XX-YY]. nodename is e.g. e132-comp.
# --gmem: (option) how much gpu memory to request for job (in gigabytes), defaults to 11.9. Currently, the smaller nodes have 11.9G, the larger ones 31.7G.
# --resume: (flag) only with explicit fold argument, if set, resumes from checkpoint in exp_dir/fold_x/last_state.pth.
# --no_parallel: (flag) if set, folds won't start as parallel jobs on cluster, but run sequentially in one job.

dataset_name="${1}"
exp_name="${2}"

#arguments not passed, e.g. $7 if no seventh argument, are null.
if [ ! -z "${18}" ]; then #-z checks if is null string
 echo "Error: Received too many arguments."
 exit
fi

#make args optional: move up if some args are missing inbetween
while [ ${#} -gt 2 ]; do
  case "${3}" in
		-c|--create)
      		create_exp="c"
			shift
      		;;
		-f|--folds)
			folds="${4}"
			shift; shift
			;;
		-m|--mode)
			mode="${4}"
			shift; shift
			;;
		-p|--exp_parent_dir)
			exp_parent_dir="${4}"
			shift; shift			
			;;
		-q|--queue)
			queue="${4}"
			shift; shift			
			;;
		-w|--which)
			which="${4}"
			shift; shift			
			;;
		-R|--resource)
			resource="${4}"
			shift; shift			
			;;
		--gmem)
			gmem="${4}"
			shift; shift
			;;
		--resume)
			resume=true
			shift
			;;
		--no_parallel)
			no_parallel=true
			shift
			;;
    *)
			echo "Invalid argument/option passed: ${3}"
			exit 1
			;;
  esac
done

# default values
if [ -z ${exp_parent_dir} ]; then 
	exp_parent_dir="experiments"
fi

if [ -z ${mode} ]; then 
	mode="train_test"
fi

if [ -z ${queue} ]; then 
	queue="gputest"
fi


if [ -z ${gmem} ]; then 
	gmem="11"
fi


root_dir=/home/ramien #assumes /home/ramien exists
prep_node=ramien@e132-comp07 #node used for prep tasks like create_exp
#medicaldetectiontoolkit
source_dir=${root_dir}/regrcnn

dataset_abs_path=${source_dir}/datasets/${dataset_name} #set as second argument passed to this script
exp_parent_dir=/datasets/datasets_ramien/${dataset_name}/${exp_parent_dir}
# /dataset is not mounted on log-in/job submission nodes (would maybe make sense, I feel), only on queue gputest's nodes e132-compXX.
exp_dir=${exp_parent_dir}/${exp_name}

#activate virtualenv that has all the packages:
source_dl="module load python/3.7.0; module load gcc/7.2.0; source ${root_dir}/.virtualenvs/regrcnn/bin/activate;"

# TODO as long as no fix available: this script needs to be started directly from the prep node. :/ would be nice if (most importantly
# 'module ...') would also work over ssh, but somehow some commands are not availabe over the ssh-induced shell (even when using it as interactive).
eval ${source_dl}

# directly from prep node:
create_cmd="python ${source_dir}/exec.py --server_env --mode create_exp --exp_dir ${exp_dir} --dataset_name ${dataset_abs_path};"


#if create_exp, check if would overwrite existing exp_dir
if [ ! -z ${create_exp} ] && [ ${create_exp} = "c" ]; then #-n doesnt work as replacement for !-z
	if [ -d ${exp_dir} ]; then
		echo "Please confirm to overwrite exp ${exp_name} settings, (Y/n): "; read confirmation
		if ([ "${confirmation}" = "y" ] || [ "${confirmation}" = "yes" ] || [ "${confirmation}" = "Y" ] || [ -z "${confirmation}" ]); then
				echo "Overwriting ${exp_name}"
		else
				echo "Exiting due to overwrite denial. Adjust options."
				exit
		fi
	fi
	echo "Creating ${exp_name}"
	eval ${create_cmd}
else
	if [ ! -d ${exp_dir} ]; then
		echo "Experiment directory ${exp_dir} does not exist."
		echo "Run create_exp? (Y/n): "; read confirmation
			if ([ "${confirmation}" = "y" ] || [ "${confirmation}" = "yes" ] || [ "${confirmation}" = "Y" ] || [ -z "${confirmation}" ]); then
				echo "Creating ${exp_name}"
				eval ${create_cmd}
			fi
	fi
fi

#if not create_exp, check if would overwrite existing folds (possibly valuable trained params!)
if [ -z ${create_exp} ] && ([ ${mode} = "train" ] || [ ${mode} = "train_test" ]) && [ -z "${resume}" ]; then
	for f in ${folds}; do #if folds is null this check won't apply and folds will be quietly overwritten.
		if [ -d ${exp_dir}/fold_${f} ]; then #-d checks if is dir
			echo "please confirm to overwrite fold_${f}, (Y/n):"; read confirmation
			if ([ "${confirmation}" = "y" ] || [ "${confirmation}" = "yes" ] || [ "${confirmation}" = "Y" ] || [ -z "${confirmation}" ]); then
				echo "Overwriting "${exp_name}/fold_${f}
			else
				echo "Exiting due to overwrite denial. Adjust options."
				exit
			fi
		fi
	done
fi



bsub_opts="bsub -N -q ${queue} -gpu num=1:j_exclusive=yes:mode=exclusive_process:gmem=${gmem}G"
if [ ! -z "$resource" ]; then
	bsub_opts=$bsub_opts $resource
fi
if [ ! -z ${which} ]; then
	bsub_opts="${bsub_opts} -m ${which}"
fi

#----- parallel/separate fold jobs (each fold in a single job) -----------
if [ ! -z "${folds}" ] && [ -z ${no_parallel} ]; then #WHY do i need to convert to string again?
	for f in ${folds}; do
		out_file=${exp_dir}/logs/fold_${f}_lsf_output.out
		bsub_opts="$bsub_opts -J '${dataset_name} ${exp_name}  fold ${f} ${mode}' -oo '${out_file}'"
		eval "${bsub_opts} sh cluster_runner_meddec.sh ${source_dir} ${exp_dir} ${dataset_abs_path} ${mode} ${f} ${resume}"
	done

#----- consecutive folds job (all folds in one single job) -----------
else 
	if [ ! -z ${resume} ]; then
		echo "You need to explicitly specify folds if you would like to resume from a checkpoint. Exiting."
		exit
	fi
	out_file=${exp_dir}/lsf_output.out
	bsub_opts="$bsub_opts -J '${dataset_name} ${exp_name}  folds ${folds} ${mode}' -oo '${out_file}'"
	eval "${bsub_opts} sh cluster_runner_meddec.sh ${source_dir} ${exp_dir} ${dataset_abs_path} ${mode} ${folds} ${resume}"
	echo "Started in no parallel, folds:" ${folds}
fi



