#!/bin/sh -x
# $Id: run_g09_chk_recovery.sh,v 1.0 2017/04/22 14:15:00 Sudhakar Exp $

if [ $# -lt 1 -o $# -gt 3 ]; then
    echo 1>&2 "Usage: $0 gaussian_input_gjf [SEAGrid_UserName] [GPU] "
    #echo 1>&2 "Usage: $0 subdir gaussian_input_gjf clobber [gcvars]"
    exit 127
fi

# subdir depends on whether we're doing freq, water or PES. For freq and water,
# it should be hardcoded in the Xbaya workflow. For PES, it should be an
# additional array generated by the frontend. The contents of this array are
# trivial, but creating an extra Xbaya service to generate it would add
# unnecessary extra complexity. Besides, the frontend cannot avoid having to
# pass at least one array: the array with gjf files.
#subdir=$1
subdir="$PWD"
#export GAUSS_SCRDIR=/oasis/scratch/comet/$USER/temp_project/$SLURM_JOBID
#export GAUSS_SCRDIR=/expanse/lustre/scratch/$USER/temp_project/$SLURM_JOBID
scratch_subid=$(id -u $user | tail -c2)
scrdir="/storage/scratch1/$scratch_subid/$USER"
export GAUSS_SCRDIR=$scrdir/$SLURM_JOBID
mkdir -p $GAUSS_SCRDIR
gaussian_input_full=$1
if [ $AIRAVATA_USERNAME ]; then
  echo " The Airavata Gateway User is $AIRAVATA_USERNAME"
  SG_UserName="$AIRAVATA_USERNAME"
#elif [ $2 ]; then
#  SG_UserName=$2
else
  echo " The Airavata Gateway User is missing "
exit
fi

        #export PATH="/storage/home/hcoda1/7/spamidig6/ideas_storage/apps/g16:$PATH"
        #export GAUSS_EXEDIR="/storage/home/hcoda1/7/spamidig6/ideas_storage/apps/g16"
        #. ~/.bash_profile
#if [ "$2" = "GPU" ]; then
	echo "Using GPU version of Gaussian 16"
	#module reset; module load  gpu/0.15.4  gaussian/16.C.01-cuda
	#module load gaussian/16.C.02
        #export PATH="/storage/home/hcoda1/7/spamidig6/ideas_storage/apps/g16:$PATH"
        #export GAUSS_EXEDIR="/storage/home/hcoda1/7/spamidig6/ideas_storage/apps/g16"
        #. ~/.bash_profile
#fi

if [ $AIRAVATA_ExptDataDir ]; then
    echo "The Airavata Storage Directory for this job is $AIRAVATA_ExptDataDir"
    echo "Preparing Cron to push log data to storage periodically"
    # Get Slurm total time
#        flds=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' | awk -F: '{print NF}'`
#        flds=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' | awk -F- '{print NF}'` + $flds
#        # if flds 3 $1 is hrs $2 is min and $3 is sec
#        if [ $flds = 4 ]; then
#          jdys=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' |awk -F- '{print $1}'`
#          hrmnse=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' |awk -F- '{print $2}'`
#          jhrs=`echo $hrmnse | awk -F: '{print $1}'`+24*$jdys
#	  jmin=`echo $hrmnse | awk -F: '{print $2}'`+60*$jhrs
#        elif [ $flds = 3 ]; then
#          jhrs=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' |awk -F: '{print $1}'`
#	  jmin=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' |awk -F: '{print $2}'`+60*$jhrs
#        elif [ $flds = 2 ]; then
#          jmin=`squeue -j $SLURM_JOBID -l | awk 'END { print $7}' |awk -F: '{print $1}'`
#        fi
#        if [ $jhrs .gt. 5 ]; then
#          upd=30
#        else
#          upd=$jmin/10
#        fi
    # For 5hrs and above uperiod is 30 min and for less than 5 hrs it is 10% of total time
    # compute uperiod
    # preapre or use prepared updateatorage script
    # i(crontab -l 2>/dev/null; echo "*/$uperiod * * * * /path/to/job -with args") | crontab - */30 * * * * /path/to/command
    # Use crontab -r to remove it after the job is finished or ended ( under trap)
#      mycron=jcron$$
#      #write out current crontab
#      crontab -l > $mycron
#      #echo new cron into cron file
#      #echo "*/$upd * * * * /home/gridchem/bin/joblogscp.sh >/dev/null 2>&1" >> $mycron
#      echo "*/$upd * * * *  scp *.log  pga@gf4.ucs.indiana.edu:$AIRAVATA_ExptDataDir >/dev/null 2>&1" >> $mycron
#      #echo "* * * * * sleep 20; /Users/spamidig1/bin/testscp.sh >/dev/null 2>&1" >> $mycron
##install new cron file
#      crontab $mycron
#      crontab -l
#      # delete this entry at the end of the job or trap...
fi
inputs=`echo $AIRAVATA_INPUTS | sed 's/[//'`
inputs=`echo $inputs | sed 's/]//'`
echo "Airavata Inputs: $inputs"
cd $subdir
dos2unix -k $gaussian_input_full
gaussian_input=${gaussian_input_full##*/}
gaussian_output=${gaussian_input%.*}.log
clobber="$3" # set to "" for rerun or debug; otherwise set to 1
gcvars=$4
null=""

# Next line will ususally return "cp: `GCVARS' and `GCVARS' are the same file"
#if [ "$gcvars" ] ; then cp -p $gcvars GCVARS 2>/dev/null ; fi

#if [ ! "$LOCAL_LOCATION" ] ; then
#  if [ -s ~/.paramchemlocation ] ; then
#    read LOCAL_LOCATION < ~/.paramchemlocation
#    . $LOCAL_LOCATION/environment
#   fi
# fi
#if [ ! -d "$LOCAL_LOCATION" ] ; then
#  echo "Warning: no valid LOCAL_LOCATION found" >&2
#gauss_mem=56GB
#gauss_nproc=24
## Escaping spaces rather than quoting because quoting prevents tilde expansion
##charmm_location=~gridchem/workflow_script/sys_exec/local/c36a6-132-serial\ -chsize\ 25140
#SCRIPT_LOCATION=~gridchem/workflow_script/sys_exec/tools
#LOCAL_LOCATION=~gridchem/workflow_script/sys_exec/local-comet
#export CONPATH=$SCRIPT_LOCATION
localarc="$HOME/scratch"
##globalarc="ccguser@gridchem.uits.iu.edu:/home/ccguser/mss/internal/$SGUserName"
##  exit 1
# fi
#. $LOCAL_LOCATION/environment

#read GC_UserName GC_ProjectName GC_WorkflowName TIMESTAMP < GCVARS
usrchkdir=$localarc/${SG_UserName}/
echo " The Airavata Gateway User Directory is $usrchkdir"
copysubdir="./"
mkdir -p $usrchkdir
#copysubdir=$localarc/${GC_UserName}/${GC_ProjectName}/${GC_WorkflowName}/$subdir
# The way a "false" boolean variable is passed seems to be unstable; it's
# been "", "0" and "false", so we try to cover all reasonable possibilities.
if [ ! "$clobber" ] ; then
  clobber=0
 fi
if [ "$clobber" = "0" -o "$clobber" = "false" -o "$clobber" = "no" ] ; then
  if [ -s $copysubdir/$gaussian_output ] ; then
    echo gaussian_output_log=$copysubdir/$gaussian_output
    exit 0
   fi
  gaussian_output_full=${gaussian_input_full%.*}.log
  if [ -s $gaussian_output_full ] ; then
    #mkdir -p $copysubdir
    rm -f $copysubdir/$gaussian_output # clean up symlink if something went wrong earlier
    if [ $gaussian_input_full -nt $copysubdir/$gaussian_input ] ; then sed 's/\r$//' $gaussian_input_full > $copysubdir/$gaussian_input ; fi
    cp -up $gaussian_output_full $copysubdir
    echo gaussian_output_log=$gaussian_output_full
    exit 0
   fi
 fi

#Process inputfile for run files and other job requirements
  # PROCESS CHECKPOINT FILE
  # Check to see if the checkpoint file is given a name in input deck
  # Input file to look into
  dummy="$gaussian_input_full"
  #dummy="$Diskinputdir/$Diskinputfile"
  checkpt="no"
  junk=`/usr/bin/head -5 $dummy | /bin/grep -i "%chk"`
  if [ "$junk" !=  "" ]; then
    junk=`echo $junk | /bin/sed 's/=/@/'`
    junk=`echo $junk | /bin/sed 's/ //'`
    #
    # Be careful: Don't lose the subdirectory information for CHKPT file
    # Also, add .chk if there is no extension to the Checkpoint file
    #
    Chkfile=`expr $junk : '.*@\(.*\)'`
    Chkdir="$Diskinputdir"
    Chkfile=`/bin/basename $Chkfile`
    dummy=`expr $Chkfile : '.*\(\..*\)'`
    Chkfile=`/bin/basename $Chkfile $dummy`
    ChkfileWNE="$Chkfile"
    Chkfile="$Chkfile${dummy:-.chk}"
//"`Chkfile=`echo $Chkfile | sed "s/
    # 2005/12/08 create name for $formated_chkfile
    formated_chkfile="$ChkfileWNE.fchk"
    Chkfile_with_arch="${Chkfile}_$architecture"
    echo "DEBUG: checkfile = $Chkfile and formated_chkfile = $formated_chkfile ";
    checkpt="yes"
#Retrieve the checkpoint file from the user archive directory
     if [ -f "$usrchkdir/$Chkfile" ]; then
        cp $usrchkdir/$Chkfile .
     fi
//"`     export PJobID=`grep -i localjobid $gaussian_input_full | awk -F= '{print $2}' | sed "s/
#     /bin/cat >> $qsubin << HERE2
    #export PJobID=`grep -i localjobid $Diskinputfile | awk -F= '{print $2}' | sed "s/^M//"`
    #cd /oasis/scratch/comet/gridchem/temp_project/$PJobID
    #cd \$SCRATCH_BATCH
    ##if [ ${PJobID:-null} != "$null" ]; then
    if [ "${PJobID}" != "" ]; then
      #cp -r /work/ccguser/batch_scratch/$PJobID*/* .
      #cp -r /oasis/scratch/comet/gridchem/temp_project/$PJobID*/* .
      cp -r $HOME/scratch/$PJobID*/* .
      ls -l
    fi
   else
    echo "******** NO CHECKPOINT FILE IDENTIFIED ******** "
  fi

mkdir -p $copysubdir
mkdir -p $subdir
cd $subdir
cwd=`pwd`
if [ $gaussian_input_full -nt $copysubdir/$gaussian_input ] ; then sed 's/\r$//' $gaussian_input_full > $copysubdir/$gaussian_input ; fi
cd $copysubdir
rm -f $gaussian_output
if [ "$cwd" != "$subdir" ]; then
   ln -s $cwd/$gaussian_output $subdir/$gaussian_output
fi
cd $cwd
if [ $gaussian_input_full -nt $gaussian_input ] ; then sed 's/\r$//' $gaussian_input_full > $gaussian_input ; fi
signals_to_trap="XCPU INT TERM CHLD"
#trap "| grep -v $AIRAVATA_ExptDataDir | crontab -; rm -rf $mycron; cp -p $gaussian_output $copysubdir; cp -p $Chkfile $copysubdir; exit 99" $signals_to_trap
#trap "crontab -l | grep -v $AIRAVATA_ExptDataDir | crontab -; rm -rf $mycron; cp -p $gaussian_output $copysubdir; cp -p $Chkfile $copysubdir; exit 99" $signals_to_trap
cd $HOME/scratch
ln -s $subdir $SLURM_JOBID
ls -l $SLURM_JOBID/
cd $cwd
#$LOCAL_LOCATION/run_gaussian_local.sh $gaussian_input $gaussian_output
which g16
g16 $gaussian_input $gaussian_output
#BEGIN{while(getline < infile) if ($0 ~ "^ *--[lL][iI][nN][kK]1--") nlink++}

#if awk -v infile=$gaussian_input '
#    BEGIN{while(getline < infile) if ($0 ~ "^ *[lL][iI][nN][kK]1") nlink++}
#    /^ *Normal termination of Gaussian/{nnormal++}
#    END{if (nnormal == nlink+1) exit 1}' $gaussian_output ; then
#  echo "Gaussian terminated abnormally." >&2
#  exit 1
#fi
# Remove the cron entry to periodically stage the data to storage
#crontab -l | grep -v "$AIRAVATA_ExptDataDir"" | crontab -
#crontab -l
#rm $mycron

#rm $copysubdir/$gaussian_output
cp -p $gaussian_output $copysubdir
 if [ -f "$Chkfile" ]; then
    cp -p $Chkfile $copysubdir
 fi
 if [ -f "$GAUSS_SCRDIR/$Chkfile" ]; then
    cp -p $GAUSS_SCRDIR/$Chkfile .
 fi
  # Save  checkpoint file to usrchkdir
  #mkdir -p $usrchkdir
  if [ -f "$Chkfile" ]; then
        formchk $Chkfile
  	cp -f $Chkfile $usrchkdir
        cp -f *.fchk $usrchkdir
        cp -f *.fchk $copysubdir
  fi
#remove rwf files
rm *.rwf*
cd $HOME/scratch
#ln -s $subdir $PBS_JOBID
ls -l $SLURM_JOBID
rm $SLURM_JOBID/*.rwf*
echo gaussian_output_log=$cwd/$gaussian_output
cat: S: No such file or directory
