#!/bin/csh -f
#PBS -N NNA.4km.sample
#PBS -q main
#PBS -l job_priority=regular
#PBS -A UCUB0117
#PBS -l select=32:ncpus=128:mpiprocs=128:ompthreads=1
#PBS -l walltime=02:00:00
#PBS -j oe
#PBS -W umask=022
#PBS -o /glade/work/tcraig/NNA/rasm_nna.derecho_sample/scripts/NNA.4km.sample/
###PBS -M username@domain.com
###PBS -m be

# ---------------------------------------- 
# PE LAYOUT: 
#   total number of tasks  = 4087 
#   maximum threads per task = 1 
#   cpl ntasks=4087  nthreads=1 rootpe=0 ninst=1 
#   wrf ntasks=4087  nthreads=1 rootpe=0 ninst=1 
#   ctsm ntasks=3636  nthreads=1 rootpe=0 ninst=1 
#   cice ntasks= 288  nthreads=1 rootpe=3636 ninst=1 
#   docn ntasks= 144  nthreads=1 rootpe=3924 ninst=1 
#   sglc ntasks=   1  nthreads=1 rootpe=0 ninst=1 
#   srof ntasks=   1  nthreads=1 rootpe=0 ninst=1 
#   
#   total number of hw pes = 4087 
#     cpl hw pe range ~ from 0 to 4086 
#     wrf hw pe range ~ from 0 to 4086 
#     ctsm hw pe range ~ from 0 to 3635 
#     cice hw pe range ~ from 3636 to 3923 
#     docn hw pe range ~ from 3924 to 4067 
#     sglc hw pe range ~ from 0 to 0 
#     srof hw pe range ~ from 0 to 0 
# ---------------------------------------- 
#-----------------------------------------------------------------------
#
# rerun loop top
# set runmax to number of times around the loop
#
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
set runmax = 1

# wall minutes in queue, set queuehours
set queuehours = 240
@ timemax = ${queuehours} * 60
set date0 = `date "+%s"`

set runcnt = 0
while ($runcnt < $runmax)
@ runcnt = $runcnt + 1
echo "`date` -- CSM RUNCNT LOOP HAS STARTED" 
echo "running runcnt $runcnt"
set date1 = `date "+%s"`
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

cd /glade/work/tcraig/NNA/rasm_nna.derecho_sample/scripts/NNA.4km.sample

./Tools/ccsm_check_lockedfiles || exit -1
source ./Tools/ccsm_getenv     || exit -2

if ($BUILD_COMPLETE != "TRUE") then
  echo "BUILD_COMPLETE is not TRUE"
  echo "Please rebuild the model interactively"
  exit -2
endif

# BATCHQUERY is in env_run.xml
setenv LBQUERY "TRUE"
if !($?BATCHQUERY) then
  setenv LBQUERY "FALSE"
  setenv BATCHQUERY "undefined"
else if ( "$BATCHQUERY" == 'UNSET' ) then
  setenv LBQUERY "FALSE"
  setenv BATCHQUERY "undefined"
endif

# BATCHSUBMIT is in env_run.xml
setenv LBSUBMIT "TRUE"
if !($?BATCHSUBMIT) then
  setenv LBSUBMIT "FALSE"
  setenv BATCHSUBMIT "undefined"
else if ( "$BATCHSUBMIT" == 'UNSET' ) then
  setenv LBSUBMIT "FALSE"
  setenv BATCHSUBMIT "undefined"
endif

# --- Create and cleanup the timing directories---

if !(-d $RUNDIR) mkdir -p $RUNDIR || "cannot make $RUNDIR" && exit -1
if (-d $RUNDIR/timing) rm -r -f $RUNDIR/timing
mkdir $RUNDIR/timing
mkdir $RUNDIR/timing/checkpoints

# --- Determine time-stamp/file-ID string ---
setenv LID "`date +%y%m%d-%H%M%S`"

set sdate = `date +"%Y-%m-%d %H:%M:%S"`
echo "run started $sdate" >>& $CASEROOT/CaseStatus

echo "-------------------------------------------------------------------------"
echo " CESM BUILDNML SCRIPT STARTING"
echo " - To prestage restarts, untar a restart.tar file into $RUNDIR"

./preview_namelists

echo " CESM BUILDNML SCRIPT HAS FINISHED SUCCESSFULLY"
echo "-------------------------------------------------------------------------"

echo "-------------------------------------------------------------------------"
echo " CESM PRESTAGE SCRIPT STARTING"
echo " - Case input data directory, DIN_LOC_ROOT, is $DIN_LOC_ROOT"
echo " - Checking the existence of input datasets in DIN_LOC_ROOT"

# This script prestages as follows
# - DIN_LOC_ROOT is the local inputdata area, check it exists
# - check whether all the data is in DIN_LOC_ROOT
# - prestage the REFCASE data if needed

cd $CASEROOT

if !(-d $DIN_LOC_ROOT) then
  echo " "
  echo "  ERROR DIN_LOC_ROOT $DIN_LOC_ROOT does not exist"
  echo " "
  exit -20
endif

if (`./check_input_data -inputdata $DIN_LOC_ROOT -check | grep "unknown" | wc -l` > 0) then
   echo " "
   echo "The following files were not found, this is informational only"
   ./check_input_data -inputdata $DIN_LOC_ROOT -check
   echo " "
endif

if (`./check_input_data -inputdata $DIN_LOC_ROOT -check | grep "missing" | wc -l` > 0) then
   echo "Attempting to download missing data:"
   ./check_input_data -inputdata $DIN_LOC_ROOT -export
endif 

if (`./check_input_data -inputdata $DIN_LOC_ROOT -check | grep "missing" | wc -l` > 0) then
   echo " "
   echo "The following files were not found, they are required"
   ./check_input_data -inputdata $DIN_LOC_ROOT -check
   echo "Invoke the following command to obtain them"
   echo "   ./check_input_data -inputdata $DIN_LOC_ROOT -export"
   echo " "
   exit -30
endif

if (($GET_REFCASE == 'TRUE') && ($RUN_TYPE != 'startup') && ($CONTINUE_RUN == 'FALSE')) then
  set refdir = "ccsm4_init/$RUN_REFCASE/$RUN_REFDATE"

  if !(-d $DIN_LOC_ROOT/$refdir) then
    echo "*****************************************************************"
    echo "ccsm_prestage ERROR: $DIN_LOC_ROOT/$refdir is not on local disk"
    echo "obtain this data from the svn input data repository:"
    echo "  > mkdir -p $DIN_LOC_ROOT/$refdir"
    echo "  > cd $DIN_LOC_ROOT/$refdir"
    echo "  > cd .."
    echo "  > svn export --force https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/$refdir"
    echo "or set GET_REFCASE to FALSE in env_run.xml, "
    echo "   and prestage the restart data to $RUNDIR manually"
    echo "*****************************************************************"
    exit -1
  endif 

  echo " - Prestaging REFCASE ($refdir) to $RUNDIR"
  if !(-d $RUNDIR) mkdir -p $RUNDIR || "cannot make $RUNDIR" && exit -1
  foreach file ($DIN_LOC_ROOT/$refdir/*${RUN_REFCASE}*) 
     if !(-f $RUNDIR/$file:t) then
        ln -s $file $RUNDIR || "cannot prestage $DIN_LOC_ROOT/$refdir data to $RUNDIR" && exit -1
     endif
  end
  cp $DIN_LOC_ROOT/$refdir/*rpointer* $RUNDIR || "cannot prestage $DIN_LOC_ROOT/$refdir rpointers to $RUNDIR" && exit -1

  cd $RUNDIR
  set cam2_list = `sh -c 'ls *.cam2.* 2>/dev/null'`
  foreach cam2_file ($cam2_list)
    set cam_file = `echo $cam2_file | sed -e 's/cam2/cam/'`
    ln -fs $cam2_file $cam_file
  end

  chmod u+w $RUNDIR/* >& /dev/null
endif

echo " CESM PRESTAGE SCRIPT HAS FINISHED SUCCESSFULLY"
echo "-------------------------------------------------------------------------"

mkdir -p /glade/derecho/scratch/tcraig/tmp

sleep 10
cd $RUNDIR
echo "`date` -- CSM EXECUTION BEGINS HERE" 

#-----
set launchjob = 1
while ($launchjob > 0)
@ launchjob = $launchjob + 1
#-----

setenv OMP_NUM_THREADS 1
if ( "$MPILIB" == "mpi-serial" ) then
  $EXEROOT/cesm.exe >&! ccsm.log.$LID
else
 mpiexec --cpu-bind depth -n 4087 -ppn 128 -d 1  $EXEROOT/cesm.exe >&! ccsm.log.$LID
endif

#-----
set fsize = `wc -c ccsm.log.$LID | cut -d " " -f 1`
if ($fsize > 500 || $launchjob > 3) then
  set launchjob = 0
  sleep 60
endif
end  # while launchjob
#-----

wait
echo "`date` -- CSM EXECUTION HAS FINISHED" 
# -------------------------------------------------------------------------
# Check for successful run
# -------------------------------------------------------------------------

set sdate = `date +"%Y-%m-%d %H:%M:%S"`

cd $RUNDIR
set CplLogFile = `ls -1t cpl.log* | head -1` 
if ($CplLogFile == "") then
  echo "Model did not complete - no cpl.log file present - exiting"
  exit -1
endif

# timeneeded is average time for 1 run + cushion, date in seconds, time in minutes                           
set date2 = `date "+%s"`
@ timeused = ((${date2} - ${date0}) / 60)
@ timeleft = ${timemax} - ${timeused}
@ timeneeded = (${timeused} / ${runcnt}) + 60
@ runtime = (${date2} - ${date1}) / 60
echo "runtime   was ${runtime} minutes for run ${runcnt}"
echo "timeused   is ${timeused} minutes"
echo "timeleft   is ${timeleft} minutes"
echo "timeneeded is ${timeneeded} minutes"
if (${timeneeded} > ${timeleft}) then
  echo "set runcnt = runmax, stopping loop"
  set runcnt = ${runmax}
endif


set slowrun = `grep 'SLOW RUN ABORT' $CplLogFile | wc -l`
if ($runcnt == 1 && $slowrun > 0) then
  echo "SLOW RUN DETECTED"
  cd $CASEROOT
  source ./Tools/ccsm_getenv

  # ability to stop slow run resubmit gracefully via RESUBMIT flag
  if ($RESUBMIT < 0) then
    echo "RESUBMIT env value < 0, stopping SLOW RUN AUTO-RESUBMIT"
    exit -1
  endif

  set batchsub = "$BATCHSUBMIT ./$CASE.run"
cat > tempres <<EOF
   $batchsub
EOF
  source tempres
  if ($status != 0) then
    echo "ccsm_postrun error: problem sourcing tempres "
  endif
  rm tempres
  echo "RUN RESUBMITTING due to SLOW RUN ABORT"
  exit -1
else
  grep 'SUCCESSFUL TERMINATION' $CplLogFile  || echo "Model did not complete - see $RUNDIR/$CplLogFile" && echo "run FAILED $sdate" >>& $CASEROOT/CaseStatus && exit -1
  echo "run SUCCESSFUL $sdate" >>& $CASEROOT/CaseStatus
endif

# -------------------------------------------------------------------------
# Update env variables in case user changed them during run
# -------------------------------------------------------------------------

cd $CASEROOT
source ./Tools/ccsm_getenv

# -------------------------------------------------------------------------
# Save model output stdout and stderr 
# -------------------------------------------------------------------------

if ($slowrun == 0) then

cd $RUNDIR
cp rsl.out.0000   rsl.logout.$LID
cp rsl.error.0000 rsl.logerr.$LID
gzip *.$LID
if ($LOGDIR != "") then
  if (! -d $LOGDIR/bld) mkdir -p $LOGDIR/bld || echo " problem in creating $LOGDIR/bld"
  cp -p *build.$LID.* $LOGDIR/bld  
  cp -p *log.$LID.*   $LOGDIR      
  cp -p rsl.log*.$LID.* $LOGDIR
endif
tar --remove-files -czf $RUNDIR/rsl.log.$LID.tar.gz rsl.{out,error}.*

# -------------------------------------------------------------------------
# Perform short term archiving of output
# -------------------------------------------------------------------------

if ($DOUT_S == 'TRUE') then
  echo "Archiving ccsm output to $DOUT_S_ROOT"
  echo "Calling the short-term archiving script st_archive.sh"
  cd $RUNDIR; $CASETOOLS/st_archive.sh
endif

# -------------------------------------------------------------------------
# Submit longer term archiver if appropriate
# -------------------------------------------------------------------------

cd $CASEROOT
if ($DOUT_L_MS == 'TRUE' && $DOUT_S == 'TRUE') then
  echo "Long term archiving ccsm output using the script $CASE.l_archive"
  set num = 0
  if ($LBQUERY == "TRUE") then
     set num = `$BATCHQUERY | grep $CASE.l_archive | wc -l`
  endif
  if ($LBSUBMIT == "TRUE" && $num < 1) then
# for spirit use, 
#   ssh $PBS_O_HOST /opt/pbs/default/bin/qsub ./$CASE.l_archive
cat > templar <<EOF
    $BATCHSUBMIT ./$CASE.l_archive
EOF
    source templar
    if ($status != 0) then
      echo "ccsm_postrun error: problem sourcing templar " 
    endif
    rm templar
  endif 
endif

# -------------------------------------------------------------------------
# RASM-NNA post-processing
# -------------------------------------------------------------------------

#set year = `ls -1t $RUNDIR/*.cpl.r* | head -1 | sed 's|.*\.cpl\.r\.\(.*\).nc|\1|' | cut -c 1-4`
#set month = `ls -1t $RUNDIR/*.cpl.r* | head -1 | sed 's|.*\.cpl\.r\.\(.*\).nc|\1|' | cut -c 6-7`
# /glade/work/tcraig/NNA/tools/nna_postproc_all_casper_asd01.csh $CASE $year $month

endif  # (slowrun == 0)

# -------------------------------------------------------------------------
# Resubmit another run script
#   for rerun loop, do not use RESUBMIT logic
#   but set CONTINUE_RUN to TRUE for next run.
#-----------------------------------------------------------------------

cd $CASEROOT
if ($runcnt < $runmax) then
  if ($?COMP_RUN_BARRIERS) then
    if (${COMP_RUN_BARRIERS} == "FALSE") then
       ./xmlchange -file env_run.xml -id CONTINUE_RUN -val TRUE
    endif
  else
    ./xmlchange -file env_run.xml -id CONTINUE_RUN -val TRUE
  endif
else

#-----------
if ($RESUBMIT > 0) then
    @ RESUBMIT = $RESUBMIT - 1
    echo RESUBMIT is now $RESUBMIT

    #tcraig: reset CONTINUE_RUN on RESUBMIT if NOT doing timing runs
    #use COMP_RUN_BARRIERS as surrogate for timing run logical
    if ($?COMP_RUN_BARRIERS) then
      if (${COMP_RUN_BARRIERS} == "FALSE") then
         ./xmlchange -file env_run.xml -id CONTINUE_RUN -val TRUE
      endif
    else
      ./xmlchange -file env_run.xml -id CONTINUE_RUN -val TRUE
    endif
    ./xmlchange -file env_run.xml -id RESUBMIT     -val $RESUBMIT

  if ($LBSUBMIT == "TRUE") then
    set batchsub = "$BATCHSUBMIT ./$CASE.run"
    echo "batchsub is $batchsub"
cat > tempres <<EOF
   $batchsub 
EOF
     source tempres
     if ($status != 0) then
       echo "ccsm_postrun error: problem sourcing tempres " 
     endif
     rm tempres
   endif 
endif
#-----------
endif   # if runcnt

# ability to stop runcnt gracefully via RESUBMIT flag
if ($RESUBMIT < 0) then
  echo "stopping runcnt loop by negative RESUBMIT value"
  set runcnt = $runmax
endif

if ($CHECK_TIMING == 'TRUE') then
  cd $CASEROOT
  if !(-d timing) mkdir timing
  $CASETOOLS/getTiming.csh -lid $LID 
  gzip timing/ccsm_timing_stats.$LID
endif

if ($SAVE_TIMING == 'TRUE') then
  cd $RUNDIR
  mv timing timing.$LID
  cd $CASEROOT
endif

#-----------------------------------------------------------------------
#
# rerun loop bottom
#
#-----------------------------------------------------------------------

echo "`date` -- CSM RUNCNT LOOP HAS FINISHED" 

#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
end   # while runcnt
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>