#! /bin/tcsh -f

#################################################################################
if ($PHASE == set_batch) then
#################################################################################

source ./Tools/ccsm_getenv || exit -1

# Determine tasks and threads for batch queue 



set maxthrds = 0
set minthrds = $MAX_TASKS_PER_NODE
@ n = 0
foreach model ($MODELS)
  @ n = $n + 1
  if ($NTHRDS[$n] > $MAX_TASKS_PER_NODE ) then
     echo "ERROR, NTHRDS maximum is $MAX_TASKS_PER_NODE"
     echo "you have set NTHRDS = ( $NTHRDS[$n] ) - must reset"
     exit 1
  endif   
  if ($NTHRDS[$n] > $maxthrds) then
     set maxthrds = $NTHRDS[$n]
  endif
  if ($NTHRDS[$n] < $minthrds) then
     set minthrds = $NTHRDS[$n]
  endif
end


./xmlchange -file env_mach_pes.xml -id COST_PES -val 0
# This is the maximum number of mpi tasks we want on a node.
@ ptile = ${MAX_TASKS_PER_NODE} / 2
set ntasks_tot = `${CASEROOT}/Tools/taskmaker.pl -sumtasks`

if ($maxthrds > $minthrds) then
# We don't need this if all we are doing is exploiting hyperthreading
  if ( $maxthrds > 2) then
    set task_geo   = `${CASEROOT}/Tools/taskmaker.pl`
  endif
  set thrd_geo   = `${CASEROOT}/Tools/taskmaker.pl -thrdgeom`
else
  if ($maxthrds > 1) then
    @ ptile = $MAX_TASKS_PER_NODE / $maxthrds
  endif 
endif

@ nodes = ${ntasks_tot} / ${ptile}
if ( ${ntasks_tot} % ${ptile} > 0) then
  @ nodes = $nodes + 1
endif
# costpes is the number of nodes used * the number of cores per node
# or the total number of cores used, that needs to be set for cost
@ costpes = ${nodes} * ${PES_PER_NODE}

./xmlchange -file env_mach_pes.xml -id COST_PES -val ${costpes}

if ($?ACCOUNT) then
  set account_name = $ACCOUNT
else
  if (-e ~/.cesm_proj) then
     set account_name = `head -1 ~/.cesm_proj`
  else
     echo "WARNING: invalid account number P00000000 used by default"
     echo "WARNING: please use the ACCOUNT environment variable or ~/.cesm_proj to set this in the future"
     set account_name = P00000000
  endif
endif

set time_limit = "2:00"
if ($?QUEUE) then
  set queue_name = $QUEUE
  set queue_x = "-x"
else
# If we are using half a node or less go to the caldera shared nodes
  if ((${ntasks_tot} * ${maxthrds}) <= 8) then
    set queue_name = caldera
    set queue_x = ""
  else
    set queue_x = "-x"
    if ($CCSM_ESTCOST > 0) set time_limit = "4:00"
    if ($CCSM_ESTCOST > 1) set time_limit = "8:00"
    if ( ${costpes} > 16384 ) then
      set queue_name = capability
    else  
      set queue_name = regular
    endif
  endif
endif


if ($?TESTMODE) then
 set file = $CASEROOT/${CASE}.test 
else
 set file = $CASEROOT/${CASE}.run 
endif

cat >! $file << EOF1
#! /bin/tcsh -f
#BSUB -n $ntasks_tot
#BSUB -R "span[ptile=${ptile}]"
#BSUB -q ${queue_name}
#BSUB -N
#BSUB -a poe
#BSUB ${queue_x}
#BSUB -o cesm.stdout.%J
#BSUB -e cesm.stderr.%J
#BSUB -J $CASE
#BSUB -W ${time_limit}
#BSUB -P ${account_name}
#BSUB -R "select[scratch_ok > 0]"
setenv OMP_NUM_THREADS $maxthrds
if($maxthrds>1) then
  setenv MP_TASK_AFFINITY cpu:$maxthrds
endif
EOF1
if ($?task_geo) then
cat >> $file << EOF2
setenv LSB_PJL_TASK_GEOMETRY "{$task_geo}"
EOF2
endif

#################################################################################
else if ($PHASE == set_exe) then
#################################################################################


cat >> ${CASEROOT}/${CASE}.run << EOF1
# -------------------------------------------------------------------------
# Run the model
# -------------------------------------------------------------------------

cd \$RUNDIR
echo "\`date\` -- CSM EXECUTION BEGINS HERE" 


setenv MP_LABELIO yes
sleep 10
if ( "\$MPILIB" == "mpi-serial" ) then
    \$EXEROOT/cesm.exe >&! cesm.log.\$LID
else
   mpirun.lsf  \$EXEROOT/cesm.exe >&! cesm.log.\$LID
endif

wait
echo "\`date\` -- CSM EXECUTION HAS FINISHED" 

# -------------------------------------------------------------------------
# For Postprocessing
# -------------------------------------------------------------------------
EOF1


#################################################################################
else if ($PHASE == set_larch) then
#################################################################################

if (-e ${CASEROOT}/${CASE}.l_archive) then
   echo ERROR: generate_batch.csh file ${CASE}.l_archive already exists in ${CASEROOT}
   exit -1
endif

# Create the archive script
touch ${CASEROOT}/${CASE}.l_archive
chmod 775 ${CASEROOT}/${CASE}.l_archive

if ($?ACCOUNT) then
  set account_name = $ACCOUNT
else
  if (-e ~/.cesm_proj) then
     set account_name = `head -1 ~/.cesm_proj`
  else
     set account_name = P00000000
  endif
endif

cat >! $CASEROOT/${CASE}.l_archive << EOF1
#! /bin/csh -f
#BSUB -n 1
#BSUB -o poe.harout.%J
#BSUB -e poe.harerr.%J
#BSUB -W 3:00
#BSUB -q caldera
#BSUB -P ${account_name}
#BSUB -J $CASE.l_archive

#-----------------------------------------------------------------------
# Determine necessary environment variables
#-----------------------------------------------------------------------

cd $CASEROOT 

source ./Tools/ccsm_getenv || exit -1

#----------------------------------------------
# run the long term archiver
#----------------------------------------------

cd \$DOUT_S_ROOT
\$CASEROOT/Tools/lt_archive.sh -m copy_dirs_hsi
exit 0

EOF1

#################################################################################
else
#################################################################################

    echo "  PHASE setting of $PHASE is not an accepted value"
    echo "  accepted values are set_batch, set_exe and set_larch"
    exit 1

#################################################################################
endif
#################################################################################