#!/bin/csh
#  Request maximum wallclock time for job
#PBS -l walltime=00:20:00
#  select=<number nodes>,ncpus=<cores/node>,mpiprocs=<MPI procs/node>
#  Total cores requested = <number of nodes> X <MPI procs/node>
#PBS -l select=2:ncpus=8:mpiprocs=8
#PBS -l application=fluent
#  Specify how to distribute MPI procs. across nodes
#PBS -l place=scatter:excl
#  Request FLUENT licenses
#PBS -l fluent=16
#  Request job name
#PBS -N linux_fluent
#  Request PBS job queue for job
#PBS -q standard
#  Specify to keep stdout/stderr job files
#PBS -k eo
#  Select Project ID
#PBS -A ARLAP96090PIO
#  Request environment variables be expported from script
#PBS -V
#
set JOBID=`echo $PBS_JOBID | cut -f1 -d.`
set NSLOTS=`wc -l $PBS_NODEFILE | cut -f1 -d " "`
echo job $JOBID starting at `date` on `hostname`
# Change directory input files area
cd ${HOME}/fluent/input
echo starting in `pwd`
module load fluent6.3.26
which fluent
#
# ARCHIVE LOCATION
# this is where to put your files before running this script file                          
#    *dat.gz   *cas.gz   *jou   *c                                                         
set indir=`pwd`
echo the input directory is $indir
set outdir=${HOME}/SCR/fluent/$JOBID
mkdir -p $outdir
echo the output directory is $outdir
#
# TEMPORARY WORKSPACE LOCATION 
set TMPD=/usr/var/tmp/$LOGNAME/$JOBID
# create temp directory in case it's not there 
if (! -e $TMPD) mkdir -p $TMPD   
echo the working directory is $TMPD   
#
set readdir=$TMPD        
set casedir=$TMPD
set datdir=$TMPD   
set writedir=$TMPD
#INPUTS          
set infile=inputp2.1    
set outfile=fluent_out.txt
set cas_in=testp2.cas
set dat_in=testp2.dat
#END INPUT      
# *  *  *  *  *  *  *  *  *      
#NO INPUT BEYOND THIS POINT    
# *  *  *  *  *  *  *  *  *                          # 
# Write out directories set:         
echo readdir:  $readdir      
echo casedir:  $casedir           
echo datdir:   $datdir 
echo writedir: $writedir                             #    
# -- copy files to working directory
#    *dat.gz   *cas.gz   *jou                     
#
cd $TMPD            
# COPY CASE FILE
cp $indir/$cas_in .
echo fluent case file $cas_in copied from $indir to $TMPD 
# COPY DATA FILE     
if ( $dat_in != 0 ) then
  cp $indir/$dat_in .
  echo fluent data file $dat_in copied from $indir to $TMPD
else
  echo fluent data file not required as input
endif
# COPY JOURNAL FILE
cp $indir/$infile .
echo fluent journal file $infile copied from $indir to $TMPD
# make sure input files are here before running   
# so that FLUENT will not hang
# should check for *.cas or *.cas.gz, etc.
chmod -R 750 *
ls -l
date
#
if ( -e $readdir/$infile) then
   echo Fluent journal file $infile was found
   if ( -e $casedir/$cas_in ) then
     echo $cas_in was found
     echo will now attempt to run fluent
     echo start fluent
     fluent 2d -g -pib.dapl -i $readdir/$infile -t${NSLOTS} -ssh -cnf=$PBS_NODEFILE 
     set st=$status
     echo execution ended at `date` with status $st
     if ( $st != 0 ) then
       echo exiting with bad status, look for files in $TMPD on `hostname`                 
       exit $st
     endif 
     unset status      
   else
      echo CASE or DATA file missing
      echo FLUENT not started
   endif   
else
   echo INPUT file missing
   echo FLUENT not started
endif
echo check $outfile to determine if Fluent ran successfully                                
echo job complete                                    #                                                   
#rm ~/.fluent.env
#
#
#  archive files now
#
echo "...copying output tar file to archive"
set MAIN_JOBID=$JOBID

if (! -e $outdir ) mkdir -p $outdir
cd $TMPD
ls -l

tar cf ../output.tar.$MAIN_JOBID *
set st=$status
if ( $st != 0 ) then
  echo Tar of files failed, look in $TMPD for output files!
  exit $st
endif

cp ../output.tar.${MAIN_JOBID} $outdir
set st=$status
if ( $st != 0 ) then
  echo tar of file failed, look in $TMPD for output files!
  exit $st
endif
echo output file copy for job $MAIN_JOBID ended at `date` with status $st
cd $outdir
tar xf output.tar.${MAIN_JOBID}
set st=$status
if ( $st != 0 ) then
  echo Tar of files failed, look in $TMPD for output files!
  exit $st
endif

echo output file copy for job $MAIN_JOBID ended at `date` with status $st
if ( $st == 0 ) then
  echo files copied back to $outdir successfully, will rmdir $TMPD
  rm -R $TMPD
endif
cd ..
chmod -R 755 $outdir