#!/bin/csh
#  Request maximum wallclock time for job
#PBS -l walltime=01:00:00
#  select=<number nodes>,ncpus=<cores/node>,mpiprocs=<MPI procs/node>
#  Total cores requested = <number of nodes> X <MPI procs/node>
#PBS -l select=1:ncpus=8:mpiprocs=2
#  Specify how to distribute MPI procs. across nodes
#PBS -l place=scatter:excl
#  Request job name
#PBS -N linux_ansys
#  Request PBS job queue for job
#PBS -q standard
#  Specify to keep stdout/stderr job files
#PBS -k eo
# Indicate Project ID
#PBS -A ARLAP96090PIO
#  Request environment variables be exported from script
#PBS -V
#
set JOBID=`echo $PBS_JOBID | cut -f1 -d.`
echo Ansys job id  $JOBID starting at `date` on `hostname`
# Change directory input files area
cd ${HOME}/ansys/input
echo starting in `pwd`

## Get the execution host name and the number of cpus
set exechost=`hostname -s`
set NSLOTS=`wc -l $PBS_NODEFILE | cut -f1 -d " "`
echo NSLOTS is $NSLOTS
echo exechost is $exechost

set jobhost=`hostname -s`
set jobid=$JOBID
chmod -R 755 *${jobid}*
set cdir=`pwd`
set outfile=out.log
set outdir=`pwd`/${JOBID}
set TMPD=/usr/var/tmp/$LOGNAME/$JOBID
if (! -e $TMPD ) mkdir -p $TMPD
if (! -e $outdir ) mkdir -p $outdir
chmod -R 755 *${jobid}*

set input=vmd3.dat
cp $input $TMPD
cd $TMPD
ls -l

module load ansys130
module show ansys130
module list
setenv OMP_NUM_THREADS $NSLOTS

echo NUM_PROC=$NSLOTS > config90.ans # Create config file for multi-processors

ansys130 -p ANE3FL -b -m -db -j $input < $input >& $outfile

set st=$status
echo execution ended at `date` with status $st

#------------Post Processing Phase------------------------------
set MAIN_JOBID=$JOBID
ls -l

tar cf ../output.tar.$MAIN_JOBID *
set st=$status
if ( $st != 0 ) then
  echo Tar of files failed, look in $TMPD for output files!
  exit $st
endif

mv ../output.tar.$MAIN_JOBID $outdir
cd $outdir
tar xf output.tar.$MAIN_JOBID
set st=$status
if ( $st != 0 ) then
  echo Tar of files failed, look in $TMPD for output files!
  exit $st
endif

echo output file mv for job $MAIN_JOBID ended at `date` with status $st

if ( $st == 0 ) then
  echo files copied back to $outdir successfully, will rmdir $TMPD
  rm -r $TMPD
endif