#!/bin/csh
#  Request maximum wallclock time for job
#PBS -l walltime=01:00:00
#  select=<number nodes>,ncpus=<cores/node>,mpiprocs=<MPI procs/node>
#  Total cores requested = <number of nodes> X <MPI procs/node>
#PBS -l select=2:ncpus=8:mpiprocs=8
#PBS -l application=lsdyna
#  Specify how to distribute MPI procs. across nodes
#PBS -l place=scatter:excl
#  Request LS-DYNA licenses
#PBS -l lsdyna=16
#  Request job name
#PBS -N linux_lsdyna
#  Request PBS job queue for job
#PBS -q standard
#  Specify to keep stdout/stderr job files
#PBS -k eo
#  Select Project ID
#PBS -A ARLAP96090PIO
#  Request environment variables be expported from script
#PBS -V
#
set JOBID=`echo $PBS_JOBID | cut -f1 -d.`
echo job $JOBID starting at `date` on `hostname`
# Change directory input files area
cd ${HOME}/ls-dyna/input
echo starting in `pwd`

## Get the execution host name and the number of cpus
#set exechost=`hostname -s | awk -F'-' '{print $1}'`
set exechost=`hostname -s`
echo exechost is $exechost
set NSLOTS=`wc -l $PBS_NODEFILE | cut -f1 -d " "`
echo NSLOTS is $NSLOTS

set jobhost=`hostname -s`
set jobid=$JOBID
chmod -R 755 *${jobid}*
set outfile=out.log
set outdir=`pwd`/$JOBID
set cdir=`pwd`
set TMPD=/usr/var/tmp/$LOGNAME/$JOBID
if (! -e $TMPD ) mkdir -p $TMPD
if (! -e $outdir ) mkdir -p $outdir
chmod -R 755 *${jobid}*

set input=fric.k
#setenv OMP_NUM_THREADS $NSLOTS

cp $input $TMPD
cd $TMPD
ls -l

#========================================================================
set host=""
foreach new_host (`cat $PBS_NODEFILE`)
  if ($new_host != $host) then
    set host=$new_host
    echo "Working on $host ...."
    /usr/bin/ssh -o StrictHostKeyChecking=no $host pwd
  endif
end
#========================================================================


unlimit
module load ls-dyna
module show ls-dyna
module list

set executable=mpp971_d
echo starting `which $executable`
set full_exec=`which $executable`
set mpi_exec=`which mpirun`

echo input file is $input

set appfil=lsdyna-appfile.${JOBID}
touch $appfil
set mppcmd="$executable NCPU=$NSLOTS i=${TMPD}/$input"
@ count=1
   while ( $count <= $NSLOTS )
          set noden=`echo $HOSTS | cut -f${count} -d" "`
          echo "-h $noden -np 1 -e PATH=`echo $PATH` -e LSTC_LICENSE=`echo $LSTC_LICENSE` -e LSTC_LICENSE_SERVER=`echo $LSTC_LICENSE_SERVER` -e LSTC_LICENSE_SERVER_PORT=`echo $LSTC_LICENSE_SERVER_PORT` -e LSTC_FILE=`echo $LSTC_FILE` -e LD_LIBRARY_PATH=`echo $LD_LIBRARY_PATH` -e MPI_WORKDIR=/usr/var/tmp/$LOGNAME/$JOBID -e MP_SINGLE_THREAD=`echo $MP_SINGLE_THREAD` -e MP_WAIT_MODE=`echo $MP_WAIT_MODE` -e MPI_PAGE_ALIGN_MEM=1 $mppcmd" >> $appfil

echo "---------------------------"
cat $appfil
echo starting /usr/cta/ls-dyna/971/hpmpi/bin/mpirun -e MPI_REMSH=/usr/bin/ssh -prot -f $appfil

$mpi_exec  -prot -udapl -e MPI_HASIC_VAPI=1 -e MPI_USE_MALLOPT_SBRK_PROTECTION=1 -e MPI_USE_MALLOPT_AVOID_MMAP=1 -f $appfil >& $outfile

set st=$status
echo LS-DYNA ended with status $st on `date`

set MAIN_JOBID=$JOBID
ls -l

tar cf ../output.tar.$MAIN_JOBID *
set st=$status
if ( $st != 0 ) then
  echo Tar of files failed, look in $TMPD for output files!
  #exit $st
endif

mv ../output.tar.$MAIN_JOBID $outdir
cd $outdir
tar xf output.tar.$MAIN_JOBID
set st=$status
if ( $st != 0 ) then
  echo Tar of files failed, look in $TMPD for output files!
  #exit $st
endif

echo output file mv for job $MAIN_JOBID ended at `date` with status $st

if ( $st == 0 ) then
  echo files copied back to $outdir successfully, will rmdir $TMPD
  rm -r $TMPD
endif