#!/usr/bin/env sh
#PBS -l nodes=1:ppn=12
#PBS -N rel160
#PBS -o oe-${PBS_JOBID%%.*}.out
#PBS -j oe
#PBS -V
#PBS -m n
#PBS -q c4
# Send me mail on job start, job end and if job aborts
#PBS -M betul.pamuk@cornell.edu
#PBS -m bea
# do not rerun this job if it fails
#PBS -r n
######################################################################
INFILES=*
#INFILES="INCAR KPOINTS POTCAR POSCAR WAVECAR log"
OUTFILES=*
EXEC="/home/fs03/bp385/SOFT/qe-6.3/bin"
######################################################################
#Setup---normally no need to change anything in this section.
#Check to see if tmpdir is set, if not complain and provide it as /tmp
if [ -z "$TMPDIR" ];
then
	echo "Torque did not set TMPDIR! Check with your admin, God knows what else is broken ..."
	TMPDIR=/tmp
	echo "TMPDIR has now been set manually, value: $TMPDIR"
else
	echo "TMPDIR has been provided by Torque: $TMPDIR"
fi
#Count the number of processors requested
np=$(wc -l < $PBS_NODEFILE)
#Count the number of nodes requested
nodes=$(uniq $PBS_NODEFILE | wc -l)
#Create a uniqued up nodefile
uniqNodeFile=$TMPDIR/uniqmachinefile
uniq $PBS_NODEFILE > $uniqNodeFile
#Create once per node and once per process alias
alias perHostMpiExec='mpiexec -machinefile $uniqNodeFile -np $nodes'
alias perProcMpiExec='mpiexec -machinefile $PBS_NODEFILE -np $np'
#Start up mpd's one/node
mpdboot -f $uniqNodeFile -r ssh -n $nodes
mpdtrace -l -v
######################################################################
. /usr/share/Modules/init/sh
module purge 
. /usr/share/Modules/init/sh
module load openmpi-1.6.2-intel-x86_64
. /usr/share/Modules/init/sh
module load intel16-compilers

#Moving data from $HOME to local temporary disk:
DATASOURCE=$PBS_O_WORKDIR
cd $DATASOURCE
perHostMpiExec cp -r $INFILES $TMPDIR
######################################################################
#Configuration of process pinning:
#Generally speaking, pinning processes is a good thing, however, if running
#multiple jobs on the same node, you can run into problems as Intel MPI
#assumes that it can start at cpu 0 and walk, which isn't neccesarily true.
#The PIN_CONTROL variables may be set to [PIN | NOPIN | SMARTPIN]
#
#       PIN - allow intel MPI to PIN the processes, useful if ppn = 8 or
#               if the nodes are restricted to one job per node
#       NOPIN - disable intel MPI pinning and put scheduler into "BATCH" mode
#               which should help reduce swapping.  SAFE MODE!
#       SMARTPIN - check to see if there are vasp processes running and if so
#               pin the processes onto cpus offset by the number of running
#               vasp processes.  A bit dangerous and ONLY detects vasp processes.
PIN_CONTROL="NOPIN"
#Control code for pin mode:
if [ "$PIN_CONTROL" = "PIN" ];
then
        echo "Enabling Intel MPI fixed pinning ..."
        export I_MPI_PIN=enable
fi
if [ "$PIN_CONTROL" = "NOPIN" ];
then
        echo "Disabling processor pinning ..."
        export I_MPI_PIN=disable
        schedtool -3 $$
fi
if [ "$PIN_CONTROL" = "SMARTPIN" ];
then
        echo "Attempting to pin processes on free CPUs ..."
        vasps=$(/sbin/pidof vasp | wc -w)
        export I_MPI_PIN=enable
        export I_MPI_PIN_PROCS=$vasps-7
fi
####################################################################
DATE=$(date)
echo ""
echo "Time at the start of the job is: $DATE"
echo ""
######################################################################
#Execute code on ALL available processors (nodes*ppn) from the directory
#where the input data was moved to.
OUTDIR="$TMPDIR"
if [ ! -d $OUTDIR ]; #True unless TMPDIR wasn't set by Torque
	then
	mkdir $OUTDIR
fi
touch $DATASOURCE/running_${PBS_JOBID%%.*}
cd $OUTDIR
echo "Starting the job ..."
######perProcMpiExec $EXEC >> log
#perProcMpiExec $EXEC/pw.x -npool 2 -inp $TMPDIR/Co3O4.vcrelax.in >$TMPDIR/Co3O4.vcrelax.out
perProcMpiExec $EXEC/pw.x -inp $TMPDIR/RuO2.vcrelax.in >$TMPDIR/RuO2.vcrelax.out
#perProcMpiExec $EXEC/ph.x -inp $TMPDIR/Co3O4.ph.in >$TMPDIR/Co3O4.ph.out
#perProcMpiExec $EXEC/pw.x -inp $TMPDIR/Co3O4.bands.in >$TMPDIR/Co3O4.bands.out
#perProcMpiExec $EXEC/bands.x -inp $TMPDIR/bands.in >$TMPDIR/bands.out
#perProcMpiExec $EXEC/dos.x -inp $TMPDIR/dos.in >$TMPDIR/dos.out
#####################################################################
DATE=$(date)
echo ""
echo "Time at the end of the job is: $DATE"
echo ""
#####################################################################
#Move results back:
cp $OUTFILES $DATASOURCE
cp -r * $DATASOURCE
cp $DATASOURCE
####################################################################
#Cleanup:
rm $DATASOURCE/running_*
mpdallexit
####################################################################