add jlse runners, more flexible node counter

jlse/job.pbs

@@ -0,0 +1,21 @@
+#!/bin/bash
+#COBALT -t 00:20:00
+#COBALT -n 2
+#COBALT --jobname cublas-nsys-test
+#COBALT -O cublas-nsys-test
+#COBALT -q gpu_v100_smx2
+
+cd $HOME/hpc/mpi-cuda/jlse
+pwd
+source ./setup.sh
+which mpirun
+which nsys
+
+./run.sh noum none 2 4
+./run.sh noum nsys 2 4
+./run.sh noum none 1 4 &
+./run.sh noum nsys 1 4
+wait
+./run.sh noum none 1 2 &
+./run.sh noum nsys 1 2
+wait

jlse/run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ $# -ne 4 ]; then
+  echo "Usage: $0 um|noum nsys|nvprof|none nodes ppn"
+  exit 1
+fi
+
+um=$1
+prof=$2
+nodes=$3
+ppn=$4
+
+tag=${um}_${prof}_${nodes}_${ppn}
+
+if [ $prof == "nsys" ]; then
+  prof_cmd="nsys profile --kill=none -c cudaProfilerApi -o profile/${tag}.%q{PMIX_RANK}"
+elif [ $prof == "nvprof" ]; then
+  prof_cmd="nvprof -o profile/nvprof.%q{PMIX_RANK}.nvvp --profile-from-start off"
+else
+  prof_cmd=""
+fi
+
+if [ $um == "um" ]; then
+  cmd=./mpi_daxpy_nvtx_managed
+else
+  cmd=./mpi_daxpy_nvtx_unmanaged
+fi
+
+total_procs=$((ppn * nodes))
+
+set +x
+mpirun -np $total_procs \
+   $prof_cmd $cmd >out-${tag}.txt 2>&1
+set -x

jlse/setup.sh

@@ -0,0 +1,5 @@
+source $HOME/fusion/spack/ivolta86/share/spack/setup-env.sh
+spack load -r openmpi
+
+module use $HOME/soft/modulefiles
+module load nsight-systems