hacky multi-node support

assumes 6 procs per node

summit/job.lsf

@@ -0,0 +1,16 @@
+#!/bin/bash
+#BSUB -P fus123
+#BSUB -W 0:20
+#BSUB -nnodes 2
+#BSUB -J cublas-nsys-test
+#BSUB -o cublas-nsys-test.%J
+#BSUB -q debug
+
+./run.sh noum none 6 2
+./run.sh noum nsys 6 2
+./run.sh noum none 6 1 &
+./run.sh noum nsys 6 1
+wait
+./run.sh noum none 3 1 &
+./run.sh noum nsys 3 1
+wait

summit/run.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+if [ $# -ne 4 ]; then
+  echo "Usage: $0 um|noum nsys|nvprof|none nodes ppn"
+  exit 1
+fi
+
+um=$1
+prof=$2
+nodes=$3
+ppn=$4
+
+tag=${um}_${prof}_${nodes}_${ppn}
+
+if [ $prof == "nsys" ]; then
+  prof_cmd="nsys profile --kill=none -c cudaProfilerApi -o profile/${tag}.%q{PMIX_RANK}"
+elif [ $prof == "nvprof" ]; then
+  prof_cmd="nvprof -o profile/nvprof.%q{PMIX_RANK}.nvvp --profile-from-start off"
+else
+  prof_cmd=""
+fi
+
+if [ $um == "um" ]; then
+  cmd=./mpi_daxpy_nvtx_managed
+else
+  cmd=./mpi_daxpy_nvtx_unmanaged
+fi
+
+set +x
+jsrun --smpiargs="-gpu" -n$nodes -c$ppn -g$ppn -a$ppn \
+   $prof_cmd $cmd >out-${tag}.txt 2>&1
+set -x