SLURM_JOB_NODELIST =  n[001-020]
num_proc =  20
host: n001
host: n002
host: n003
host: n004
host: n005
host: n006
host: n007
host: n008
host: n009
host: n010
host: n011
host: n012
host: n013
host: n014
host: n015
host: n016
host: n017
host: n018
host: n019
host: n020

==================================================================================================
mpiexec options:
----------------
  Base path: /home/kmccall/mpich-slurm-patch-install-4.0/bin/
  Launcher: ssh
  Debug level: 1
  Enable X: -1

  Global environment:
  -------------------
    LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64
    LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:
    ODBG=
    SLURM_NODEID=0
    SLURM_TASK_PID=359456
    NEEDLES_NUM_MANAGERS=20
    SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22
    SLURM_PRIO_PROCESS=0
    n_vars=-n_vars  54
    CDC_PREW2KHOST=rocci
    all_orders=-all_orders  0
    MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD
    LANG=en_US.UTF-8
    SLURM_SUBMIT_DIR=/home/kmccall/Needles2
    HISTCONTROL=ignoredups
    NEEDLES_INPUT_FILE=haystack_out.bin
    HOSTNAME=n001
    OLDPWD=/home/kmccall/Needles2
    NEEDLES_OUTPUT_FILE=needles_out.txt
    ENVIRONMENT=BATCH
    PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1
    LOADEDMODULES_modshare=intel/intelmpi:1
    CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov
    SLURM_PROCID=0
    SLURM_JOB_GID=513
    CDC_LOCALHOST=rocci.ndc.nasa.gov
    work_dir=/home/kmccall/Needles2
    SLURMD_NODENAME=n001
    FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov
    SLURM_TASKS_PER_NODE=1(x20)
    met_name=-metric m_disp_contin
    which_declare=declare -f
    LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1
    CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar
    XDG_SESSION_ID=61786
    MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl
    SLURM_NNODES=20
    USER=kmccall
    XTERM_SHELL=/bin/bash
    man_mas_cmd=NeedlesMpiMM
    job_id_cmd=-job_id 656
    PWD=/home/kmccall/Needles2
    SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass
    SLURM_JOB_NODELIST=n[001-020]
    HOME=/home/kmccall
    SLURM_CLUSTER_NAME=cluster
    SLURM_NODELIST=n[001-020]
    SSH_CLIENT=156.68.209.19 59333 22
    SLURM_NTASKS=20
    SLURM_JOB_CPUS_PER_NODE=24(x20)
    XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share
    SLURM_TOPOLOGY_ADDR=n001
    _LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1
    debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args 
    SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101
    SLURM_JOB_NAME=Ndls_592.293347072886
    TMPDIR=/tmp
    LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib
    SLURM_JOBID=123656
    NEEDLES_METRIC_NAME=m_disp_contin
    SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf
    n_ndxs=-n_ndxs  2
    LOADEDMODULES=intel/intelmpi
    XTERM_VERSION=XTerm(331)
    SLURM_NODE_ALIASES=(null)
    SLURM_JOB_QOS=normal
    SLURM_TOPOLOGY_ADDR_PATTERN=node
    DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6
    MAIL=/var/spool/mail/kmccall
    SLURM_CPUS_ON_NODE=24
    SLURM_JOB_NUM_NODES=20
    epoch_arg=-epoch 1645214633
    SHELL=/bin/bash
    TERM=xterm
    EPOCH=1645214633
    SLURM_JOB_UID=384580260
    TC_LIB_DIR=/usr/lib64/tc
    NEEDLES_NUM_NDXS=2
    STRACE=
    output_file=-output_file needles_out.txt
    SLURM_JOB_PARTITION=normal
    num_proc=20
    NEEDLES_NUM_WORKERS=23
    job_id=656
    SLURM_JOB_USER=kmccall
    SLURM_NPROCS=20
    CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov
    NEEDLES_ALL_ORDERS=0
    SHLVL=5
    SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov
    SLURM_JOB_ACCOUNT=users
    CDC_JOINED_DOMAIN=ndc.nasa.gov
    MANPATH=::
    worker_cmd=-worker_cmd NeedlesMpiWork
    WINDOWID=18874402
    workers_per_manager=-works_per_man 23
    manager_cmd=-manager_cmd NeedlesMpiMM
    GDK_BACKEND=x11
    MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles
    SLURM_GTIDS=0
    LOGNAME=kmccall
    DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030
    XDG_RUNTIME_DIR=/run/user/384580260
    MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1
    PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin
    SLURM_JOB_ID=123656
    _LMFILES_=/opt/modulefiles/intel/intelmpi
    in_file=-input_file haystack_out.bin
    MODULESHOME=/usr/share/Modules
    I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi
    HISTSIZE=1000
    USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV
    NEEDLES_NUM_VARS=54
    XTERM_LOCALE=en_US.UTF-8
    SLURM_LOCALID=0
    CDC_JOINED_SITE=MSFCPrivate
    LESSOPEN=||/usr/bin/lesspipe.sh %s
    BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}
    BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}
    BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}
    BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}
    BASH_FUNC_ml%%=() {  module ml "$@"
}
    _=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec

  Hydra internal environment:
  ---------------------------
    GFORTRAN_UNBUFFERED_PRECONNECTED=y


    Proxy information:
    *********************
      [1] proxy: n001 (1 cores)
      Exec list: hostname (1 processes); 

      [2] proxy: n002 (1 cores)
      Exec list: hostname (1 processes); 

      [3] proxy: n003 (1 cores)
      Exec list: hostname (1 processes); 

      [4] proxy: n004 (1 cores)
      Exec list: hostname (1 processes); 

      [5] proxy: n005 (1 cores)
      Exec list: hostname (1 processes); 

      [6] proxy: n006 (1 cores)
      Exec list: hostname (1 processes); 

      [7] proxy: n007 (1 cores)
      Exec list: hostname (1 processes); 

      [8] proxy: n008 (1 cores)
      Exec list: hostname (1 processes); 

      [9] proxy: n009 (1 cores)
      Exec list: hostname (1 processes); 

      [10] proxy: n010 (1 cores)
      Exec list: hostname (1 processes); 

      [11] proxy: n011 (1 cores)
      Exec list: hostname (1 processes); 

      [12] proxy: n012 (1 cores)
      Exec list: hostname (1 processes); 

      [13] proxy: n013 (1 cores)
      Exec list: hostname (1 processes); 

      [14] proxy: n014 (1 cores)
      Exec list: hostname (1 processes); 

      [15] proxy: n015 (1 cores)
      Exec list: hostname (1 processes); 

      [16] proxy: n016 (1 cores)
      Exec list: hostname (1 processes); 

      [17] proxy: n017 (1 cores)
      Exec list: hostname (1 processes); 

      [18] proxy: n018 (1 cores)
      Exec list: hostname (1 processes); 

      [19] proxy: n019 (1 cores)
      Exec list: hostname (1 processes); 

      [20] proxy: n020 (1 cores)
      Exec list: hostname (1 processes); 


==================================================================================================

[mpiexec@n001.cluster.pssclabs.com] Timeout set to -1 (-1 means infinite)
[mpiexec@n001.cluster.pssclabs.com] Got a control port string of n001.cluster.pssclabs.com:45705

Proxy launch args: /home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 

Arguments being passed to proxy 0:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n001 --global-core-map 0,1,20 --pmi-id-map 0,0 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 1:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n002 --global-core-map 0,1,20 --pmi-id-map 0,1 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 2:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n003 --global-core-map 0,1,20 --pmi-id-map 0,2 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 3:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n004 --global-core-map 0,1,20 --pmi-id-map 0,3 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 4:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n005 --global-core-map 0,1,20 --pmi-id-map 0,4 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 5:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n006 --global-core-map 0,1,20 --pmi-id-map 0,5 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 6:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n007 --global-core-map 0,1,20 --pmi-id-map 0,6 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 7:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n008 --global-core-map 0,1,20 --pmi-id-map 0,7 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 8:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n009 --global-core-map 0,1,20 --pmi-id-map 0,8 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 9:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n010 --global-core-map 0,1,20 --pmi-id-map 0,9 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 10:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n011 --global-core-map 0,1,20 --pmi-id-map 0,10 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 11:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n012 --global-core-map 0,1,20 --pmi-id-map 0,11 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 12:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n013 --global-core-map 0,1,20 --pmi-id-map 0,12 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 13:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n014 --global-core-map 0,1,20 --pmi-id-map 0,13 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 14:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n015 --global-core-map 0,1,20 --pmi-id-map 0,14 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 15:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n016 --global-core-map 0,1,20 --pmi-id-map 0,15 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 16:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n017 --global-core-map 0,1,20 --pmi-id-map 0,16 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 17:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n018 --global-core-map 0,1,20 --pmi-id-map 0,17 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 18:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n019 --global-core-map 0,1,20 --pmi-id-map 0,18 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

Arguments being passed to proxy 19:
--version 4.0 --iface-ip-env-name MPIR_CVAR_CH3_INTERFACE_HOSTNAME --hostname n020 --global-core-map 0,1,20 --pmi-id-map 0,19 --global-process-count 20 --auto-cleanup 1 --pmi-kvsname kvs_359459_0_1501630043_n001.cluster.pssclabs.com --pmi-process-mapping (vector,(0,20,1)) --global-inherited-env 123 'LD_LIBRARY_PATH=/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/mpich/lib:/opt/boost/lib:/usr/local/cuda/lib64:/home/kmccall/mpich-slurm-patch-install-4.0/lib:/opt/mpich/lib:/opt/boost/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:/usr/local/cuda/lib64' 'LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=01;05;37;41:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=01;36:*.au=01;36:*.flac=01;36:*.m4a=01;36:*.mid=01;36:*.midi=01;36:*.mka=01;36:*.mp3=01;36:*.mpc=01;36:*.ogg=01;36:*.ra=01;36:*.wav=01;36:*.oga=01;36:*.opus=01;36:*.spx=01;36:*.xspf=01;36:' 'ODBG=' 'SLURM_NODEID=0' 'SLURM_TASK_PID=359456' 'NEEDLES_NUM_MANAGERS=20' 'SSH_CONNECTION=156.68.209.19 59333 128.158.5.241 22' 'SLURM_PRIO_PROCESS=0' 'n_vars=-n_vars  54' 'CDC_PREW2KHOST=rocci' 'all_orders=-all_orders  0' 'MODULES_RUN_QUARANTINE=LD_LIBRARY_PATH LD_PRELOAD' 'LANG=en_US.UTF-8' 'SLURM_SUBMIT_DIR=/home/kmccall/Needles2' 'HISTCONTROL=ignoredups' 'NEEDLES_INPUT_FILE=haystack_out.bin' 'HOSTNAME=n001' 'OLDPWD=/home/kmccall/Needles2' 'NEEDLES_OUTPUT_FILE=needles_out.txt' 'ENVIRONMENT=BATCH' 'PATH_modshare=/usr/sbin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:1:/usr/bin:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:1:/usr/local/sbin:1:/opt/pssc/bin:1:/usr/local/cuda/bin:1:/usr/share/Modules/bin:1:/usr/local/bin:1' 'LOADEDMODULES_modshare=intel/intelmpi:1' 'CDC_JOINED_ZONE=CN=MSMCS,CN=NASA,CN=Zones,CN=Centrify,CN=Program Data,DC=ndc,DC=nasa,DC=gov' 'SLURM_PROCID=0' 'SLURM_JOB_GID=513' 'CDC_LOCALHOST=rocci.ndc.nasa.gov' 'work_dir=/home/kmccall/Needles2' 'SLURMD_NODENAME=n001' 'FI_PROVIDER_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib/prov' 'SLURM_TASKS_PER_NODE=1(x20)' 'met_name=-metric m_disp_contin' 'which_declare=declare -f' 'LD_LIBRARY_PATH_modshare=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/release:1:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib:1:/usr/local/cuda/lib64:1' 'CLASSPATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/lib/mpi.jar' 'XDG_SESSION_ID=61786' 'MODULES_CMD=/usr/share/Modules/libexec/modulecmd.tcl' 'SLURM_NNODES=20' 'USER=kmccall' 'XTERM_SHELL=/bin/bash' 'man_mas_cmd=NeedlesMpiMM' 'job_id_cmd=-job_id 656' 'PWD=/home/kmccall/Needles2' 'SSH_ASKPASS=/usr/libexec/openssh/gnome-ssh-askpass' 'SLURM_JOB_NODELIST=n[001-020]' 'HOME=/home/kmccall' 'SLURM_CLUSTER_NAME=cluster' 'SLURM_NODELIST=n[001-020]' 'SSH_CLIENT=156.68.209.19 59333 22' 'SLURM_NTASKS=20' 'SLURM_JOB_CPUS_PER_NODE=24(x20)' 'XDG_DATA_DIRS=/home/kmccall/.local/share/flatpak/exports/share:/var/lib/flatpak/exports/share:/usr/local/share:/usr/share' 'SLURM_TOPOLOGY_ADDR=n001' '_LMFILES__modshare=/opt/modulefiles/intel/intelmpi:1' 'debug_args=xterm  -hold -display localhost:16.0  -e gdb -x /home/kmccall/Needles2/gdb_cmds  -args ' 'SLURM_WORKING_CLUSTER=cluster:rocci.ndc.nasa.gov:6817:9216:101' 'SLURM_JOB_NAME=Ndls_592.293347072886' 'TMPDIR=/tmp' 'LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/lib' 'SLURM_JOBID=123656' 'NEEDLES_METRIC_NAME=m_disp_contin' 'SLURM_CONF=/var/spool/slurm/conf-cache/slurm.conf' 'n_ndxs=-n_ndxs  2' 'LOADEDMODULES=intel/intelmpi' 'XTERM_VERSION=XTerm(331)' 'SLURM_NODE_ALIASES=(null)' 'SLURM_JOB_QOS=normal' 'SLURM_TOPOLOGY_ADDR_PATTERN=node' 'DA_SESSION_ID_AUTH=644c6124-e8cd-a843-9a47-42896ca123c6' 'MAIL=/var/spool/mail/kmccall' 'SLURM_CPUS_ON_NODE=24' 'SLURM_JOB_NUM_NODES=20' 'epoch_arg=-epoch 1645214633' 'SHELL=/bin/bash' 'TERM=xterm' 'EPOCH=1645214633' 'SLURM_JOB_UID=384580260' 'TC_LIB_DIR=/usr/lib64/tc' 'NEEDLES_NUM_NDXS=2' 'STRACE=' 'output_file=-output_file needles_out.txt' 'SLURM_JOB_PARTITION=normal' 'num_proc=20' 'NEEDLES_NUM_WORKERS=23' 'job_id=656' 'SLURM_JOB_USER=kmccall' 'SLURM_NPROCS=20' 'CDC_JOINED_DC=ndmsadc20.ndc.nasa.gov' 'NEEDLES_ALL_ORDERS=0' 'SHLVL=5' 'SLURM_SUBMIT_HOST=rocci.ndc.nasa.gov' 'SLURM_JOB_ACCOUNT=users' 'CDC_JOINED_DOMAIN=ndc.nasa.gov' 'MANPATH=::' 'worker_cmd=-worker_cmd NeedlesMpiWork' 'WINDOWID=18874402' 'workers_per_manager=-works_per_man 23' 'manager_cmd=-manager_cmd NeedlesMpiMM' 'GDK_BACKEND=x11' 'MODULEPATH=/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/opt/modulefiles:/opt/modulefiles' 'SLURM_GTIDS=0' 'LOGNAME=kmccall' 'DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-yLUFutrxhK,guid=68d26626ec0087d02b45d80c620fb030' 'XDG_RUNTIME_DIR=/run/user/384580260' 'MODULEPATH_modshare=/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1' 'PATH=/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/opt/mpich/bin:/opt/mpich/bin:/usr/local/cuda/bin:/home/kmccall/mpich-slurm-patch-install-4.0/bin:.:/home/kmccall/.local/bin:/home/kmccall/bin:/opt/mpich/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/libfabric/bin:/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin:/usr/local/cuda/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/pssc/bin:/opt/pssc/bin' 'SLURM_JOB_ID=123656' '_LMFILES_=/opt/modulefiles/intel/intelmpi' 'in_file=-input_file haystack_out.bin' 'MODULESHOME=/usr/share/Modules' 'I_MPI_ROOT=/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi' 'HISTSIZE=1000' 'USER_PRINCIPAL_NAME=kmccall@NDC.NASA.GOV' 'NEEDLES_NUM_VARS=54' 'XTERM_LOCALE=en_US.UTF-8' 'SLURM_LOCALID=0' 'CDC_JOINED_SITE=MSFCPrivate' 'LESSOPEN=||/usr/bin/lesspipe.sh %s' 'BASH_FUNC_which%%=() {  ( alias;
 eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot "$@"
}' 'BASH_FUNC_module%%=() {  _module_raw "$@" 2>&1
}' 'BASH_FUNC__module_raw%%=() {  unset _mlshdbg;
 if [ "${MODULES_SILENT_SHELL_DEBUG:-0}" = '1' ]; then
 case "$-" in 
 *v*x*)
 set +vx;
 _mlshdbg='vx'
 ;;
 *v*)
 set +v;
 _mlshdbg='v'
 ;;
 *x*)
 set +x;
 _mlshdbg='x'
 ;;
 *)
 _mlshdbg=''
 ;;
 esac;
 fi;
 unset _mlre _mlIFS;
 if [ -n "${IFS+x}" ]; then
 _mlIFS=$IFS;
 fi;
 IFS=' ';
 for _mlv in ${MODULES_RUN_QUARANTINE:-};
 do
 if [ "${_mlv}" = "${_mlv##*[!A-Za-z0-9_]}" -a "${_mlv}" = "${_mlv#[0-9]}" ]; then
 if [ -n "`eval 'echo ${'$_mlv'+x}'`" ]; then
 _mlre="${_mlre:-}${_mlv}_modquar='`eval 'echo ${'$_mlv'}'`' ";
 fi;
 _mlrv="MODULES_RUNENV_${_mlv}";
 _mlre="${_mlre:-}${_mlv}='`eval 'echo ${'$_mlrv':-}'`' ";
 fi;
 done;
 if [ -n "${_mlre:-}" ]; then
 eval `eval ${_mlre} /usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash '"$@"'`;
 else
 eval `/usr/bin/tclsh /usr/share/Modules/libexec/modulecmd.tcl bash "$@"`;
 fi;
 _mlstatus=$?;
 if [ -n "${_mlIFS+x}" ]; then
 IFS=$_mlIFS;
 else
 unset IFS;
 fi;
 unset _mlre _mlv _mlrv _mlIFS;
 if [ -n "${_mlshdbg:-}" ]; then
 set -$_mlshdbg;
 fi;
 unset _mlshdbg;
 return $_mlstatus
}' 'BASH_FUNC_switchml%%=() {  typeset swfound=1;
 if [ "${MODULES_USE_COMPAT_VERSION:-0}" = '1' ]; then
 typeset swname='main';
 if [ -e /usr/share/Modules/libexec/modulecmd.tcl ]; then
 typeset swfound=0;
 unset MODULES_USE_COMPAT_VERSION;
 fi;
 else
 typeset swname='compatibility';
 if [ -e /usr/share/Modules/libexec/modulecmd-compat ]; then
 typeset swfound=0;
 MODULES_USE_COMPAT_VERSION=1;
 export MODULES_USE_COMPAT_VERSION;
 fi;
 fi;
 if [ $swfound -eq 0 ]; then
 echo "Switching to Modules $swname version";
 source /usr/share/Modules/init/bash;
 else
 echo "Cannot switch to Modules $swname version, command not found";
 return 1;
 fi
}' 'BASH_FUNC_ml%%=() {  module ml "$@"
}' '_=/home/kmccall/mpich-slurm-patch-install-4.0/bin/mpiexec' --global-user-env 0 --global-system-env 1 'GFORTRAN_UNBUFFERED_PRECONNECTED=y' --proxy-core-count 1 --exec --exec-appnum 0 --exec-proc-count 1 --exec-local-env 0 --exec-wdir /home/kmccall/Needles2 --exec-args 1 hostname 

[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n001 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 0 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n002 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 1 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n003 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 2 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n004 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 3 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n005 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 4 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n006 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 5 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n007 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 6 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n008 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 7 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n009 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 8 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n010 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 9 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n011 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 10 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n012 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 11 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n013 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 12 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n014 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 13 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n015 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 14 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n016 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 15 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n017 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 16 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n018 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 17 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n019 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 18 
[mpiexec@n001.cluster.pssclabs.com] Launch arguments: /usr/bin/ssh -x n020 "/home/kmccall/mpich-slurm-patch-install-4.0/bin/hydra_pmi_proxy" --control-port n001.cluster.pssclabs.com:45705 --debug --rmk user --launcher ssh --demux poll --pgid 0 --retries 10 --usize -2 --pmi-port 0 --gpus-per-proc -2 --proxy-id 19 

(gnome-ssh-askpass:359482): Gtk-WARNING **: 14:03:53.299: cannot open display: localhost:16.0

(gnome-ssh-askpass:359484): Gtk-WARNING **: 14:03:53.300: cannot open display: localhost:16.0

(gnome-ssh-askpass:359483): Gtk-WARNING **: 14:03:53.300: cannot open display: localhost:16.0

(gnome-ssh-askpass:359488): Gtk-WARNING **: 14:03:53.300: cannot open display: localhost:16.0

(gnome-ssh-askpass:359485): Gtk-WARNING **: 14:03:53.300: cannot open display: localhost:16.0

(gnome-ssh-askpass:359486): Gtk-WARNING **: 14:03:53.300: cannot open display: localhost:16.0

(gnome-ssh-askpass:359487): Gtk-WARNING **: 14:03:53.300: cannot open display: localhost:16.0
Host key verification failed.

(gnome-ssh-askpass:359489): Gtk-WARNING **: 14:03:53.301: cannot open display: localhost:16.0

(gnome-ssh-askpass:359490): Gtk-WARNING **: 14:03:53.301: cannot open display: localhost:16.0
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.

(gnome-ssh-askpass:359491): Gtk-WARNING **: 14:03:53.302: cannot open display: localhost:16.0

(gnome-ssh-askpass:359492): Gtk-WARNING **: 14:03:53.302: cannot open display: localhost:16.0

(gnome-ssh-askpass:359493): Gtk-WARNING **: 14:03:53.302: cannot open display: localhost:16.0

(gnome-ssh-askpass:359496): Gtk-WARNING **: 14:03:53.303: cannot open display: localhost:16.0

(gnome-ssh-askpass:359495): Gtk-WARNING **: 14:03:53.303: cannot open display: localhost:16.0

(gnome-ssh-askpass:359494): Gtk-WARNING **: 14:03:53.303: cannot open display: localhost:16.0

(gnome-ssh-askpass:359497): Gtk-WARNING **: 14:03:53.303: cannot open display: localhost:16.0
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.
Host key verification failed.

(gnome-ssh-askpass:359498): Gtk-WARNING **: 14:03:53.304: cannot open display: localhost:16.0

(gnome-ssh-askpass:359499): Gtk-WARNING **: 14:03:53.305: cannot open display: localhost:16.0
Host key verification failed.
Host key verification failed.
By accessing and using this information system, you acknowledge and consent to the following: You are accessing a U.S. Government information system, which includes: (1) this computer";" (2) this computer network";" (3) all computers connected to this network including end user systems";" (4) all devices and storage media attached to this network or to any computer on this network";" and (5) cloud and remote information services. This information system is provided for U.S. Government-authorized use only. You have no reasonable expectation of privacy regarding any communication transmitted through or data stored on this information system. At any time, and for any lawful purpose, the U.S. Government may monitor, intercept, search, and seize any communication or data transiting, stored on, or traveling to or from this information system. You are NOT authorized to process classified information on this information system. Unauthorized or improper use of this system may result in suspension or loss of access privileges, disciplinary action, and civil and/or criminal penalties.
By accessing and using this information system, you acknowledge and consent to the following: You are accessing a U.S. Government information system, which includes: (1) this computer";" (2) this computer network";" (3) all computers connected to this network including end user systems";" (4) all devices and storage media attached to this network or to any computer on this network";" and (5) cloud and remote information services. This information system is provided for U.S. Government-authorized use only. You have no reasonable expectation of privacy regarding any communication transmitted through or data stored on this information system. At any time, and for any lawful purpose, the U.S. Government may monitor, intercept, search, and seize any communication or data transiting, stored on, or traveling to or from this information system. You are NOT authorized to process classified information on this information system. Unauthorized or improper use of this system may result in suspension or loss of access privileges, disciplinary action, and civil and/or criminal penalties.
n001.cluster.pssclabs.com
n002.cluster.pssclabs.com
slurmstepd: error: *** JOB 123656 ON n001 CANCELLED AT 2022-02-18T14:04:42 ***
