#! /bin/sh

### runra: marty sereno -- run n recon-all's in bg, rotate new in when done
# 14/01/09 -- 01a: initial csh hack
# 14/01/10 -- 01b: cleanup, comment, bold start/stop msgs, ps uxww on Linux
# 14/01/12 -- port to pure sh, func to write listmemb (what an ugly language!)
# 14/10/20 -- 01c: save list of curr running subj names, use when one finishes
# 18/05/14 -- 01d: INPROG add tksurfer
#TODO: redirect stdout/stderr of bg process to log, so can completely detach
#TODO: generalize to allow run cmdline tksurfer tcl script

############################################################################
# user-editable parms edon't change the variable names: e.g., sublist)
############################################################################
#xxxxxxx
# parallel run for each each subject in list
prog=recon-all
#prog=tksurfer

# list of subjects to reconstruct (recon-all will create these directories)
sublist="\
 AMIELB ANNASH ANNSNI ARADAV AURRUS BENOHA CHAHAR CHALOC CHYHIL DAVRAU \
 ETHTIL FELPIC FLOHOC GEOSAN GRADEN HANMOR ISADES JACADD JOKHER JONTIL \
 JOSDAV LOLYOU NELCAM PHIKEL PHIPEL REBBRO RHITOM RHYHOL SALKRI VICKNO"

#xxxxxxx
# hemi (for TODO: tksurfer t1morph.tcl)
hemi=rh
#hemi=lh

# dir outside SUBJECTS_DIR w/rawscan subject dirs in it w/same names as above
rawdir=/media/Data/Inverse/TessaData/MRI_RECON

# name of rawscan NIFTI files (currently req'd to be same for all)
rawscan=MPRAGE1.nii

# maximum number of background processes to run
maxprocs=6
############################################################################

### help
log=./runra.$$.log
helpflag=1
debugflag=0
if [ $# -eq 1 ]; then
  arg=$1
  if [ "$arg" = "-test" ] || [ "$arg" == "-debug" ]; then
    helpflag=0
    debugflag=1
  fi
  if [ "$arg" = "-go" ]; then
    helpflag=0
  fi
fi
if [ $helpflag -eq 1 ]; then
  echo ""
  echo "use: runra [-go,-test]"
  echo ""
  echo "  --cp script and edit 4 variables at top of it"
  echo '  --requires MGH freesurfer env setup (recon-all, others on $path)'
  echo '  --start/finish of each subj written to ./runra.<pid>.log'
  echo "  --option -test makes/runs small sh sleep script for demo"
  echo ""
  echo "  Example raw scan directories:"
  echo "    /tmp/rawdata/marty/MPRAGE1.nii"
  echo "    /tmp/rawdata/trevor/MPRAGE1.nii"
  echo "    /tmp/rawdata/tessa/MPRAGE1.nii"
  echo "  Resulting subject dirs:"
  echo '    $SUBJECTS_DIR/marty'
  echo '    $SUBJECTS_DIR/trevor'
  echo '    $SUBJECTS_DIR/tessa'
  echo ""
  echo "  Current state of editable parms:"
  echo ""
  echo "   (1) sublist = $sublist"
  echo "   (2) rawdir = $rawdir"
  echo "   (3) rawscan = $rawscan"
  echo "   (4) maxprocs = $maxprocs"
  echo "                                                       version: 01d"
  exit
fi

### internal vars
chkinterval=15
#xxxxxxx
#prog=recon-all
tmpscript=/tmp/TmpShRunra.$$

### funcs
cleanupfunc() {     # use: no args: kill any remaining
  echo "runra: cleanup"
  if [ $debugflag -eq 1 ]; then rm -f $tmpscript; fi
  if [ -n "${pids+_}" ]; then   # test exist
    i=1
    while [ $i -le $maxprocs ]; do
      # ugly sh way to get i'th member of list (no arrays in sh)
      set -- $pids        # redef positional parms $1,$2,..
      eval pid="\$${i}"   # construct $1,etc w/eval
      if [ "$pid" != "-1" ]; then
        kill -9 $pid
        echo ""
        echo "###############################################################"
        echo "runra: killed pid: $pid"
        echo "runra: killed pid: $pid" >> $log
        echo "###############################################################"
        echo ""
      fi
      i=`expr $i + 1`
    done
  fi
  if [ $finishedflag -eq 1 ]; then
    echo ""
    echo "###############################################################"
    echo "runra: done w/sublist: $sublist"
    echo "runra: done w/sublist: $sublist" >> $log
    echo "runra: finished: `date`"
    echo "runra: finished: `date`" >> $log
    echo "###############################################################"
    echo ""
  fi
  exit
}
trap 'cleanupfunc' INT   # catch ctrl-C

writelistmemb () {    # to use: writelistmemb listname $i $newval
  local listname=$1
  local ind=$2
  local val=$3
  local _loclist=""
  local _loclistnew=""
  local i=1    # one-based (or 0 for zero-based)
  eval "_loclist=\$$listname"
  for _memb in $_loclist; do    # slow: parse, rewrite entire list
    if [ $i = $ind ]; then _memb2=$val; else _memb2=$_memb; fi
    _loclistnew="$_loclistnew $_memb2"
    i=`expr $i + 1`
  done
  eval "$listname=\$_loclistnew"
}

### debug
if [ $debugflag -eq 1 ]; then
  prog=$tmpscript
  chkinterval=2
  sublist="marty paul trevor tessa joan"
  maxprocs=3
  # make tmp script that takes some time to run
  echo '#! /bin/sh'                      > $tmpscript
  echo 'name=test'                      >> $tmpscript
  echo 'if [ $# -eq 1 ]; then'          >> $tmpscript
  echo '  name=$1'                      >> $tmpscript
  echo 'fi'                             >> $tmpscript
  echo 'i=1'                            >> $tmpscript
  echo 'while [ $i -le 8 ]; do'         >> $tmpscript
  echo '  sleep 2'                      >> $tmpscript
  echo '  echo "from pid $$ for $name"' >> $tmpscript
  echo '  i=`expr $i + 1`'              >> $tmpscript
  echo 'done'                           >> $tmpscript
  echo 'echo "$0 done"'                 >> $tmpscript
  chmod ugo+x $tmpscript
fi

### check env, sublist, infiles
if [ $debugflag = 0 ]; then
  if [ "`which $prog`" == "" ]; then
    echo 'runra: ### recon-all not on $path   ...quitting'
    exit
  fi
  if [ -z "$SUBJECTS_DIR" ]; then
    echo "runra: ### env variable SUBJECTS_DIR undefined (use setenv)"
    exit
  fi
  existing=""
  for sub in $sublist; do
    if [ -e $SUBJECTS_DIR/$sub ]; then
      echo "runra: ### $sub already exists in $SUBJECTS_DIR"
      existing="$existing $sub"
    fi
  done
  if [ "$existing" != "" ]; then
    echo "runra: ###  ...quitting"
    exit
  fi
  notfound=0
  for sub in $sublist; do
    if [ ! -e $rawdir/$sub/$rawscan ]; then
      echo "runra: ### infile not found:"
      echo "  $rawdir/$sub/$rawscan"
      echo ""
      notfound=1
    fi
  done
  if [ $notfound -eq 1 ]; then
    echo ""
    echo "runra: ### one or more infiles missing   ...quitting"
    echo ""
    exit 
  fi
fi

### wait confirm start
finishedflag=0
echo ""
echo "subjects to reconstruct:"
echo ""
for sub in $sublist; do
  echo "  $sub"
done
echo ""
echo " => ctrl-D to go"
echo " => ctrl-C to quit"
read resp
echo ""
echo "runra (sh): starting: `date`"
echo "runra (sh): starting: `date`" > $log

### start up maxprocs
set -- $sublist
subcnt="$#"
if [ $subcnt -lt $maxprocs ]; then maxprocs=$subcnt; fi
pids=""    # pids list (cnt=maxprocs)
subs=""    # curr running subs (names) in pids list order
subnum=1
i=1
while [ $i -le $maxprocs ]; do
  set -- $sublist
  eval sub="\$${subnum}"    # ugh: array member in pure sh
  if [ $debugflag -eq 1 ]; then
    $prog $sub &
    newpid=$!
  else
    ### run recon-all in bg
    $prog -subjid $sub -all -i $rawdir/$sub/$rawscan & 
#xxxxxxx
#   $prog $sub $hemi sphere.reg -tcl t1morph.tcl &
    newpid=$!
  fi
  pids="$pids $newpid"
  subs="$subs $sub"
  echo ""
  echo "###############################################################"
  echo \
    "runra: start $prog in bg for subj ${subnum}: $sub (pid=$newpid)"
  echo \
    "runra: start $prog in bg for subj ${subnum}: $sub (pid=$newpid)" >> $log
  echo "###############################################################"
  i=`expr $i + 1`
  subnum=`expr $subnum + 1`
done

### periodically check if any proc finishes, if subs left, start another
while [ 1 ]; do
  i=1
  while [ $i -le $maxprocs ]; do
    set -- $pids
    eval pid="\$${i}"
    psout=-1
    if [ $pid -ne -1 ]; then
      if [ `uname` = "Darwin" ]; then
        psout=`ps uxww | grep "$pid " | grep -v grep`
      elif [ `uname` = "Linux" ]; then
        psout=`ps uxww | grep "$pid " | grep -v grep`
      else
        psout=`ps af | grep "$pid " | grep -v grep`  # old sys5
      fi
    fi
    pidnamerunning=0
    case "$psout" in *${prog}*) pidnamerunning=1 ;; esac  # ugly portable match
    if [ $pidnamerunning = 0 ]; then
      set -- $pids
      eval pid="\$${i}"
      set -- $subs
      eval sub="\$${i}"
      if [ $pid -ne -1 ]; then
        echo ""
        echo "##################################################"
        echo "runra: subj: $sub (pid=$pid)   ### FINISHED ###"
        echo "runra: subj: $sub (pid=$pid)   ### FINISHED ###" >> $log
        echo "##################################################"
        echo ""
      fi
      writelistmemb pids $i -1
      writelistmemb subs $i FINISHED
      if [ $subnum -le $subcnt ]; then
        set -- $sublist
        eval sub="\$${subnum}"
        if [ $debugflag -eq 1 ]; then
          $prog $sub &
          newpid=$!
        else
          $prog -subjid $sub -all -i $rawdir/$sub/$rawscan & 
          newpid=$!
        fi
        writelistmemb pids $i $newpid
        writelistmemb subs $i $sub
        echo ""
        echo "###############################################################"
        echo \
          "runra: start $prog in bg for subj ${subnum}: $sub (pid=$newpid)"
        echo \
      "runra: start $prog in bg for subj ${subnum}: $sub (pid=$newpid)" >> $log
        echo "###############################################################"
        echo ""
        subnum=`expr $subnum + 1`
      fi
    fi
    i=`expr $i + 1`
  done
  # break if none left running
  running=0
  i=1
  while [ $i -le $maxprocs ]; do
    set -- $pids
    eval pid="\$${i}"
    if [ $pid -ne -1 ]; then
      running=`expr $running + 1`
    fi
    i=`expr $i + 1`
  done
  if [ $running = 0 ]; then break; fi
  sleep $chkinterval
done
finishedflag=1
cleanupfunc
