#! /bin/csh -f

### runra: marty sereno -- run n recon-all's in bg, rotate new in when done
# 14/01/09 -- 01a: initial hack
# 14/01/10 -- 01b: cleanup, comment, bold start/stop msgs, ps uxww on Linux
# 14/10/20 -- 01c: save list of curr running subj names, use when one finishes

############################################################################
# 4 user-editable parms (don't change the variable names: e.g., sublist)
############################################################################
# list of subjects to reconstruct (recon-all will create these directories)
set sublist = ( \
 AMIELB ANNASH ANNSNI ARADAV AURRUS BENOHA CHAHAR CHALOC CHYHIL DAVRAU \
 ETHTIL FELPIC FLOHOC GEOSAN GRADEN HANMOR ISADES JACADD JOKHER JONTIL \
 JOSDAV LOLYOU NELCAM PHIKEL PHIPEL REBBRO RHITOM RHYHOL SALKRI VICKNO )

# dir outside SUBJECTS_DIR w/rawscan subject dirs in it w/same names as above
set rawdir = /media/Data/Inverse/TessaData/MRI_RECON

# name of rawscan NIFTI files (currently req'd to be same for all)
set rawscan = MPRAGE1.nii

# maximum number of background processes to run
set maxprocs = 6
############################################################################

### help
set log = ./runra.$$.log
set helpflag = 1
set debugflag = 0
if ($#argv == 1) then
  set arg = $argv[1]
  if ("$arg" == "-test" || "$arg" == "-debug") then
    set helpflag = 0
    set debugflag = 1
  endif
  if ("$arg" == "-go") then
    set helpflag = 0
  endif
endif
if ($helpflag) then
  echo ""
  echo "use: runra [-go,-test]"
  echo ""
  echo "  --cp script and edit 4 variables at top of it"
  echo '  --requires MGH freesurfer env setup (recon-all, others on $path)'
  echo '  --start/finish of each subj written to ./runra.<pid>.log'
  echo "  --option -test makes/runs small csh sleep script for demo"
  echo ""
  echo "  Example raw scan directories:"
  echo "    /tmp/rawdata/marty/MPRAGE1.nii"
  echo "    /tmp/rawdata/trevor/MPRAGE1.nii"
  echo "    /tmp/rawdata/tessa/MPRAGE1.nii"
  echo "  Resulting subject dirs:"
  echo '    $SUBJECTS_DIR/marty'
  echo '    $SUBJECTS_DIR/trevor'
  echo '    $SUBJECTS_DIR/tessa'
  echo ""
  echo "  Current state of editable parms:"
  echo ""
  echo "   (1) sublist = $sublist"
  echo "   (2) rawdir = $rawdir"
  echo "   (3) rawscan = $rawscan"
  echo "   (4) maxprocs = $maxprocs"
  echo "                                                       version: 01c"
  exit
endif

### internal
onintr cleanup       # goto cleanup on ctrl-C interrrupt
set chkinterval = 15
set prog = recon-all
set tmpscript = /tmp/TmpCshRunra.$$

### debug
if ($debugflag) then
  set prog = $tmpscript
  set chkinterval = 2
  set sublist = ( marty paul trevor tessa joan )
  set maxprocs = 3
  # make tmp script that takes some time to run
  echo '#! /bin/csh -f'                  > $tmpscript
  echo 'set name = test'                >> $tmpscript
  echo 'if ($#argv == 1) then'          >> $tmpscript
  echo '  set name = $argv[1]'          >> $tmpscript
  echo 'endif'                          >> $tmpscript
  echo 'set i = 1'                      >> $tmpscript
  echo 'while ($i < 8)'                 >> $tmpscript
  echo '  sleep 2'                      >> $tmpscript
  echo '  echo "from pid $$ for $name"' >> $tmpscript
  echo '  @ i++'                        >> $tmpscript
  echo 'end'                            >> $tmpscript
  echo 'echo "$0 done"'                 >> $tmpscript
  chmod ugo+x $tmpscript
endif

### check env, sublist, infiles
if (! $debugflag) then
  if ("`which $prog`" == "${prog}: Command not found.") then
    echo 'runra: ### recon-all not on $path   ...quitting'
    exit
  endif
  if (! $?SUBJECTS_DIR) then
    echo "runra: ### env variable SUBJECTS_DIR undefined (use setenv)"
    exit
  endif
  set existing = ""
  foreach sub ($sublist)
    if (-e $SUBJECTS_DIR/$sub) then
      echo "runra: ### $sub already exists in $SUBJECTS_DIR"
      set existing = "$existing $sub"
    endif  
  end
  if ("$existing" != "") then
    echo "runra: ###  ...quitting"
    exit
  endif
  set notfound = 0
  foreach sub ($sublist)
    if (! -e $rawdir/$sub/$rawscan) then
      echo "runra: ### infile not found:"
      echo "  $rawdir/$sub/$rawscan"
      echo ""
      set notfound = 1
    endif
  end
  if ($notfound) then
    echo ""
    echo "runra: ### one or more infiles missing   ...quitting"
    echo ""
    exit 
  endif
endif

### wait confirm start
set finishedflag = 0
echo ""
echo "subjects to reconstruct:"
echo ""
foreach sub ($sublist)
  echo "  $sub"
end
echo ""
echo " => ctrl-D to go"
echo " => ctrl-C to quit"
set resp = "$<"
echo ""
echo "runra (csh): starting: `date`"
echo "runra (csh): starting: `date`" > $log

### start up maxprocs
set subcnt = $#sublist
if ($subcnt < $maxprocs) set maxprocs = $subcnt
set i = 1
set pids = ()   # pids list (cnt=maxprocs)
set subs = ()   # curr running subs (names) in pids list order
while ($i <= $maxprocs)
  set pids = ($pids -1)
  set subs = ($subs FINISHED)
  @ i++
end
set subnum = 1
set i = 1
while ($i <= $maxprocs)
  set sub = $sublist[$subnum]
  if ($debugflag) then
    $prog $sub &
    set pids[$i] = $!
    set subs[$i] = $sub
  else
    ### run recon-all in bg
    $prog -subjid $sub -all -i $rawdir/$sub/$rawscan & 
    set pids[$i] = $!
    set subs[$i] = $sub
  endif
  echo ""
  echo "###############################################################"
  echo \
    "runra: start $prog in bg for subj ${subnum}: $sub (pid=$pids[$i])"
  echo \
    "runra: start $prog in bg for subj ${subnum}: $sub (pid=$pids[$i])" >> $log
  echo "###############################################################"
  @ i++
  @ subnum++
end

### periodically check if any proc finishes, if subs left, start another
while (1)
  set i = 1
  while ($i <= $maxprocs)
    set pid = $pids[$i]
    set psout = -1
    if ($pid != -1) then
      if (`uname` == "Darwin") then
        set psout = `ps uxww |& grep "$pid " |& grep -v grep`
      else if (`uname` == "Linux" ) then
        set psout = `ps uxww |& grep "$pid " |& grep -v grep`
      else
        set psout = `ps af |& grep "$pid " |& grep -v grep`  # old sys5
      endif
    endif
    set pidnamerunning = 0
    if ("$psout" =~ "*${prog}*") set pidnamerunning = 1
    if (! $pidnamerunning) then
      if ("$pids[$i]" != -1) then
        echo ""
        echo "##################################################"
        echo "runra: subj: $subs[$i] (pid=$pids[$i])   ### FINISHED ###"
        echo "runra: subj: $subs[$i] (pid=$pids[$i])   ### FINISHED ###" >> $log
        echo "##################################################"
        echo ""
      endif
      set pids[$i] = -1
      if ($subnum <= $subcnt) then
        set sub = $sublist[$subnum]
        if ($debugflag) then
          $prog $sub &
          set pids[$i] = $!
        else
          ### run recon-all in bg
          $prog -subjid $sub -all -i $rawdir/$sub/$rawscan & 
          set pids[$i] = $!
        endif
        echo ""
        echo "###############################################################"
        echo \
          "runra: start $prog in bg for subj ${subnum}: $sub (pid=$pids[$i])"
        echo \
    "runra: start $prog in bg for subj ${subnum}: $sub (pid=$pids[$i])" >> $log
        echo "###############################################################"
        echo ""
        @ subnum++
      endif
    endif
    @ i++
  end
  # break if none left running
  set running = 0
  set i = 1
  while ($i <= $maxprocs)
    if ("$pids[$i]" != "-1") then
      @ running++
    endif
    @ i++
  end
  if ($running == 0) break
  sleep $chkinterval
end
set finishedflag = 1

### manually kill detached running bg procs
cleanup:
if ($debugflag) rm -f $tmpscript
if ($?pids) then
  set i = 1
  while ($i <= $maxprocs)
    set pid = $pids[$i]
    if ("$pid" != "-1") then
      kill -9 $pid
      echo ""
      echo "###############################################################"
      echo "runra: killed pid: $pid"
      echo "runra: killed pid: $pid" >> $log
      echo "###############################################################"
      echo ""
    endif
    @ i++
  end
endif
if ($finishedflag) then
  echo ""
  echo "###############################################################"
  echo "runra: done w/sublist: $sublist"
  echo "runra: done w/sublist: $sublist" >> $log
  echo "runra: finished: `date`"
  echo "runra: finished: `date`" >> $log
  echo "###############################################################"
  echo ""
endif
