From 8f4469eecab0c67591da8e711f6433d178007a30 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 18 Jul 2019 14:00:08 -0400
Subject: [PATCH 01/31] Initial sketch for the mriqc/fmriprep singularity based
 workflow

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 147 +++++++++++++++++++++
 1 file changed, 147 insertions(+)
 create mode 100755 docs/usecases/bids-fmriprep-workflow-NP.sh

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
new file mode 100755
index 000000000..79ebfbcb2
--- /dev/null
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -0,0 +1,147 @@
+#!/bin/bash
+#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*- 
+#ex: set sts=4 ts=4 sw=4 noet:
+#
+#
+# COPYRIGHT: Yaroslav Halchenko 2019
+#
+# LICENSE: MIT
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+#
+
+set -eu
+
+# $STUDY is a variable used in a paper this workflow mimics
+STUDY="$1"
+
+# Create study dataset
+datalad create -c text2git "$STUDY"
+cd "$STUDY"
+
+#
+# Install containers dataset for guaranteed/unambigous containers versioning
+# and datalad containers-run
+#
+# TODO: specific version, TODO - reference datalad issue
+datalad install -d . ///repronim/containers
+
+# possibly downgrade versions to match the ones used in the "paper"
+# TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
+# and possibly providing some helper to accomplish that more easily
+cd containers
+echo -n "\
+poldracklab-ds003-example	0.0.3
+bids-mriqc					0.15.0
+bids-fmriprep				1.4.1
+"| while read img ver; do
+	git config -f .datalad/config --replace-all datalad.containers.$img.image images/${img%%-*}/${img}--${ver}.sing;
+done
+datalad save -d^ -m "Possibly downgraded containers versions to the ones in the paper" $PWD/.datalad/config
+cd ..
+
+#
+# Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)
+#
+# In original paper name for the dataset was used as is, and placed at the
+# top level.  Here, to make this demo easier to apply to other studies,
+# and also check on other datasets, we install input dataset under a generic
+# "data/bids" path.  "data/" will also collect all other derivatives etc
+mkdir data
+
+# For now we will work with minimized version with only 2 subjects
+# datalad install -d . -s ///openneuro/ds000003 data/bids
+datalad install -d . -s https://github.com/ReproNim/ds000003-demo data/bids
+
+
+#
+# Licenses
+#
+
+# we will not prepopulate this one
+mkdir licenses/
+echo freesurfer.txt > licenses/.gitignore
+
+cat > licenses/README.md <<EOF
+
+Freesurfer
+----------
+
+Place your FreeSurfer license into freesurfer.txt file in this directory.
+Visit https://surfer.nmr.mgh.harvard.edu/registration.html to obtain one if
+you don't have it yet - it is free.
+
+EOF
+datalad save -m "DOC: licenses/ directory stub" licenses/
+
+
+#
+# Execution.
+#
+# That is where access to the powerful resource (HPC) etc would be useful.
+# Every of those containerized apps might need custom options to be added.
+#
+#
+
+# Define common parameters for the reproman run
+
+# ReproMan orchestrator to be used - determines how data/results would be
+# transferred and execution protocoled
+# Use  reproman run --list orchestrators  to get an updated list
+RM_ORC=datalad-pair-run  # ,plain,datalad-pair,datalad-local-run
+
+# Which batch processing system supported by ReproMan will be used
+# Use  reproman run --list submitters  to get an updated list
+# RM_SUB=condor,pbs,local
+
+# Which resource to use
+# It would require (if was not done before) to configure
+# a resource where execution will happen.  For now will just use smaug below.
+# TODO: provide pointers to doc ( ;-) )
+# RM_RESOURCE=
+
+#RM_RESOURCE=discovery
+#RM_SUB=PBS
+#
+# Necessary modules to be loaded in that session:
+#  - singularity/2.4.2
+# Necessary installations/upgrades to be done (TODO: contact John)
+#  - datalad (0.11.6, TODO: release first)
+#  - datalad-container
+
+RM_RESOURCE=smaug
+RM_SUB=condor
+
+# TODO: at reproman level allow to specify ORC and SUB for a resource, so there would
+#   be no need to specify for each invocation.  Could be a new (meta) resource such as
+#   "smaug-condor" which would link smaug physical resource with those parameters
+# TODO: point to the issue in ReproMan
+
+# 1. bids-mriqc -- QA
+
+reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
+  --bp 'thing=thing-*' \
+  --input '{p[thing]}' \
+  sh -c 'cat {p[thing]} {p[thing]} >doubled-{p[thing]}'
+
+# 2. bids-fmriprep -- preprocessing
+
+# 3. poldracklab-ds003-example -- analysis
+
+# X. Later? visualization etc - used nilearn

From de0442f666acf6106ea978efc551ed26edf02445 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 18 Jul 2019 14:20:26 -0400
Subject: [PATCH 02/31] DOC: added a few comments

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 79ebfbcb2..0a75a3932 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -2,6 +2,10 @@
 #emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*- 
 #ex: set sts=4 ts=4 sw=4 noet:
 #
+#  This script is intended to demonstrate a sample workflow on a BIDS
+#  dataset using mriqc, fmriprep, and custom analysis pipeline, mimicing the
+#  steps presented in an fmriprep paper currently under review but using
+#  DataLad, ReproNim/containers, and ReproNim.
 #
 # COPYRIGHT: Yaroslav Halchenko 2019
 #
@@ -135,6 +139,8 @@ RM_SUB=condor
 
 # 1. bids-mriqc -- QA
 
+# Q/TODO: Is there a way to execute/reference the container?
+#   for now doing manually
 reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
   --bp 'thing=thing-*' \
   --input '{p[thing]}' \

From a18521d10ce085083a670372393bcd022cc5fbb6 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 18 Jul 2019 14:41:18 -0400
Subject: [PATCH 03/31] DOC: note on execution of mriqc

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 0a75a3932..f22014028 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -141,6 +141,18 @@ RM_SUB=condor
 
 # Q/TODO: Is there a way to execute/reference the container?
 #   for now doing manually
+datalad create -d . data/mriqc
+
+# Sample run without any parallelization, and doing both levels (participant and group)
+reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
+		 --jp container=containers/bids-mriqc data/bids data/mriqc participant,group
+
+# ultimately we should be able to parallelize across subjects. Here is the sample invocation for subj 02
+# singularity run containers/images/bids/bids-mriqc--0.15.0.sing  \
+#			data/bids/ data/mriqc/ participant --participant_label 02
+# and at the "group" level should have no --participant_label option
+
+
 reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
   --bp 'thing=thing-*' \
   --input '{p[thing]}' \

From 92388edac8035b0f6fcf4d85448f5f376941e97e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 6 Aug 2019 11:11:15 -0400
Subject: [PATCH 04/31] ENH: make possible to quickly switch from reproman to
 datalad + some docs on howto

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 44 ++++++++++++++++++++--
 1 file changed, 40 insertions(+), 4 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index f22014028..b47c5d440 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -29,6 +29,21 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 #
+#  Description
+#
+#  Environment variables
+#   - RUNNER - datalad or reproman
+#   - CONTAINERS_REPO - an alternative (could be local) location for containers
+#                       repository
+#   - INPUT_DATASET_REPO - an alternative (could be local) location for input
+#                       BIDS dataset
+#
+#  Sample invocations
+#   - Pointing to the existing local clones of input repositories for faster
+#     "get"
+#     RUNNER=datalad CONTAINERS_REPO=~/proj/repronim/containers \
+#       INPUT_DATASET_REPO=$PWD/bids-fmriprep-workflow-NP/ds000003-demo \
+#       ./bids-fmriprep-workflow-NP.sh bids-fmriprep-workflow-NP/out2
 
 set -eu
 
@@ -44,7 +59,10 @@ cd "$STUDY"
 # and datalad containers-run
 #
 # TODO: specific version, TODO - reference datalad issue
-datalad install -d . ///repronim/containers
+
+# Local copy to avoid heavy network traffic while testing locally could be
+# referenced in CONTAINERS_REPO env var
+datalad install -d . -s ${CONTAINERS_REPO:-///repronim/containers}
 
 # possibly downgrade versions to match the ones used in the "paper"
 # TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
@@ -71,7 +89,7 @@ mkdir data
 
 # For now we will work with minimized version with only 2 subjects
 # datalad install -d . -s ///openneuro/ds000003 data/bids
-datalad install -d . -s https://github.com/ReproNim/ds000003-demo data/bids
+datalad install -d . -s ${INPUT_DATASET_REPO:-https://github.com/ReproNim/ds000003-demo} data/bids
 
 
 #
@@ -143,9 +161,27 @@ RM_SUB=condor
 #   for now doing manually
 datalad create -d . data/mriqc
 
+: ${RUNNER:=reproman}
+
+unknown_runner () {
+    echo "ERROR: Unknown runner $RUNNER.  Known reproman and datalad" >&2
+    exit 1
+}
+
 # Sample run without any parallelization, and doing both levels (participant and group)
-reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
-		 --jp container=containers/bids-mriqc data/bids data/mriqc participant,group
+RUNNER_ARGS=( --input 'data/bids' --output data/mriqc )
+MRIQC_ARGS=( "{inputs}" "{outputs}" participant group )
+case "$RUNNER" in
+    reproman)
+        reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
+	    	 --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}";;
+	datalad)
+        datalad containers-run -n containers/bids-mriqc \
+            "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}";;
+    *) unknown_runner;;
+esac
+
+exit 0  # done for now
 
 # ultimately we should be able to parallelize across subjects. Here is the sample invocation for subj 02
 # singularity run containers/images/bids/bids-mriqc--0.15.0.sing  \

From 1588f7641dbb99c4a511d8df25cc03c6aef49d8d Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Tue, 6 Aug 2019 11:40:02 -0400
Subject: [PATCH 05/31] ENH: text2git for mriqc output

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index b47c5d440..04dbed4f8 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -159,7 +159,7 @@ RM_SUB=condor
 
 # Q/TODO: Is there a way to execute/reference the container?
 #   for now doing manually
-datalad create -d . data/mriqc
+datalad create -d . -c text2git data/mriqc
 
 : ${RUNNER:=reproman}
 

From 5b95ded7eadb19731063b86f27f1b829a04fec0a Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 15 Aug 2019 14:53:08 -0400
Subject: [PATCH 06/31] BF: make get_participants_ids work + set -x for
 debugging

Unfortunately initial run has failed with

	2019-08-15 14:32:13,311 [INFO   ] Waiting on job 1848: running
	2019-08-15 14:32:23,478 [INFO   ] Fetching results for 20190815-142046-33ea
	2019-08-15 14:35:51,720 [INFO   ] Creating run commit in /home/yoh/proj/repronim/reproman-master/docs/usecases/bids-fmriprep-workflow-NP/out7
	2019-08-15 14:36:06,509 [INFO   ] Unregistered job 20190815-142046-33ea
	+ reproman_run --jp container=containers/bids-mriqc --input data/bids --output data/mriqc "{inputs}" "{outputs}" group
	+ reproman run --follow -r smaug --sub condor --orc datalad-pair-run --jp container=containers/bids-mriqc --input data/bids --output data/mriqc "{inputs}" "{outputs}" group
	2019-08-15 14:36:10,588 [INFO   ] No root directory supplied for smaug; using "/home/yoh/.reproman/run-root"
	[INFO   ] Publishing <Dataset path=/home/yoh/proj/repronim/reproman-master/docs/usecases/bids-fmriprep-workflow-NP/out7/data/mriqc> to smaug
	ECDSA host key for IP address "129.170.233.9" not in list of known hosts.
	[INFO   ] Publishing <Dataset path=/home/yoh/proj/repronim/reproman-master/docs/usecases/bids-fmriprep-workflow-NP/out7> to smaug
	[ERROR  ] failed to push to smaug: master -> smaug/master [rejected] (non-fast-forward); pushed: ["d145d97..97de059"] [publish(/home/yoh/proj/repronim/reproman-master/docs/usecases/bids-fmriprep-workflow-NP/out7)]
	2019-08-15 14:36:59,238 [ERROR  ] "datalad publish" failed. Try running "datalad update -s smaug --merge --recursive" first [orchestrators.py:prepare_remote:792] (OrchestratorError)
	CONTAINERS_REPO=~/proj/repronim/containers INPUT_DATASET_REPO=    70.57s user 22.71s system 9% cpu 16:50.41 total

and stderr.1 on remote end showed that tar failed to find some output file:

    $> tail -n 3 stderr.1
    tar: ./work/workflow_enumerator/anatMRIQCT1w/ComputeIQMs/_in_file_..home..yoh...reproman..run-root..44671e06-bf85-11e9-95c1-8019340ce7f2..data..bids..sub-02..anat..sub-02_T1w.nii.gz/ComputeQI2/_0x9713a172faade86794f9c56a3080a44e_unfinished.json: Cannot stat: No such file or directory
    tar: ./work/workflow_enumerator/anatMRIQCT1w/ComputeIQMs/_in_file_..home..yoh...reproman..run-root..44671e06-bf85-11e9-95c1-8019340ce7f2..data..bids..sub-02..anat..sub-02_T1w.nii.gz/ComputeQI2/error.svg: Cannot stat: No such file or directory
    tar: Exiting with failure status due to previous errors
---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 52 +++++++++++++++++-----
 1 file changed, 42 insertions(+), 10 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 04dbed4f8..b7b7fed28 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
-#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*- 
-#ex: set sts=4 ts=4 sw=4 noet:
+#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- 
+#ex: set sts=4 ts=4 sw=4 et:
 #
 #  This script is intended to demonstrate a sample workflow on a BIDS
 #  dataset using mriqc, fmriprep, and custom analysis pipeline, mimicing the
@@ -46,6 +46,7 @@
 #       ./bids-fmriprep-workflow-NP.sh bids-fmriprep-workflow-NP/out2
 
 set -eu
+set -x
 
 # $STUDY is a variable used in a paper this workflow mimics
 STUDY="$1"
@@ -155,12 +156,6 @@ RM_SUB=condor
 #   "smaug-condor" which would link smaug physical resource with those parameters
 # TODO: point to the issue in ReproMan
 
-# 1. bids-mriqc -- QA
-
-# Q/TODO: Is there a way to execute/reference the container?
-#   for now doing manually
-datalad create -d . -c text2git data/mriqc
-
 : ${RUNNER:=reproman}
 
 unknown_runner () {
@@ -168,19 +163,56 @@ unknown_runner () {
     exit 1
 }
 
+# Common invocation of ReproMan
+# TODO: just make it configurable per project/env?
+reproman_run () {
+    reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" "$@"
+}
+
+get_participant_ids () {
+    # Would go through provided paths and current directory to find participants.tsv
+    # and return participant ids, comma-separated
+    for p in "$@" .; do
+        f="$p/participants.tsv"
+        if [ -e "$f" ]; then
+            sed -n -e '/^sub-/s/sub-\([^\t]*\)\t.*/\1/gp' < "$f" \
+                | tr '\n' ',' \
+                | sed -e 's/,$//g'
+            break
+        fi
+    done
+}
+
+# 1. bids-mriqc -- QA
+
+# Q/TODO: Is there a way to execute/reference the container?
+#   for now doing manually
+datalad create -d . -c text2git data/mriqc
+datalad save -d . -m "Due to https://github.com/datalad/datalad/issues/3591" data/mriqc
+
 # Sample run without any parallelization, and doing both levels (participant and group)
 RUNNER_ARGS=( --input 'data/bids' --output data/mriqc )
 MRIQC_ARGS=( "{inputs}" "{outputs}" participant group )
+
 case "$RUNNER" in
     reproman)
-        reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
-	    	 --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}";;
+        # Serial run
+        # reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}"
+        # Parallel requires two runs -- parallel across participants:
+        reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" \
+             --bp "pl=$(get_participant_ids data/bids)" \
+             '{inputs}' '{outputs}' participant --participant_label '{p[pl]}'
+        # serial for the group
+        reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" \
+            '{inputs}' '{outputs}' group
+	;;
 	datalad)
         datalad containers-run -n containers/bids-mriqc \
             "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}";;
     *) unknown_runner;;
 esac
 
+
 exit 0  # done for now
 
 # ultimately we should be able to parallelize across subjects. Here is the sample invocation for subj 02

From 79c3a99894065335314988356f66a44038b9a0ed Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 21 Aug 2019 15:09:23 -0400
Subject: [PATCH 07/31] another perspective one on kwyk

---
 docs/usecases/simple_kwyk.sh | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100755 docs/usecases/simple_kwyk.sh

diff --git a/docs/usecases/simple_kwyk.sh b/docs/usecases/simple_kwyk.sh
new file mode 100755
index 000000000..c62bd402b
--- /dev/null
+++ b/docs/usecases/simple_kwyk.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*- 
+#ex: set sts=4 ts=4 sw=4 noet:
+#
+#
+# COPYRIGHT: Yaroslav Halchenko 2019
+#
+# LICENSE: MIT
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+#
+
+set -eu
+
+cd $(mktemp -d --tmpdir=. ds-XXXX)
+pwd
+datalad create .
+datalad install -d . ///repronim/containers
+datalad install -d . -s https://github.com/ReproNim/ds000003-demo data/bids
+
+mkdir data/kwyked
+datalad containers-run \
+		--input data/bids/sub-02/anat/sub-02_T1w.nii.gz \
+		--output data/kwyked/sub-02_T1w \
+		-n containers/neuronets-kwyk \
+		'{inputs}' '{outputs}'

From 57b6b6c29cded74415c9948e2c21c87c845d89d2 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 10 Oct 2019 13:58:43 -0400
Subject: [PATCH 08/31] ENH/RF: shellcheck, group common bids-app logic into
 run_bids_app, extend how to deal with freesurfer license

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 175 +++++++++++++--------
 1 file changed, 108 insertions(+), 67 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index b7b7fed28..cb532f728 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -46,15 +46,46 @@
 #       ./bids-fmriprep-workflow-NP.sh bids-fmriprep-workflow-NP/out2
 
 set -eu
-set -x
+# set -x
 
 # $STUDY is a variable used in a paper this workflow mimics
 STUDY="$1"
 
+
+#
+# Check asap for licenses since fmriprep needs one for FreeSurfer
+#
+
+if [ -z "${FS_LICENSE:-}" ]; then
+    if [ -e "${FREESURFER_HOME:-/XXXX}/.license" ]; then
+        FS_LICENSE="${FREESURFER_HOME}/.license"
+    else
+        cat >&2 <<EOF
+Error: No FreeSurfer license found!
+    Either define FREESURFER_HOME environment variable pointing to a directory
+    with .license file for FreeSurfer or define FS_LICENSE environment variable
+    which would either point to the license file or contain the license
+    (with "\\n" for new lines) to be used for FreeSurfer
+EOF
+        exit 1
+    fi
+fi
+
 # Create study dataset
 datalad create -c text2git "$STUDY"
 cd "$STUDY"
 
+if [ -e "$FS_LICENSE" ]; then
+    cp "$FS_LICENSE" containers/licenses/freesurfer
+else
+    echo -n "$FS_LICENSE" >| containers/licenses/freesurfer
+fi
+echo "* annex.largefiles=(anything)" >| containers/licenses/.gitattributes
+rm containers/licences/.gitignore  # we will store them
+datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
+( cd containers; git annex metadata licenses/freesurfer -s distribution-restrictions=sensitive; )
+
+
 #
 # Install containers dataset for guaranteed/unambigous containers versioning
 # and datalad containers-run
@@ -63,21 +94,22 @@ cd "$STUDY"
 
 # Local copy to avoid heavy network traffic while testing locally could be
 # referenced in CONTAINERS_REPO env var
-datalad install -d . -s ${CONTAINERS_REPO:-///repronim/containers}
+datalad install -d . -s "${CONTAINERS_REPO:-///repronim/containers}"
 
 # possibly downgrade versions to match the ones used in the "paper"
 # TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
 # and possibly providing some helper to accomplish that more easily
-cd containers
-echo -n "\
-poldracklab-ds003-example	0.0.3
-bids-mriqc					0.15.0
-bids-fmriprep				1.4.1
-"| while read img ver; do
-	git config -f .datalad/config --replace-all datalad.containers.$img.image images/${img%%-*}/${img}--${ver}.sing;
-done
-datalad save -d^ -m "Possibly downgraded containers versions to the ones in the paper" $PWD/.datalad/config
-cd ..
+(
+    cd containers
+    echo -n "\
+    poldracklab-ds003-example	0.0.3
+    bids-mriqc					0.15.0
+    bids-fmriprep				1.4.1
+    "| while read -r img ver; do
+        git config -f .datalad/config --replace-all "datalad.containers.$img.image" "images/${img%%-*}/${img}--${ver}.sing";
+    done
+    datalad save -d^ -m "Possibly downgraded containers versions to the ones in the paper" "$PWD/.datalad/config"
+)
 
 #
 # Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)
@@ -90,29 +122,7 @@ mkdir data
 
 # For now we will work with minimized version with only 2 subjects
 # datalad install -d . -s ///openneuro/ds000003 data/bids
-datalad install -d . -s ${INPUT_DATASET_REPO:-https://github.com/ReproNim/ds000003-demo} data/bids
-
-
-#
-# Licenses
-#
-
-# we will not prepopulate this one
-mkdir licenses/
-echo freesurfer.txt > licenses/.gitignore
-
-cat > licenses/README.md <<EOF
-
-Freesurfer
-----------
-
-Place your FreeSurfer license into freesurfer.txt file in this directory.
-Visit https://surfer.nmr.mgh.harvard.edu/registration.html to obtain one if
-you don't have it yet - it is free.
-
-EOF
-datalad save -m "DOC: licenses/ directory stub" licenses/
-
+datalad install -d . -s "${INPUT_DATASET_REPO:-https://github.com/ReproNim/ds000003-demo}" data/bids
 
 #
 # Execution.
@@ -156,7 +166,7 @@ RM_SUB=condor
 #   "smaug-condor" which would link smaug physical resource with those parameters
 # TODO: point to the issue in ReproMan
 
-: ${RUNNER:=reproman}
+: "${RUNNER:=reproman}"
 
 unknown_runner () {
     echo "ERROR: Unknown runner $RUNNER.  Known reproman and datalad" >&2
@@ -184,41 +194,72 @@ get_participant_ids () {
 }
 
 # 1. bids-mriqc -- QA
+# datalad save -d . -m "Due to https://github.com/datalad/datalad/issues/3591" data/mriqc
+
+PARTICIPANT_LABELS="$(get_participant_ids data/bids)"
+
+function run_bids_app() {
+    app="$1"; shift
+    do_group="$1"; shift
+    app_args=( "$@" -w work )
+
+    outds=data/$app
+    container=containers/bids-$app
+    app_runner_args=( --input 'data/bids' --output "$outds" )
+
+    mkdir -p work
+    grep -e '^work$' .gitignore \
+    || { echo "work" >> .gitignore; datalad save -m "Ignore work directory"; }
+
+
+    # Create target output dataset
+    datalad create -d . -c text2git "$outds"
+
+    case "$RUNNER" in
+        reproman)
+            # Serial run
+            # reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}"
+            # Parallel requires two runs -- parallel across participants:
+            reproman_run --jp "container=$container" "${app_runner_args[@]}" \
+                 --bp "pl=$PARTICIPANT_LABELS" \
+                 '{inputs}' '{outputs}' participant --participant_label '{p[pl]}' "${app_args[@]}"
+            case "$do_group" in
+                1|yes)
+                    # serial for the group
+                    reproman_run --jp "container=$container" "${app_runner_args[@]}" \
+                        '{inputs}' '{outputs}' group "${app_args[@]}"
+                    ;;
+                0|no)
+                    ;;
+                *)
+                    echo "Unknown value APP_GROUP=$do_group" >&2
+                    exit 1
+                    ;;
+            esac
+        ;;
+        datalad)
+            case "$do_group" in
+                1|yes) app_args=( group "${app_args[@]}" ) ;;
+                0|no) ;;
+                *) exit 1 ;;
+            esac
+            datalad containers-run -n "$container" "${app_runner_args[@]}" \
+                '{inputs}' '{outputs}' participant "${app_args[@]}"
+            ;;
+        *) unknown_runner;;
+    esac
+}
 
-# Q/TODO: Is there a way to execute/reference the container?
-#   for now doing manually
-datalad create -d . -c text2git data/mriqc
-datalad save -d . -m "Due to https://github.com/datalad/datalad/issues/3591" data/mriqc
-
-# Sample run without any parallelization, and doing both levels (participant and group)
-RUNNER_ARGS=( --input 'data/bids' --output data/mriqc )
-MRIQC_ARGS=( "{inputs}" "{outputs}" participant group )
-
-case "$RUNNER" in
-    reproman)
-        # Serial run
-        # reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}"
-        # Parallel requires two runs -- parallel across participants:
-        reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" \
-             --bp "pl=$(get_participant_ids data/bids)" \
-             '{inputs}' '{outputs}' participant --participant_label '{p[pl]}'
-        # serial for the group
-        reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" \
-            '{inputs}' '{outputs}' group
-	;;
-	datalad)
-        datalad containers-run -n containers/bids-mriqc \
-            "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}";;
-    *) unknown_runner;;
-esac
+# mriqc
+app=mriqc
+do_group=yes
+run_bids_app mriqc yes
 
+fmriprep no --fs-license-file=licenses/freesurfer
+run_bids_app
 
-exit 0  # done for now
 
-# ultimately we should be able to parallelize across subjects. Here is the sample invocation for subj 02
-# singularity run containers/images/bids/bids-mriqc--0.15.0.sing  \
-#			data/bids/ data/mriqc/ participant --participant_label 02
-# and at the "group" level should have no --participant_label option
+exit 0  # done for now
 
 
 reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \

From fedb8b0d63f4ad711078f0509980eee7ec74ebc4 Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Thu, 12 Dec 2019 14:18:45 -0500
Subject: [PATCH 09/31] moved datalad install containers before working in
 containers subdir fixed typo in ".../licenses/..."

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index cb532f728..43ff97388 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -75,17 +75,6 @@ fi
 datalad create -c text2git "$STUDY"
 cd "$STUDY"
 
-if [ -e "$FS_LICENSE" ]; then
-    cp "$FS_LICENSE" containers/licenses/freesurfer
-else
-    echo -n "$FS_LICENSE" >| containers/licenses/freesurfer
-fi
-echo "* annex.largefiles=(anything)" >| containers/licenses/.gitattributes
-rm containers/licences/.gitignore  # we will store them
-datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
-( cd containers; git annex metadata licenses/freesurfer -s distribution-restrictions=sensitive; )
-
-
 #
 # Install containers dataset for guaranteed/unambigous containers versioning
 # and datalad containers-run
@@ -96,6 +85,17 @@ datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/lic
 # referenced in CONTAINERS_REPO env var
 datalad install -d . -s "${CONTAINERS_REPO:-///repronim/containers}"
 
+if [ -e "$FS_LICENSE" ]; then
+    cp "$FS_LICENSE" containers/licenses/freesurfer
+else
+    echo -n "$FS_LICENSE" >| containers/licenses/freesurfer
+fi
+echo "* annex.largefiles=(anything)" >| containers/licenses/.gitattributes
+rm containers/licenses/.gitignore  # we will store them
+datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
+( cd containers; git annex metadata licenses/freesurfer -s distribution-restrictions=sensitive; )
+
+
 # possibly downgrade versions to match the ones used in the "paper"
 # TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
 # and possibly providing some helper to accomplish that more easily

From 76450afa0a8971926bbc3d75937a1b9cd3a74567 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 19 Dec 2019 09:40:31 -0500
Subject: [PATCH 10/31] RF: compose proper call for fmriprep, inline querying
 participant labels

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 43ff97388..5be6a1bef 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -41,7 +41,9 @@
 #  Sample invocations
 #   - Pointing to the existing local clones of input repositories for faster
 #     "get"
-#     RUNNER=datalad CONTAINERS_REPO=~/proj/repronim/containers \
+#     RUNNER=datalad \
+#       FS_LICENSE=~/.freesurfer-license \
+#       CONTAINERS_REPO=~/proj/repronim/containers \
 #       INPUT_DATASET_REPO=$PWD/bids-fmriprep-workflow-NP/ds000003-demo \
 #       ./bids-fmriprep-workflow-NP.sh bids-fmriprep-workflow-NP/out2
 
@@ -196,8 +198,6 @@ get_participant_ids () {
 # 1. bids-mriqc -- QA
 # datalad save -d . -m "Due to https://github.com/datalad/datalad/issues/3591" data/mriqc
 
-PARTICIPANT_LABELS="$(get_participant_ids data/bids)"
-
 function run_bids_app() {
     app="$1"; shift
     do_group="$1"; shift
@@ -221,7 +221,7 @@ function run_bids_app() {
             # reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}"
             # Parallel requires two runs -- parallel across participants:
             reproman_run --jp "container=$container" "${app_runner_args[@]}" \
-                 --bp "pl=$PARTICIPANT_LABELS" \
+                 --bp "pl=$(get_participant_ids data/bids)" \
                  '{inputs}' '{outputs}' participant --participant_label '{p[pl]}' "${app_args[@]}"
             case "$do_group" in
                 1|yes)
@@ -250,13 +250,13 @@ function run_bids_app() {
     esac
 }
 
-# mriqc
-app=mriqc
-do_group=yes
 run_bids_app mriqc yes
 
-fmriprep no --fs-license-file=licenses/freesurfer
-run_bids_app
+run_bids_app fmriprep no --fs-license-file=licenses/freesurfer
+
+# 3. poldracklab-ds003-example -- analysis
+
+# X. Later? visualization etc - used nilearn
 
 
 exit 0  # done for now
@@ -267,8 +267,4 @@ reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" \
   --input '{p[thing]}' \
   sh -c 'cat {p[thing]} {p[thing]} >doubled-{p[thing]}'
 
-# 2. bids-fmriprep -- preprocessing
-
-# 3. poldracklab-ds003-example -- analysis
 
-# X. Later? visualization etc - used nilearn

From 850b36f6b8cc3b2b5780214b0bdbcee2f1e4460c Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 19 Dec 2019 11:38:07 -0500
Subject: [PATCH 11/31] RF: reordered commands so settings come first and then
 all the actions

also uses new freeze_version --save-dataset helper from containers etc
---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 119 ++++++++++-----------
 1 file changed, 57 insertions(+), 62 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 5be6a1bef..a27cfb4b6 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -73,67 +73,6 @@ EOF
     fi
 fi
 
-# Create study dataset
-datalad create -c text2git "$STUDY"
-cd "$STUDY"
-
-#
-# Install containers dataset for guaranteed/unambigous containers versioning
-# and datalad containers-run
-#
-# TODO: specific version, TODO - reference datalad issue
-
-# Local copy to avoid heavy network traffic while testing locally could be
-# referenced in CONTAINERS_REPO env var
-datalad install -d . -s "${CONTAINERS_REPO:-///repronim/containers}"
-
-if [ -e "$FS_LICENSE" ]; then
-    cp "$FS_LICENSE" containers/licenses/freesurfer
-else
-    echo -n "$FS_LICENSE" >| containers/licenses/freesurfer
-fi
-echo "* annex.largefiles=(anything)" >| containers/licenses/.gitattributes
-rm containers/licenses/.gitignore  # we will store them
-datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
-( cd containers; git annex metadata licenses/freesurfer -s distribution-restrictions=sensitive; )
-
-
-# possibly downgrade versions to match the ones used in the "paper"
-# TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
-# and possibly providing some helper to accomplish that more easily
-(
-    cd containers
-    echo -n "\
-    poldracklab-ds003-example	0.0.3
-    bids-mriqc					0.15.0
-    bids-fmriprep				1.4.1
-    "| while read -r img ver; do
-        git config -f .datalad/config --replace-all "datalad.containers.$img.image" "images/${img%%-*}/${img}--${ver}.sing";
-    done
-    datalad save -d^ -m "Possibly downgraded containers versions to the ones in the paper" "$PWD/.datalad/config"
-)
-
-#
-# Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)
-#
-# In original paper name for the dataset was used as is, and placed at the
-# top level.  Here, to make this demo easier to apply to other studies,
-# and also check on other datasets, we install input dataset under a generic
-# "data/bids" path.  "data/" will also collect all other derivatives etc
-mkdir data
-
-# For now we will work with minimized version with only 2 subjects
-# datalad install -d . -s ///openneuro/ds000003 data/bids
-datalad install -d . -s "${INPUT_DATASET_REPO:-https://github.com/ReproNim/ds000003-demo}" data/bids
-
-#
-# Execution.
-#
-# That is where access to the powerful resource (HPC) etc would be useful.
-# Every of those containerized apps might need custom options to be added.
-#
-#
-
 # Define common parameters for the reproman run
 
 # ReproMan orchestrator to be used - determines how data/results would be
@@ -211,7 +150,7 @@ function run_bids_app() {
     grep -e '^work$' .gitignore \
     || { echo "work" >> .gitignore; datalad save -m "Ignore work directory"; }
 
-
+    set -x
     # Create target output dataset
     datalad create -d . -c text2git "$outds"
 
@@ -248,8 +187,64 @@ function run_bids_app() {
             ;;
         *) unknown_runner;;
     esac
+    set +x
 }
 
+# Create study dataset
+datalad create -c text2git "$STUDY"
+cd "$STUDY"
+
+#
+# Install containers dataset for guaranteed/unambigous containers versioning
+# and datalad containers-run
+#
+# TODO: specific version, TODO - reference datalad issue
+
+# Local copy to avoid heavy network traffic while testing locally could be
+# referenced in CONTAINERS_REPO env var
+datalad install -d . -s "${CONTAINERS_REPO:-///repronim/containers}"
+
+if [ -e "$FS_LICENSE" ]; then
+    cp "$FS_LICENSE" containers/licenses/freesurfer
+else
+    echo -n "$FS_LICENSE" >| containers/licenses/freesurfer
+fi
+datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
+( cd containers; git annex metadata licenses/freesurfer -s distribution-restrictions=sensitive; )
+
+
+# possibly downgrade versions to match the ones used in the "paper"
+# TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
+# and possibly providing some helper to accomplish that more easily
+(
+    cd containers
+    scripts/freeze_versions --save-dataset=^ \
+        poldracklab-ds003-example=0.0.3 \
+        bids-mriqc=0.15.0 \
+        bids-fmriprep=1.4.1
+)
+
+#
+# Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)
+#
+# In original paper name for the dataset was used as is, and placed at the
+# top level.  Here, to make this demo easier to apply to other studies,
+# and also check on other datasets, we install input dataset under a generic
+# "data/bids" path.  "data/" will also collect all other derivatives etc
+mkdir data
+
+# For now we will work with minimized version with only 2 subjects
+# datalad install -d . -s ///openneuro/ds000003 data/bids
+datalad install -d . -s "${INPUT_DATASET_REPO:-https://github.com/ReproNim/ds000003-demo}" data/bids
+
+#
+# Execution.
+#
+# That is where access to the powerful resource (HPC) etc would be useful.
+# Every of those containerized apps might need custom options to be added.
+#
+#
+
 run_bids_app mriqc yes
 
 run_bids_app fmriprep no --fs-license-file=licenses/freesurfer

From 7110ec6d6df1c3f2a197bfda84478cf59736fb4f Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 19 Dec 2019 11:39:29 -0500
Subject: [PATCH 12/31] BF+RF: improve handling of fs license, more TODO
 comments (seems to work now!)

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 76 +++++++++++-----------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index a27cfb4b6..71c6091a7 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -53,26 +53,6 @@ set -eu
 # $STUDY is a variable used in a paper this workflow mimics
 STUDY="$1"
 
-
-#
-# Check asap for licenses since fmriprep needs one for FreeSurfer
-#
-
-if [ -z "${FS_LICENSE:-}" ]; then
-    if [ -e "${FREESURFER_HOME:-/XXXX}/.license" ]; then
-        FS_LICENSE="${FREESURFER_HOME}/.license"
-    else
-        cat >&2 <<EOF
-Error: No FreeSurfer license found!
-    Either define FREESURFER_HOME environment variable pointing to a directory
-    with .license file for FreeSurfer or define FS_LICENSE environment variable
-    which would either point to the license file or contain the license
-    (with "\\n" for new lines) to be used for FreeSurfer
-EOF
-        exit 1
-    fi
-fi
-
 # Define common parameters for the reproman run
 
 # ReproMan orchestrator to be used - determines how data/results would be
@@ -120,6 +100,9 @@ reproman_run () {
     reproman run --follow -r "${RM_RESOURCE}" --sub "${RM_SUB}" --orc "${RM_ORC}" "$@"
 }
 
+
+# TODO: see where such functionality could be provided within reproman, so could
+# be easily reused
 get_participant_ids () {
     # Would go through provided paths and current directory to find participants.tsv
     # and return participant ids, comma-separated
@@ -134,9 +117,6 @@ get_participant_ids () {
     done
 }
 
-# 1. bids-mriqc -- QA
-# datalad save -d . -m "Due to https://github.com/datalad/datalad/issues/3591" data/mriqc
-
 function run_bids_app() {
     app="$1"; shift
     do_group="$1"; shift
@@ -152,6 +132,8 @@ function run_bids_app() {
 
     set -x
     # Create target output dataset
+    # TODO: per app specific configuration?  some might have too heavy xml etc
+    # files
     datalad create -d . -c text2git "$outds"
 
     case "$RUNNER" in
@@ -190,6 +172,26 @@ function run_bids_app() {
     set +x
 }
 
+#
+# Check asap for licenses since fmriprep needs one for FreeSurfer
+#
+
+if [ -z "${FS_LICENSE:-}" ]; then
+    if [ -e "${FREESURFER_HOME:-/XXXX}/.license" ]; then
+        FS_LICENSE="${FREESURFER_HOME}/.license"
+    else
+        cat >&2 <<EOF
+Error: No FreeSurfer license found!
+    Either define FREESURFER_HOME environment variable pointing to a directory
+    with .license file for FreeSurfer or define FS_LICENSE environment variable
+    which would either point to the license file or contain the license
+    (with "\\n" for new lines) to be used for FreeSurfer
+EOF
+        exit 1
+    fi
+fi
+
+
 # Create study dataset
 datalad create -c text2git "$STUDY"
 cd "$STUDY"
@@ -204,25 +206,22 @@ cd "$STUDY"
 # referenced in CONTAINERS_REPO env var
 datalad install -d . -s "${CONTAINERS_REPO:-///repronim/containers}"
 
+# TODO: shift that into some helper script in the containers
+CONTAINERS_FS_LICENSE=containers/licenses/freesurfer
 if [ -e "$FS_LICENSE" ]; then
-    cp "$FS_LICENSE" containers/licenses/freesurfer
+    cp "$FS_LICENSE" "$CONTAINERS_FS_LICENSE"
 else
-    echo -n "$FS_LICENSE" >| containers/licenses/freesurfer
+    echo -n "$FS_LICENSE" >| "$CONTAINERS_FS_LICENSE"
 fi
-datalad save -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
+datalad save -d . -m "Added licenses/freesurfer (needed for fmriprep)" containers/licenses/
 ( cd containers; git annex metadata licenses/freesurfer -s distribution-restrictions=sensitive; )
 
 
 # possibly downgrade versions to match the ones used in the "paper"
-# TODO see  https://github.com/ReproNim/containers/issues/8 for relevant discussion
-# and possibly providing some helper to accomplish that more easily
-(
-    cd containers
-    scripts/freeze_versions --save-dataset=^ \
-        poldracklab-ds003-example=0.0.3 \
-        bids-mriqc=0.15.0 \
-        bids-fmriprep=1.4.1
-)
+containers/scripts/freeze_versions --save-dataset=^ \
+    poldracklab-ds003-example=0.0.3 \
+    bids-mriqc=0.15.0 \
+    bids-fmriprep=1.4.1
 
 #
 # Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)
@@ -245,9 +244,12 @@ datalad install -d . -s "${INPUT_DATASET_REPO:-https://github.com/ReproNim/ds000
 #
 #
 
-run_bids_app mriqc yes
+# datalad save -d . -m "Due to https://github.com/datalad/datalad/issues/3591" data/mriqc
 
-run_bids_app fmriprep no --fs-license-file=licenses/freesurfer
+
+run_bids_app mriqc yes
+# note: not using $CONTAINERS_FS_LICENSE just to make things a bit more explicit
+run_bids_app fmriprep no --fs-license-file=containers/licenses/freesurfer
 
 # 3. poldracklab-ds003-example -- analysis
 

From a81e457f0a3bb242619035ff76d836126a1bd779 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 19 Dec 2019 21:10:52 -0500
Subject: [PATCH 13/31] ENH: do not create bids app results dataset if
 directory exists already

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 71c6091a7..822df5d9a 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -134,7 +134,7 @@ function run_bids_app() {
     # Create target output dataset
     # TODO: per app specific configuration?  some might have too heavy xml etc
     # files
-    datalad create -d . -c text2git "$outds"
+    [ -e "$outds" ] || datalad create -d . -c text2git "$outds"
 
     case "$RUNNER" in
         reproman)

From d5e90283e010a24bb29c10d5cf24a454054f62d2 Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Fri, 3 Jan 2020 15:57:28 -0500
Subject: [PATCH 14/31] defaulting RM_RESOURCE and RM_SUB to local but allowing
 overrides

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 822df5d9a..177dc1fe8 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -79,8 +79,8 @@ RM_ORC=datalad-pair-run  # ,plain,datalad-pair,datalad-local-run
 #  - datalad (0.11.6, TODO: release first)
 #  - datalad-container
 
-RM_RESOURCE=smaug
-RM_SUB=condor
+: ${RM_RESOURCE:=local}
+: ${RM_SUB:=local}
 
 # TODO: at reproman level allow to specify ORC and SUB for a resource, so there would
 #   be no need to specify for each invocation.  Could be a new (meta) resource such as

From 5ab5a8b5c46b7d363aafc3164795e5b2f09c207e Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Tue, 14 Jan 2020 16:56:53 -0500
Subject: [PATCH 15/31] clean the containers repo after freezing versions

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 177dc1fe8..48feafb09 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -222,6 +222,7 @@ containers/scripts/freeze_versions --save-dataset=^ \
     poldracklab-ds003-example=0.0.3 \
     bids-mriqc=0.15.0 \
     bids-fmriprep=1.4.1
+( cd containers/ ; git clean -dfx ; )
 
 #
 # Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)

From 7ae46ab700bb8864c7213381d337deb561680298 Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Tue, 28 Jan 2020 15:27:02 -0500
Subject: [PATCH 16/31] Revert "clean the containers repo after freezing
 versions"

This reverts commit 5ab5a8b5c46b7d363aafc3164795e5b2f09c207e.
---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 48feafb09..177dc1fe8 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -222,7 +222,6 @@ containers/scripts/freeze_versions --save-dataset=^ \
     poldracklab-ds003-example=0.0.3 \
     bids-mriqc=0.15.0 \
     bids-fmriprep=1.4.1
-( cd containers/ ; git clean -dfx ; )
 
 #
 # Install dataset to be analyzed (no data - analysis might run in the cloud or on HPC)

From 5a2f0f31b1e32e6aaa5e29aa70cf6555bc701bdd Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Wed, 29 Apr 2020 10:23:32 -0400
Subject: [PATCH 17/31] Fix runscript regexp to work on Mac OS

---
 reproman/support/jobs/job_templates/runscript/base.template.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reproman/support/jobs/job_templates/runscript/base.template.sh b/reproman/support/jobs/job_templates/runscript/base.template.sh
index cb1cecf34..646bbdb7a 100755
--- a/reproman/support/jobs/job_templates/runscript/base.template.sh
+++ b/reproman/support/jobs/job_templates/runscript/base.template.sh
@@ -53,7 +53,7 @@ then
 # least for now, below is a brittle solution were the last job waits until it
 # sees that all other jobs have exited and then runs the post-command stuff.
 nstatus () {
-    find "$metadir" -regex '.*/status\.[0-9]+' | wc -l
+    find "$metadir" -regex '.*/status\.[0-9][0-9]*' | wc -l
 }
 
 # Ugly, but this sleep makes it less likely for the post-command tar to fail

From b70144e993660c271831e4ea8d2f4bb436bb7eeb Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 21 May 2020 13:55:15 -0400
Subject: [PATCH 18/31] ENH: always set -x, add env vars to not require
 patching for FS license etc

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 43 ++++++++++++++++------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 177dc1fe8..fbe75b2b3 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -32,12 +32,25 @@
 #  Description
 #
 #  Environment variables
-#   - RUNNER - datalad or reproman
+#   - RUNNER - datalad or reproman (default: reproman)
+#   - Options to  reproman run  invocation
+#     - RM_ORC - orchestrator to use (default: datalad-pair-run)
+#     - RM_RESOURCE - resource to use (default: local)
+#     - RM_SUBMITTED - submitter to use (default: local)
+#   - BIDS_APPS - if set -- ,-separated list of apps to consider (out of
+#     mriqc and fmriprep ATM)
+#   - FS_LICENSE - filename or content of the license for freesurfer
 #   - CONTAINERS_REPO - an alternative (could be local) location for containers
 #                       repository
 #   - INPUT_DATASET_REPO - an alternative (could be local) location for input
 #                       BIDS dataset
 #
+#  Note that if FS_LICENSE does not point to a file and is not empty, it would
+#  assume to contain the license content.  If you are not interested in running
+#  only MRIQC, just set it to some bogus value.
+#  So to run only mriqc if you don't have freesurfer license, do
+#  BIDS_APPS=mriqc FS_LICENSE=bogus ...
+#
 #  Sample invocations
 #   - Pointing to the existing local clones of input repositories for faster
 #     "get"
@@ -46,19 +59,24 @@
 #       CONTAINERS_REPO=~/proj/repronim/containers \
 #       INPUT_DATASET_REPO=$PWD/bids-fmriprep-workflow-NP/ds000003-demo \
 #       ./bids-fmriprep-workflow-NP.sh bids-fmriprep-workflow-NP/out2
+#
 
 set -eu
-# set -x
+export PS4='ex:$? > '
+set -x
 
 # $STUDY is a variable used in a paper this workflow mimics
 STUDY="$1"
 
+#  Which runner - reproman or datalad
+: "${RUNNER:=reproman}"
+
 # Define common parameters for the reproman run
 
 # ReproMan orchestrator to be used - determines how data/results would be
 # transferred and execution protocoled
 # Use  reproman run --list orchestrators  to get an updated list
-RM_ORC=datalad-pair-run  # ,plain,datalad-pair,datalad-local-run
+: "${RM_ORC:=datalad-pair-run}"  # ,plain,datalad-pair,datalad-local-run
 
 # Which batch processing system supported by ReproMan will be used
 # Use  reproman run --list submitters  to get an updated list
@@ -68,26 +86,22 @@ RM_ORC=datalad-pair-run  # ,plain,datalad-pair,datalad-local-run
 # It would require (if was not done before) to configure
 # a resource where execution will happen.  For now will just use smaug below.
 # TODO: provide pointers to doc ( ;-) )
-# RM_RESOURCE=
 
-#RM_RESOURCE=discovery
-#RM_SUB=PBS
-#
+# On discovery resource use PBS, and
 # Necessary modules to be loaded in that session:
 #  - singularity/2.4.2
 # Necessary installations/upgrades to be done (TODO: contact John)
 #  - datalad (0.11.6, TODO: release first)
 #  - datalad-container
 
-: ${RM_RESOURCE:=local}
-: ${RM_SUB:=local}
+: "${RM_RESOURCE:=local}"
+: "${RM_SUB:=local}"
 
 # TODO: at reproman level allow to specify ORC and SUB for a resource, so there would
 #   be no need to specify for each invocation.  Could be a new (meta) resource such as
 #   "smaug-condor" which would link smaug physical resource with those parameters
 # TODO: point to the issue in ReproMan
 
-: "${RUNNER:=reproman}"
 
 unknown_runner () {
     echo "ERROR: Unknown runner $RUNNER.  Known reproman and datalad" >&2
@@ -118,10 +132,15 @@ get_participant_ids () {
 }
 
 function run_bids_app() {
+    set -eu
     app="$1"; shift
     do_group="$1"; shift
     app_args=( "$@" -w work )
 
+    if [ -n "$BIDS_APPS" ] && ! echo "$BIDS_APPS" | grep -q "\<$app\>" ; then
+      echo "I: skipping $app since BIDS_APPS=$BIDS_APPS"
+      return
+    fi
     outds=data/$app
     container=containers/bids-$app
     app_runner_args=( --input 'data/bids' --output "$outds" )
@@ -130,7 +149,7 @@ function run_bids_app() {
     grep -e '^work$' .gitignore \
     || { echo "work" >> .gitignore; datalad save -m "Ignore work directory"; }
 
-    set -x
+    # set -x
     # Create target output dataset
     # TODO: per app specific configuration?  some might have too heavy xml etc
     # files
@@ -169,7 +188,7 @@ function run_bids_app() {
             ;;
         *) unknown_runner;;
     esac
-    set +x
+    # set +x
 }
 
 #

From 82df2489d56ad453b9a12c1c879afc8b64d995b4 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 22 May 2020 17:24:02 -0400
Subject: [PATCH 19/31] Comments on which images must be prepropulated in
 containers repo and remove not needed set -eu in function

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index fbe75b2b3..779086198 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -41,7 +41,10 @@
 #     mriqc and fmriprep ATM)
 #   - FS_LICENSE - filename or content of the license for freesurfer
 #   - CONTAINERS_REPO - an alternative (could be local) location for containers
-#                       repository
+#                       repository.
+#     Make sure that you have got the images for specific versions we freeze to below:
+#       datalad get images/bids/bids-mriqc--0.15.0.sing images/bids/bids-fmriprep--1.4.1.sing
+#
 #   - INPUT_DATASET_REPO - an alternative (could be local) location for input
 #                       BIDS dataset
 #
@@ -132,7 +135,6 @@ get_participant_ids () {
 }
 
 function run_bids_app() {
-    set -eu
     app="$1"; shift
     do_group="$1"; shift
     app_args=( "$@" -w work )

From 6009dd071fc007207aa613487466487c1fd5de8b Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 22 May 2020 17:23:11 -0400
Subject: [PATCH 20/31] ENH: Script for reproducible rerun of the demo script

---
 .../bids-fmriprep-workflow-NP-reproduce.sh    | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100755 docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh

diff --git a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
new file mode 100755
index 000000000..443410a3c
--- /dev/null
+++ b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -eu
+
+export PS4="> "
+
+set -x
+setup=${1:-pip}
+cd "$(mktemp -d ${TMPDIR:-/tmp}/rm-XXXXXXX)"
+
+trap "echo Finished for setup=$setup under PWD=`pwd`" SIGINT SIGHUP SIGABRT EXIT
+
+py=3
+d=venv$py;
+(
+virtualenv --python=python$py --system-site-packages $d
+) 2>&1 | tee venv-setup.log
+
+source "$d/bin/activate"  # should be outside of () to take effect
+
+(
+case "$setup" in
+ kyle1)
+   # Kyle's setup from https://github.com/ReproNim/reproman/issues/511#issuecomment-632776223
+  pip install git+http://github.com/datalad/datalad@53765be03838ee8b07d4b44a2a27bbbe259fe160
+  # This one seems to be for older datalad
+  pip install git+http://github.com/ReproNim/reproman@a9c9842302cad707bbdaf56fa4050fe0136ffe23
+  # with unbuffered io:
+  #pip install git+http://github.com/ReproNim/reproman@4f05f3aa96c7ab550aa218d5de705ea3cfe5f600
+  ;;
+ debug1) # the "default
+  # Current master of datalad
+  pip install git+http://github.com/datalad/datalad@0.13.0rc1-109-g7f24491b2
+  # ReproMan PR https://github.com/ReproNim/reproman/pull/506 with support of datalad master
+  pip install git+http://github.com/kyleam/niceman@v0.2.1-80-g45baab0
+  ;;
+ pip)  # should be our target -- install via pip everything and it must be working
+   pip install datalad reproman;;
+ *)
+   echo "Unknown setup $setup" >&2
+   exit 1
+   ;;
+esac
+
+# in either of the cases default datalad-container should be ok
+pip install datalad-container
+
+# Actual script to run from the current state of the PR
+# https://github.com/ReproNim/reproman/pull/438
+wget https://raw.githubusercontent.com/ReproNim/reproman/b70144e993660c271831e4ea8d2f4bb436bb7eeb/docs/usecases/bids-fmriprep-workflow-NP.sh
+) 2>&1 | tee install.log
+
+(
+    BIDS_APPS=mriqc FS_LICENSE=bogus RM_ORC=datalad-pair bash ./bids-fmriprep-workflow-NP.sh output
+) 2>&1 | tee run.log
\ No newline at end of file

From 295d9d23548394d3c70bfc63fca6d75378099727 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Mon, 25 May 2020 14:15:28 -0400
Subject: [PATCH 21/31] kyle1-ps4 setup

---
 docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
index 443410a3c..22b58ab5c 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
@@ -28,6 +28,11 @@ case "$setup" in
   # with unbuffered io:
   #pip install git+http://github.com/ReproNim/reproman@4f05f3aa96c7ab550aa218d5de705ea3cfe5f600
   ;;
+ kyle1-ps4)
+  # Like above but for reproman have #513 merged for PS4 details
+  pip install git+http://github.com/datalad/datalad@53765be03838ee8b07d4b44a2a27bbbe259fe160
+  pip install git+http://github.com/ReproNim/reproman@setup-kyle1-ps4
+  ;;
  debug1) # the "default
   # Current master of datalad
   pip install git+http://github.com/datalad/datalad@0.13.0rc1-109-g7f24491b2

From 8905cc4cb32308594b8033e8be4c1a992cf05fd0 Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Tue, 26 May 2020 18:56:36 +0000
Subject: [PATCH 22/31] BF: Fix failure on unset BIDS_APP with set -u

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 779086198..bfbc411bf 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -139,7 +139,7 @@ function run_bids_app() {
     do_group="$1"; shift
     app_args=( "$@" -w work )
 
-    if [ -n "$BIDS_APPS" ] && ! echo "$BIDS_APPS" | grep -q "\<$app\>" ; then
+    if [ -n "${BIDS_APPS:=}" ] && ! echo "$BIDS_APPS" | grep -q "\<$app\>" ; then
       echo "I: skipping $app since BIDS_APPS=$BIDS_APPS"
       return
     fi

From cffa3a3f6002a26d561ad7b9ea977aa89a483f40 Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Wed, 27 May 2020 10:27:36 -0400
Subject: [PATCH 23/31] BF: Add mac workaround in get_participant_ids

Mac sed doesn't handle tabs as linux does, so we start by converting the
TSV file to CSV, then process from there.
---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index bfbc411bf..6cb879a25 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -126,7 +126,8 @@ get_participant_ids () {
     for p in "$@" .; do
         f="$p/participants.tsv"
         if [ -e "$f" ]; then
-            sed -n -e '/^sub-/s/sub-\([^\t]*\)\t.*/\1/gp' < "$f" \
+            awk -F'\t' '/^sub-/{print $1}' "$f" \
+                | sed 's/sub-//' \
                 | tr '\n' ',' \
                 | sed -e 's/,$//g'
             break

From 85a41dc2ec83799a00b4395b9d6f641376dd30e6 Mon Sep 17 00:00:00 2001
From: Christian Haselgrove <christian.haselgrove@umassmed.edu>
Date: Wed, 27 May 2020 10:29:52 -0400
Subject: [PATCH 24/31] ENH: Update parallel install message for mac users

---
 reproman/support/jobs/job_templates/submission/local.template | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/reproman/support/jobs/job_templates/submission/local.template b/reproman/support/jobs/job_templates/submission/local.template
index 029567103..f389978fe 100755
--- a/reproman/support/jobs/job_templates/submission/local.template
+++ b/reproman/support/jobs/job_templates/submission/local.template
@@ -10,7 +10,8 @@ then
 else
     if test -z $(which parallel)
     then
-       echo "parallel (moreutils) is required to concurrent jobs locally" >&2
+       echo "parallel is required to concurrent jobs locally" >&2
+       echo "install parallel on Debian from moreutils or on macOS using MacPorts or Homebrew" >&2
        exit 1
     fi
 

From baab9917b581ccaf623ea17c3299e3b23f83788e Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 27 May 2020 15:47:47 -0400
Subject: [PATCH 25/31] BF: export PS1 within -reproduce.sh

For some reason otherwise it immediately fails on smaug if I do not do that

...
    Installing setuptools, pkg_resources, pip, wheel...done.
    > source venv3/bin/activate
    >> deactivate nondestructive
    >> unset -f pydoc
    >> '[' -z '' ']'
    >> '[' -z '' ']'
    >> '[' -n /bin/bash ']'
    >> hash -r
    >> '[' -z '' ']'
    >> unset VIRTUAL_ENV
    >> '[' '!' nondestructive = nondestructive ']'
    >> VIRTUAL_ENV=/home/yoh/.tmp/rm-TIAZarc/venv3
    >> export VIRTUAL_ENV
    >> _OLD_VIRTUAL_PATH=/home/yoh/bin:/home/yoh/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/sbin:/usr/sbin:/
    usr/local/sbin
    >> PATH=/home/yoh/.tmp/rm-TIAZarc/venv3/bin:/home/yoh/bin:/home/yoh/.local/bin:/usr/local/bin:/usr/bin:/bin:/usr/games:/sbin:/usr/sbin:/usr/local/sbin
    >> export PATH
    >> '[' -z '' ']'
    >> '[' -z '' ']'
    venv3/bin/activate: line 57: PS1: unbound variable
    >>> echo Finished for setup=pip under PWD=/home/yoh/.tmp/rm-TIAZarc
    Finished for setup=pip under PWD=/home/yoh/.tmp/rm-TIAZarc
---
 docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
index 22b58ab5c..b1bc1112e 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
@@ -2,6 +2,7 @@
 
 set -eu
 
+export PS1="$ "
 export PS4="> "
 
 set -x
@@ -57,4 +58,4 @@ wget https://raw.githubusercontent.com/ReproNim/reproman/b70144e993660c271831e4e
 
 (
     BIDS_APPS=mriqc FS_LICENSE=bogus RM_ORC=datalad-pair bash ./bids-fmriprep-workflow-NP.sh output
-) 2>&1 | tee run.log
\ No newline at end of file
+) 2>&1 | tee run.log

From 75fa815426c6c3ec1ea927860f78e9affc9b3f72 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 27 May 2020 15:56:18 -0400
Subject: [PATCH 26/31] ENH: Use temporary HOME, cp .gitconfig and
 .freesurfer-license, configure local resource

---
 docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
index b1bc1112e..229d38ce3 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
@@ -9,6 +9,12 @@ set -x
 setup=${1:-pip}
 cd "$(mktemp -d ${TMPDIR:-/tmp}/rm-XXXXXXX)"
 
+mkdir HOME
+cp ~/.gitconfig HOME/  # needed by datalad et al
+cp ~/.freesurfer-license HOME/ 2>&1 || echo "No FreeSurfer license copied"
+
+export HOME=$PWD/HOME
+
 trap "echo Finished for setup=$setup under PWD=`pwd`" SIGINT SIGHUP SIGABRT EXIT
 
 py=3
@@ -54,8 +60,14 @@ pip install datalad-container
 # Actual script to run from the current state of the PR
 # https://github.com/ReproNim/reproman/pull/438
 wget https://raw.githubusercontent.com/ReproNim/reproman/b70144e993660c271831e4ea8d2f4bb436bb7eeb/docs/usecases/bids-fmriprep-workflow-NP.sh
+
+# Ensure that we have local resource for default execution
 ) 2>&1 | tee install.log
 
+(
+    reproman create -t shell local
+) 2>&1 | tee configure.log
+
 (
     BIDS_APPS=mriqc FS_LICENSE=bogus RM_ORC=datalad-pair bash ./bids-fmriprep-workflow-NP.sh output
 ) 2>&1 | tee run.log

From 7920234e8a9e78f03f6bf7b93f553a04b1e1acf5 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 27 May 2020 16:05:38 -0400
Subject: [PATCH 27/31] reproman-master setup for -reproduce and min datalad
 0.12.7

locally had 0.12.6 and it might be interfering
---
 docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
index 229d38ce3..ac0facb4b 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
@@ -46,8 +46,13 @@ case "$setup" in
   # ReproMan PR https://github.com/ReproNim/reproman/pull/506 with support of datalad master
   pip install git+http://github.com/kyleam/niceman@v0.2.1-80-g45baab0
   ;;
+ reproman-master)  # master of reproman with default datalad (min 0.12.7)
+   # 0.12.7 just to ensure we are not using outdated system wide
+   pip install 'datalad>=0.12.7'
+   pip install git+http://github.com/ReproNim/reproman@master
+   ;;
  pip)  # should be our target -- install via pip everything and it must be working
-   pip install datalad reproman;;
+   pip install 'datalad>=0.12.7' reproman;;
  *)
    echo "Unknown setup $setup" >&2
    exit 1

From 29500a789539e2a636e80b6656311558465c6a6b Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Wed, 27 May 2020 22:41:18 -0400
Subject: [PATCH 28/31] master reproman now has [datalad] installation target

---
 docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
index ac0facb4b..b205bb583 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP-reproduce.sh
@@ -46,12 +46,11 @@ case "$setup" in
   # ReproMan PR https://github.com/ReproNim/reproman/pull/506 with support of datalad master
   pip install git+http://github.com/kyleam/niceman@v0.2.1-80-g45baab0
   ;;
- reproman-master)  # master of reproman with default datalad (min 0.12.7)
-   # 0.12.7 just to ensure we are not using outdated system wide
-   pip install 'datalad>=0.12.7'
-   pip install git+http://github.com/ReproNim/reproman@master
+ reproman-master)
+   pip install 'git+http://github.com/ReproNim/reproman.git#egg=reproman[datalad]'
    ;;
  pip)  # should be our target -- install via pip everything and it must be working
+   # until we release reproman with [datalad] - do manually
    pip install 'datalad>=0.12.7' reproman;;
  *)
    echo "Unknown setup $setup" >&2

From d895cc41b82cb4273cfb9839d214db0209fa0bd8 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 28 May 2020 11:29:21 -0400
Subject: [PATCH 29/31] ENH: add containers/licenses into --input, specify
 data/bids explicitly (not as {inputs})

That should allow to upload all needed licenses to the submitting host
---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index bfbc411bf..5981e1684 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -145,7 +145,7 @@ function run_bids_app() {
     fi
     outds=data/$app
     container=containers/bids-$app
-    app_runner_args=( --input 'data/bids' --output "$outds" )
+    app_runner_args=( --input containers/licenses --input 'data/bids' --output "$outds" )
 
     mkdir -p work
     grep -e '^work$' .gitignore \
@@ -164,7 +164,7 @@ function run_bids_app() {
             # Parallel requires two runs -- parallel across participants:
             reproman_run --jp "container=$container" "${app_runner_args[@]}" \
                  --bp "pl=$(get_participant_ids data/bids)" \
-                 '{inputs}' '{outputs}' participant --participant_label '{p[pl]}' "${app_args[@]}"
+                 data/bids '{outputs}' participant --participant_label '{p[pl]}' "${app_args[@]}"
             case "$do_group" in
                 1|yes)
                     # serial for the group

From e28d0117013652e003665d0eeeb83b8c4a094d04 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 28 May 2020 11:31:14 -0400
Subject: [PATCH 30/31] DOC: note that datalad runner group analysis probably
 does nothing

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index 5981e1684..c0a13d00a 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -180,6 +180,7 @@ function run_bids_app() {
             esac
         ;;
         datalad)
+            # Note: this is not in effect!  TODO
             case "$do_group" in
                 1|yes) app_args=( group "${app_args[@]}" ) ;;
                 0|no) ;;

From 795eed875cd2eab8b5b7b29b624bdae1e82062d1 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Thu, 28 May 2020 11:33:17 -0400
Subject: [PATCH 31/31] ENH: point to subject specific input data

---
 docs/usecases/bids-fmriprep-workflow-NP.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/usecases/bids-fmriprep-workflow-NP.sh b/docs/usecases/bids-fmriprep-workflow-NP.sh
index c0a13d00a..f901af2cc 100755
--- a/docs/usecases/bids-fmriprep-workflow-NP.sh
+++ b/docs/usecases/bids-fmriprep-workflow-NP.sh
@@ -145,7 +145,7 @@ function run_bids_app() {
     fi
     outds=data/$app
     container=containers/bids-$app
-    app_runner_args=( --input containers/licenses --input 'data/bids' --output "$outds" )
+    app_runner_args=( --input containers/licenses --output "$outds" )
 
     mkdir -p work
     grep -e '^work$' .gitignore \
@@ -163,12 +163,14 @@ function run_bids_app() {
             # reproman_run --jp container=containers/bids-mriqc "${RUNNER_ARGS[@]}" "${MRIQC_ARGS[@]}"
             # Parallel requires two runs -- parallel across participants:
             reproman_run --jp "container=$container" "${app_runner_args[@]}" \
+                 --input "data/bids/sub-{p[pl]}" \
                  --bp "pl=$(get_participant_ids data/bids)" \
                  data/bids '{outputs}' participant --participant_label '{p[pl]}' "${app_args[@]}"
             case "$do_group" in
                 1|yes)
                     # serial for the group
                     reproman_run --jp "container=$container" "${app_runner_args[@]}" \
+                        --input "data/bids" \
                         '{inputs}' '{outputs}' group "${app_args[@]}"
                     ;;
                 0|no)