Use git sparse-checkout

Author: tien-tong

babs/bootstrap.py

@@ -410,7 +410,11 @@ def _bootstrap_single_app_scripts(
         print('This bash script will be named as `participant_job.sh`')
         bash_path = op.join(self.analysis_path, 'code', 'participant_job.sh')
         container.generate_bash_participant_job(
-            bash_path, self.input_datasets, self.processing_level, system
+            bash_path,
+            self.input_datasets,
+            self.processing_level,
+            system,
+            project_root=op.dirname(self.analysis_path),
         )
 
         # also, generate a bash script of a test job used by `babs check-setup`:
@@ -480,6 +484,7 @@ def _bootstrap_pipeline_scripts(self, container_ds, container_config, system):
             run_script_relpath='code/pipeline_zip.sh',
             container_images=container_images,
             datalad_run_message='pipeline',
+            project_root=op.dirname(self.analysis_path),
         )
 
         with open(bash_path, 'w') as f:

babs/container.py

@@ -145,7 +145,9 @@ def generate_bash_run_bidsapp(self, bash_path, input_ds, processing_level):
         print('Below is the generated BIDS App run script:')
         print(script_content)
 
-    def generate_bash_participant_job(self, bash_path, input_ds, processing_level, system):
+    def generate_bash_participant_job(
+        self, bash_path, input_ds, processing_level, system, project_root=None
+    ):
         """Generate bash script for participant job.
 
         Parameters
@@ -158,6 +160,9 @@ def generate_bash_participant_job(self, bash_path, input_ds, processing_level, s
             whether processing is done on a subject-wise or session-wise basis
         system: class `System`
             information on cluster management system
+        project_root : str, optional
+            Absolute path to the BABS project root (parent of `analysis/`).
+            Shown in the script error message when PROJECT_ROOT is unset.
         """
 
         script_content = generate_submit_script(
@@ -169,6 +174,7 @@ def generate_bash_participant_job(self, bash_path, input_ds, processing_level, s
             processing_level=processing_level,
             container_name=self.container_name,
             zip_foldernames=self.config['zip_foldernames'],
+            project_root=project_root,
         )
 
         with open(bash_path, 'w') as f:

babs/generate_submit_script.py

@@ -29,6 +29,7 @@ def generate_submit_script(
     run_script_relpath=None,
     container_images=None,
     datalad_run_message=None,
+    project_root=None,
 ):
     """
     Generate a bash script that runs the BIDS App singularity image.
@@ -57,6 +58,10 @@ def generate_submit_script(
         List of container image paths. None for single-app mode.
     datalad_run_message : str, optional
         Custom message for datalad run. None uses container name.
+    project_root : str, optional
+        Absolute path to the BABS project root (parent of `analysis/`).
+        Passed to the template; used in the error message when PROJECT_ROOT
+        is unset. If None, the placeholder ``{project_root}`` is shown.
 
     Returns
     -------
@@ -116,6 +121,7 @@ def generate_submit_script(
         run_script_relpath=run_script_relpath,
         container_images=container_images,
         datalad_run_message=datalad_run_message,
+        project_root=project_root,
     )
 
 

babs/templates/bidsapp_pipeline_run.sh.jinja2

@@ -15,12 +15,14 @@ sesid="$2"
 filterfile="${PWD}/${sesid}_filter.json"
 {% raw %}{{% endraw %}
     echo "{"
-    echo "'fmap': {'datatype': 'fmap'},"
+    echo "'fmap': {'datatype': 'fmap', 'session': '$sesid'},"
 {% set first_filter_step = steps_with_filter[0] %}
 {% if 'fmriprep' in first_filter_step['container_name'].lower() %}
     echo "'bold': {'datatype': 'func', 'session': '$sesid', 'suffix': 'bold'},"
 {% elif 'qsiprep' in first_filter_step['container_name'].lower() %}
     echo "'dwi': {'datatype': 'dwi', 'session': '$sesid', 'suffix': 'dwi'},"
+{% elif 'aslprep' in first_filter_step['container_name'].lower() %}
+    echo "'asl': {'datatype': 'perf', 'session': '$sesid', 'suffix': 'asl'},"
 {% endif %}
     echo "'sbref': {'datatype': 'func', 'session': '$sesid', 'suffix': 'sbref'},"
     echo "'flair': {'datatype': 'anat', 'session': '$sesid', 'suffix': 'FLAIR'},"

babs/templates/bidsapp_run.sh.jinja2

@@ -14,11 +14,13 @@ sesid="$2"
 filterfile="${PWD}/${sesid}_filter.json"
 {% raw %}{{% endraw %}
     echo "{"
-    echo "'fmap': {'datatype': 'fmap'},"
+    echo "'fmap': {'datatype': 'fmap', 'session': '$sesid'},"
 {% if 'fmriprep' in container_name.lower() %}
     echo "'bold': {'datatype': 'func', 'session': '$sesid', 'suffix': 'bold'},"
 {% elif 'qsiprep' in container_name.lower() %}
     echo "'dwi': {'datatype': 'dwi', 'session': '$sesid', 'suffix': 'dwi'},"
+{% elif 'aslprep' in container_name.lower() %}
+    echo "'asl': {'datatype': 'perf', 'session': '$sesid', 'suffix': 'asl'},"
 {% endif %}
     echo "'sbref': {'datatype': 'func', 'session': '$sesid', 'suffix': 'sbref'},"
     echo "'flair': {'datatype': 'anat', 'session': '$sesid', 'suffix': 'FLAIR'},"

babs/templates/determine_zipfilename.sh.jinja2

@@ -1,32 +1,31 @@
 # shellcheck disable=SC1091
 {% if has_a_zipped_input_dataset %}
-find_single_zipfile() {% raw %}{{% endraw %}
+find_single_zip_in_git_tree() {{ '{' }}
+  local zip_search_path="$1"
+  local name="$2"
+  local hits count
 
-    local path="$1"
-    local name="$2"
-    local pattern="${path}/${subid}{% if processing_level == 'session' %}_${sesid}{%endif%}_*${name}*.zip"
-    local zipfile
+  hits="$(
+    git -C "${zip_search_path}" ls-tree -r --name-only HEAD \
+      | grep -E "{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}{% raw %}.*${sesid}{% endraw %}{% endif %}{% raw %}.*${name}.*\.zip${% endraw %}" \
+      || true
+  )"
 
-    # Use find instead of ls for better handling of non-alphanumeric filenames
-    zipfile=$(find "$path" -maxdepth 1 -wholename "${pattern}" -print -quit || true)
-    if [ -z "$zipfile" ]; then
-        echo "ERROR: No zip file found matching pattern: ${pattern}" >&2
-        exit 1
-    fi
+  count="$(printf "%s\n" "${hits}" | sed '/^$/d' | wc -l | tr -d ' ')"
 
-    # Check if there's a second match
-    another_zipfile=$(find "$path" -maxdepth 1 -name "${pattern}" -print -quit -skip 1 || true)
-    if [ -n "${another_zipfile}" ]; then
-        echo "Multiple zip files found matching pattern: $pattern" >&2
-        exit 98
-    fi
+  if [ "${count}" -ne 1 ]; then
+{% raw %}    echo "ERROR: Expected exactly 1 matching ${name} zip in ${zip_search_path}, found ${count}" 1>&2
+    printf "%s\n" "${hits}" 1>&2
+    exit 1
+{% endraw %}
+  fi
 
-    echo "$zipfile"
-{% raw %}}{% endraw %}
+  printf "%s/%s\n" "${zip_search_path}" "${hits}"
+{{ '}' }}
 
 {% for input_dataset in input_datasets %}
 {% if input_dataset['is_zipped'] %}
-{{ input_dataset['name'].upper() }}_ZIP="$(find_single_zipfile {{ input_dataset['path_in_babs'] }} {{ input_dataset['name'] }})"
+{{ input_dataset['name'].upper() }}_ZIP="$(find_single_zip_in_git_tree {{ input_dataset['path_in_babs'] }} {{ input_dataset['name'] }})"
 echo 'found {{ input_dataset['name'] }} zipfile:'
 echo "${%raw%}{{%endraw%}{{ input_dataset['name'].upper() }}_ZIP{%raw%}}{%endraw%}"
 {% endif %}

babs/templates/participant_job.sh.jinja2

@@ -32,12 +32,24 @@ BRANCH="job-${%raw%}{{%endraw%}{{varname_jobid}}{%raw%}}{%endraw%}-${%raw%}{{%en
 BRANCH="job-${%raw%}{{%endraw%}{{varname_jobid}}{%raw%}}{%endraw%}-${%raw%}{{%endraw%}{{varname_taskid}}{%raw%}}{%endraw%}-${subid}"
 {% endif %}
 
+cleanup() {
+  set +e
+  if [ -d "{{ job_scratch_directory }}/{% raw %}${BRANCH}{% endraw %}/ds" ]; then
+    cd "{{ job_scratch_directory }}/{% raw %}${BRANCH}{% endraw %}/ds" 2>/dev/null || true
+    datalad drop -r . --reckless availability --reckless modification >/dev/null 2>&1 || true
+    git annex dead here >/dev/null 2>&1 || true
+  fi
+  cd "{{ job_scratch_directory }}" 2>/dev/null || true
+  rm -rf "{{ job_scratch_directory }}/{% raw %}${BRANCH}{% endraw %}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
 mkdir "${BRANCH}"
 cd "${BRANCH}"
 
 # datalad clone the input ria:
 echo '# Clone the data from input RIA:'
-datalad clone "${dssource}" ds
+datalad clone "${dssource}" ds -- --no-checkout
 cd ds
 
 # set up the result deposition:
@@ -48,32 +60,69 @@ git remote add outputstore "${pushgitremote}"
 echo "# Create a new branch for this job's results:"
 git checkout -b "${BRANCH}"
 
+# always use sparse-checkout, print error when not available
+if ! git sparse-checkout init --cone; then
+    echo "ERROR: git sparse-checkout is not available (or failed to initialize) on this system." 1>&2
+    exit 1
+fi
+
+git sparse-checkout set \
+  code \
+  containers \
+{% for input_dataset in input_datasets %}
+  {{ input_dataset['path_in_babs'] }}{% if not loop.last %} \
+{% endif %}
+{% endfor %}
+
+git checkout -f
+
 # Start of the application-specific code: ------------------------------
 
-# pull down input data (but don't retrieve the data content) and remove other sub's data:
-echo "# Pull down the input subject (or dataset) but don't retrieve data contents:"
+# pull down only needed session path and explicit dataset-level metadata:
+echo "# Pull down the input session but don't retrieve data contents:"
 {% for input_dataset in input_datasets %}
 {% if not input_dataset['is_zipped'] %}
-datalad get -n "{{ input_dataset['path_in_babs'] }}/${subid}"
-(cd {{ input_dataset['path_in_babs'] }} && find . -type d -name 'sub*' | grep -v "$subid" | xargs rm -rf)
-{% if processing_level == 'session' %}
-(cd {{ input_dataset['path_in_babs'] }}/"${subid}" && find . -type d -name 'ses*' | grep -v "$sesid" | xargs rm -rf)
-{% endif %}
+datalad get -n "{{ input_dataset['path_in_babs'] }}/{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}"
+
+datalad get -n \
+  "{{ input_dataset['path_in_babs'] }}/dataset_description.json" \
+  "{{ input_dataset['path_in_babs'] }}/participants.tsv" \
+  "{{ input_dataset['path_in_babs'] }}/participants.json" || true
 {% else %}
 datalad get -n "{{ input_dataset['path_in_babs'] }}"
-(cd {{ input_dataset['path_in_babs'] }} && find . -type f -name 'sub*.zip' | grep -v "$subid" | xargs rm -f)
 {% endif %}
 {% endfor %}
 
 {{ zip_locator_text }}
 
+# Link to shared container image so each job does not re-clone the same image
+PROJECT_ROOT="${PROJECT_ROOT:?ERROR: PROJECT_ROOT env var must be set to {{ project_root | default('{project_root}') }}}"
+CONTAINER_SHARED="${PROJECT_ROOT}/analysis/containers/.datalad/environments/{{ container_name }}/image"
+CONTAINER_JOB="containers/.datalad/environments/{{ container_name }}/image"
+
+if [ ! -e "${CONTAINER_SHARED}" ]; then
+  echo "ERROR: shared container image not found at ${CONTAINER_SHARED}" 1>&2
+  exit 1
+fi
+
+mkdir -p "containers/.datalad/environments/{{ container_name }}"
+# Replace any existing path (e.g. sparse-checkout placeholder or annex pointer) with symlink to shared image
+ln -sf "${CONTAINER_SHARED}" "${CONTAINER_JOB}"
+
+if [ ! -e "${CONTAINER_JOB}" ]; then
+  echo "ERROR: failed to create symlink ${CONTAINER_JOB}" 1>&2
+  exit 1
+fi
+
 # datalad run:
 datalad run \
 	-i "{{ run_script_relpath if run_script_relpath else 'code/' + container_name + '_zip.sh' }}" \
 {% for input_dataset in input_datasets %}
 {% if not input_dataset['is_zipped'] %}
-	-i "{{ input_dataset['unzipped_path_containing_subject_dirs'] }}/${subid}{% if processing_level == 'session' %}/${sesid}{% endif %}" \
-	-i "{{ input_dataset['unzipped_path_containing_subject_dirs'] }}/*json" \
+	-i "{{ input_dataset['unzipped_path_containing_subject_dirs'] }}/{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}/{% raw %}${sesid}{% endraw %}{% endif %}" \
+	-i "{{ input_dataset['path_in_babs'] }}/dataset_description.json" \
+	-i "{{ input_dataset['path_in_babs'] }}/participants.tsv" \
+	-i "{{ input_dataset['path_in_babs'] }}/participants.json" \
 {% else %}
 	-i "${%raw%}{{%endraw%}{{ input_dataset['name'].upper() }}_ZIP{%raw%}}{%endraw%}" \
 {% endif %}
@@ -82,7 +131,7 @@ datalad run \
 {% for image_path in container_images %}
 	-i "{{ image_path }}" \
 {% endfor %}
-{% elif not run_script_relpath %}
+{% else %}
 	-i "containers/.datalad/environments/{{container_name}}/image" \
 {% endif %}
 {% if datalad_expand_inputs %}
@@ -91,11 +140,11 @@ datalad run \
 	--explicit \
 {% if zip_foldernames is not none %}
 {% for key, value in zip_foldernames.items() %}
-	-o "${subid}{% if processing_level == 'session' %}_${sesid}{% endif %}_{{ key }}-{{ value }}.zip" \
+	-o "{% raw %}${subid}{% endraw %}{% if processing_level == 'session' %}_{% raw %}${sesid}{% endraw %}{% endif %}_{{ key }}-{{ value }}.zip" \
 {% endfor %}
 {% endif %}
-	-m "{{ datalad_run_message if datalad_run_message is defined else container_name }} ${subid}{% if processing_level == 'session' %} ${sesid}{% endif %}" \
-    "bash ./{{ run_script_relpath if run_script_relpath else 'code/' + container_name + '_zip.sh' }} ${subid} {% if processing_level == 'session' %} ${sesid}{% endif %}{% for input_dataset in input_datasets %}{% if input_dataset['is_zipped'] %} ${%raw%}{{%endraw%}{{ input_dataset['name'].upper() }}_ZIP{%raw%}}{%endraw%}{%endif%}{%endfor%}"
+	-m "{{ (datalad_run_message if datalad_run_message is defined and datalad_run_message else container_name) }} {% raw %}${subid}{% endraw %}{% if processing_level == 'session' %} {% raw %}${sesid}{% endraw %}{% endif %}" \
+    "bash ./{{ run_script_relpath if run_script_relpath else 'code/' + container_name + '_zip.sh' }} {% raw %}${subid}{% endraw %} {% if processing_level == 'session' %} {% raw %}${sesid}{% endraw %}{% endif %}{% for input_dataset in input_datasets %}{% if input_dataset['is_zipped'] %} ${%raw%}{{%endraw%}{{ input_dataset['name'].upper() }}_ZIP{%raw%}}{%endraw%}{%endif%}{%endfor%}"
 
 # Finish up:
 # push result file content to output RIA storage:
@@ -106,13 +155,4 @@ datalad push --to output-storage
 echo '# Push the branch with provenance records:'
 flock "${DSLOCKFILE}" git push outputstore
 
-# Delete:
-datalad drop -r . --reckless availability --reckless modification
-
-git annex dead here
-
-# cd out of $BRANCH:
-cd ../..
-rm -rf "${BRANCH}"
-
-echo SUCCESS
+echo SUCCESS