Use datalad containers-run for single-container jobs

.circleci/config.yml

@@ -18,26 +18,23 @@ jobs:
           name: pytest of BABS
           no_output_timeout: 1h
           command: |
-            docker build \
-            -t pennlinc/slurm-docker-ci:unstable \
-            -f Dockerfile_testing .
-
-            # Make a directory that will hold the test artifacts
             mkdir -p ${HOME}/e2e-testing
             docker run -it \
-                -v ${PWD}:/tests \
+                -v ${PWD}:/babs \
                 -v ${HOME}/e2e-testing:/test-temp:rw \
+                -w /babs \
                 -h slurmctl --cap-add sys_admin \
                 --privileged \
-                pennlinc/slurm-docker-ci:unstable \
+                pennlinc/slurm-docker-ci:0.14 \
+                bash -c "pip install -e .[tests] && \
                 pytest -n 4 -sv \
                     --durations=0 \
                     --timeout=300 \
                     --junitxml=/test-temp/junit.xml \
                     --cov-report term-missing \
                     --cov-report xml:/test-temp/coverage.xml \
                     --cov=babs \
-                    /babs
+                    /babs/tests/"
 
       - store_test_results:
           path: /home/circleci/e2e-testing/junit.xml
@@ -54,22 +51,10 @@ jobs:
       - checkout:
           path: /home/circleci/src/babs
       - run:
-          name: pytest of BABS
+          name: e2e SLURM tests
           no_output_timeout: 1h
           command: |
-            docker build \
-            -t pennlinc/slurm-docker-ci:unstable \
-            -f Dockerfile_testing .
-
-            # Make a directory that will hold the test artifacts
-            mkdir -p ${HOME}/e2e-testing
-            docker run -it \
-                -v ${PWD}:/tests \
-                -v ${HOME}/e2e-testing:/test-temp:rw \
-                -h slurmctl --cap-add sys_admin \
-                --privileged \
-                pennlinc/slurm-docker-ci:unstable \
-                /tests/tests/e2e-slurm/container/walkthrough-tests.sh
+            E2E_DIR=${HOME}/e2e-testing bash tests/e2e_in_docker.sh
 
       - run:
           name: clean up test artifacts

babs/bootstrap.py

@@ -3,6 +3,7 @@
 import os
 import os.path as op
 import subprocess
+import tempfile
 from pathlib import Path
 
 import datalad.api as dlapi
@@ -77,8 +78,11 @@ def babs_bootstrap(
                 f"The parent folder '{parent_dir}' does not exist! `babs init` won't proceed."
             )
 
-        # check if parent directory is writable:
-        if not os.access(parent_dir, os.W_OK):
+        # check if parent directory is writable (os.access unreliable on NFS/ACL):
+        try:
+            with tempfile.TemporaryFile(dir=parent_dir):
+                pass
+        except OSError:
             raise ValueError(
                 f"The parent folder '{parent_dir}' is not writable! `babs init` won't proceed."
             )
@@ -223,10 +227,51 @@ def babs_bootstrap(
         )
         # into `analysis/containers` folder
 
-        # Create initial container for sanity check
-        container = Container(container_ds, container_name, container_config)
+        # Discover container image path from the container dataset:
+        containers_path = op.join(self.analysis_path, 'containers')
+        result = dlapi.containers_list(dataset=containers_path, result_renderer='disabled')
+        container_info = [
+            r for r in result if r['action'] == 'containers' and r['name'] == container_name
+        ]
+        if not container_info:
+            available = [r['name'] for r in result if r['action'] == 'containers']
+            raise ValueError(
+                f"Container '{container_name}' not found in container dataset. "
+                f'Available: {available}'
+            )
+        image_path_in_ds = op.relpath(container_info[0]['path'], containers_path)
+        container_image_path = op.join('containers', image_path_in_ds)
+
+        # Build call_fmt from user's singularity_args:
+        with open(container_config) as f:
+            user_config = yaml.safe_load(f)
+        singularity_args = user_config.get('singularity_args', [])
+        singularity_args_str = ' '.join(singularity_args) if singularity_args else ''
+        call_fmt = f'singularity run -B $PWD --pwd $PWD {singularity_args_str} {{img}} {{cmd}}'
 
-        # sanity check of container ds:
+        # Register container at analysis level so datalad containers-run works:
+        print(f'\nRegistering container at analysis level: {container_image_path}')
+        dlapi.containers_add(
+            dataset=self.analysis_path,
+            name=container_name,
+            image=container_image_path,
+            call_fmt=call_fmt,
+        )
+
+        # Fetch container image so the PROJECT_ROOT symlink at job time
+        # resolves to actual content, not a dangling annex pointer:
+        print('\nFetching container image...')
+        dlapi.get(
+            dataset=self.analysis_path,
+            path=op.join(self.analysis_path, container_image_path),
+        )
+
+        container = Container(
+            container_ds,
+            container_name,
+            container_config,
+            container_image_path=container_image_path,
+        )
         container.sanity_check(self.analysis_path)
 
         # ==============================================================
@@ -250,9 +295,18 @@ def babs_bootstrap(
             container = containers[0]
         else:
             self._bootstrap_single_app_scripts(
-                container_ds, container_name, container_config, system
+                container_ds,
+                container_name,
+                container_config,
+                system,
+                container_image_path=container_image_path,
+            )
+            container = Container(
+                container_ds,
+                container_name,
+                container_config,
+                container_image_path=container_image_path,
             )
-            container = Container(container_ds, container_name, container_config)
 
         # Copy in any other files needed:
         self._init_import_files(container.config.get('imported_files', []))
@@ -390,21 +444,30 @@ def babs_bootstrap(
         print('`babs init` was successful!')
 
     def _bootstrap_single_app_scripts(
-        self, container_ds, container_name, container_config, system
+        self,
+        container_ds,
+        container_name,
+        container_config,
+        system,
+        container_image_path=None,
     ):
         """Bootstrap scripts for single BIDS app configuration."""
-        container = Container(container_ds, container_name, container_config)
+        container = Container(
+            container_ds,
+            container_name,
+            container_config,
+            container_image_path=container_image_path,
+        )
 
         # Generate `<containerName>_zip.sh`: ----------------------------------
-        # which is a bash script of singularity run + zip
-        # in folder: `analysis/code`
-        print('\nGenerating a bash script for running container and zipping the outputs...')
-        print('This bash script will be named as `' + container_name + '_zip.sh`')
+        # Zip-only script (container execution is now handled by containers-run
+        # in participant_job.sh)
+        print('\nGenerating zip script: ' + container_name + '_zip.sh')
         bash_path = op.join(self.analysis_path, 'code', container_name + '_zip.sh')
-        container.generate_bash_run_bidsapp(bash_path, self.input_datasets, self.processing_level)
+        container.generate_bash_zip_outputs(bash_path, self.processing_level)
         self.datalad_save(
             path='code/' + container_name + '_zip.sh',
-            message='Generate script of running container',
+            message='Generate zip script',
         )
 
         # make another folder within `code` for test jobs:

babs/container.py

@@ -6,15 +6,19 @@
 import yaml
 from jinja2 import Environment, PackageLoader, StrictUndefined
 
-from babs.generate_bidsapp_runscript import generate_bidsapp_runscript
+from babs.generate_bidsapp_runscript import (
+    bids_app_args_from_config,
+    generate_bidsapp_runscript,
+    get_output_zipping_cmds,
+)
 from babs.generate_submit_script import generate_submit_script, generate_test_submit_script
 from babs.utils import app_output_settings_from_config
 
 
 class Container:
     """This class is for the BIDS App Container"""
 
-    def __init__(self, container_ds, container_name, config_yaml_file):
+    def __init__(self, container_ds, container_name, config_yaml_file, container_image_path=None):
         """
         This is to initialize Container class.
 
@@ -67,9 +71,7 @@ def __init__(self, container_ds, container_name, config_yaml_file):
         with open(self.config_yaml_file) as f:
             self.config = yaml.safe_load(f)
 
-        self.container_path_relToAnalysis = op.join(
-            'containers', '.datalad', 'environments', self.container_name, 'image'
-        )
+        self.container_path_relToAnalysis = container_image_path
 
     def sanity_check(self, analysis_path):
         """
@@ -101,6 +103,28 @@ def sanity_check(self, analysis_path):
             + "'."
         )
 
+    def generate_bash_zip_outputs(self, bash_path, processing_level):
+        """Generate a bash script that only zips BIDS App outputs."""
+        dict_zip_foldernames, _ = app_output_settings_from_config(self.config)
+        cmd_zip = get_output_zipping_cmds(dict_zip_foldernames, processing_level)
+
+        env = Environment(
+            loader=PackageLoader('babs', 'templates'),
+            trim_blocks=True,
+            lstrip_blocks=True,
+            autoescape=False,
+            undefined=StrictUndefined,
+        )
+        template = env.get_template('zip_outputs.sh.jinja2')
+        script_content = template.render(
+            processing_level=processing_level,
+            cmd_zip=cmd_zip,
+        )
+
+        with open(bash_path, 'w') as f:
+            f.write(script_content)
+        os.chmod(bash_path, 0o700)
+
     def generate_bash_run_bidsapp(self, bash_path, input_ds, processing_level):
         """
         This is to generate a bash script that runs the BIDS App singularity image.
@@ -165,16 +189,34 @@ def generate_bash_participant_job(
             Shown in the script error message when PROJECT_ROOT is unset.
         """
 
+        input_datasets = input_ds.as_records()
+        _, bids_app_output_dir = app_output_settings_from_config(self.config)
+
+        raw_bids_app_args = self.config.get('bids_app_args', None)
+        if raw_bids_app_args:
+            bids_app_args, subject_selection_flag, _, _, bids_app_input_dir = (
+                bids_app_args_from_config(raw_bids_app_args, input_datasets)
+            )
+        else:
+            bids_app_args = []
+            subject_selection_flag = '--participant-label'
+            bids_app_input_dir = input_datasets[0]['unzipped_path_containing_subject_dirs']
+
         script_content = generate_submit_script(
             queue_system=system.type,
             cluster_resources_config=self.config['cluster_resources'],
             script_preamble=self.config['script_preamble'],
             job_scratch_directory=self.config['job_compute_space'],
-            input_datasets=input_ds.as_records(),
+            input_datasets=input_datasets,
             processing_level=processing_level,
             container_name=self.container_name,
             zip_foldernames=self.config['zip_foldernames'],
             project_root=project_root,
+            container_image_path=self.container_path_relToAnalysis,
+            bids_app_args=bids_app_args,
+            bids_app_input_dir=bids_app_input_dir,
+            bids_app_output_dir=bids_app_output_dir,
+            subject_selection_flag=subject_selection_flag,
         )
 
         with open(bash_path, 'w') as f:

babs/generate_submit_script.py

@@ -30,6 +30,11 @@ def generate_submit_script(
     container_images=None,
     datalad_run_message=None,
     project_root=None,
+    container_image_path=None,
+    bids_app_args=None,
+    bids_app_input_dir=None,
+    bids_app_output_dir=None,
+    subject_selection_flag=None,
 ):
     """
     Generate a bash script that runs the BIDS App singularity image.
@@ -122,6 +127,11 @@ def generate_submit_script(
         container_images=container_images,
         datalad_run_message=datalad_run_message,
         project_root=project_root,
+        container_image_path=container_image_path,
+        bids_app_args=bids_app_args or [],
+        bids_app_input_dir=bids_app_input_dir or '',
+        bids_app_output_dir=bids_app_output_dir or '',
+        subject_selection_flag=subject_selection_flag or '',
     )