Develop: fixes for test_environments

.github/workflows/testdask.yml

@@ -28,7 +28,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
-
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Setup Python version ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:

.github/workflows/testpsijlocal.yml

@@ -28,7 +28,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
-
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Setup Python version ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:

.github/workflows/testpsijslurm.yml

@@ -25,6 +25,8 @@ jobs:
     - name: Disable etelemetry
       run: echo "NO_ET=TRUE" >> $GITHUB_ENV
     - uses: actions/checkout@v4
+    - name: Fetch tags
+      run: git fetch --prune --unshallow
     - name: Pull docker image
       run: |
         docker pull $DOCKER_IMAGE
@@ -48,7 +50,7 @@ jobs:
             docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
         fi
         docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
-        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra; print(pydra.__version__)'"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
     - name: Run pytest
       run: |
         docker exec slurm bash -c "pytest --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"

.github/workflows/testpydra.yml

@@ -52,74 +52,35 @@ jobs:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
         python-version: ['3.11', '3.12', '3.13']
-        install: ['wheel']
-        include:
-          - os: 'ubuntu-latest'
-            python-version: '3.11'
-            install: 'sdist'
-          - os: 'ubuntu-latest'
-            python-version: '3.11'
-            install: 'repo'
-          - os: 'ubuntu-latest'
-            python-version: '3.11'
-            install: 'archive'
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
 
     steps:
-      - name: Fetch sdist/wheel
-        uses: actions/download-artifact@v4
-        if: matrix.install == 'sdist' || matrix.install == 'wheel'
-        with:
-          name: dist
-          path: dist/
-      - name: Fetch git archive
-        uses: actions/download-artifact@v4
-        if: matrix.install == 'archive'
-        with:
-          name: archive
-          path: archive/
       - name: Fetch repository
         uses: actions/checkout@v4
-        if: matrix.install == 'repo'
-
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Update pip
         run: python -m pip install --upgrade pip
 
-      - name: Determine installation target
-        run: |
-          if [[ "$INSTALL" = "sdist" ]]; then
-              echo "ARCHIVE=$( ls dist/*.tar.gz )" >> $GITHUB_ENV
-          elif [[ "$INSTALL" = "wheel" ]]; then
-              echo "ARCHIVE=$( ls dist/*.whl )" >> $GITHUB_ENV
-          elif [[ "$INSTALL" = "archive" ]]; then
-              echo "ARCHIVE=$( ls archive/*.zip )" >> $GITHUB_ENV
-          elif [[ "$INSTALL" = "repo" ]]; then
-              echo "ARCHIVE=." >> $GITHUB_ENV
-          fi
-        env:
-          INSTALL: ${{ matrix.install }}
-
       - name: Install Pydra
-        run: pip install $ARCHIVE
+        run: pip install .[test]
 
       - name: Print version
-        run: python -c "import pydra; print(pydra.__version__)"
-
-      - name: Install Pydra tests dependencies
-        run: pip install pydra[test]
+        run: python -c "import pydra.engine; print(pydra.engine.__version__)"
 
       - name: Disable etelemetry
         run:  echo "NO_ET=TRUE" >> $GITHUB_ENV
 
       - name: Pytest
         run: |
-          pytest -vs -n auto --doctest-modules --pyargs pydra \
+          pytest -vs -n auto --doctest-modules \
                  --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
 
       - name: Upload to codecov

.github/workflows/testsingularity.yml

@@ -14,7 +14,7 @@ concurrency:
 jobs:
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: ['3.11', '3.12', '3.13']
@@ -65,11 +65,13 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
+      - name: Fetch tags
+        run: git fetch --prune --unshallow
       - name: Install pydra (test)
         run: pip install -e ".[test]"
 
 
       - name: Pytest
-        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py
+        run: pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml pydra/engine/tests/test_singularity.py pydra/engine/tests/test_environments.py
       - name: Upload to codecov
         run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW

.github/workflows/testslurm.yml

@@ -25,6 +25,8 @@ jobs:
     - name: Disable etelemetry
       run: echo "NO_ET=TRUE" >> $GITHUB_ENV
     - uses: actions/checkout@v4
+    - name: Fetch tags
+      run: git fetch --prune --unshallow
     - name: Pull docker image
       run: |
         docker pull $DOCKER_IMAGE
@@ -48,7 +50,7 @@ jobs:
             docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
         fi
         docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
-        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra.engine; print(pydra.engine.__version__)'"
     - name: Run pytest
       run: |
         docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"

new-docs/source/tutorial/5-shell.ipynb

@@ -109,11 +109,6 @@
     "print(trim_png.cmdline)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -180,6 +175,68 @@
     "print(f\"'--int-arg' default: {fields_dict(Cp)['int_arg'].default}\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Path templates for output files\n",
+    "\n",
+    "By default, when an output file argument is defined, a `path_template` attribute will\n",
+    "be assigned to the field based on its name and extension (if applicable). For example,\n",
+    "the `zipped` output field in the following Gzip command will be assigned a\n",
+    "`path_template` of `out_file.gz`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydra.design import shell\n",
+    "from fileformats.generic import File\n",
+    "\n",
+    "Gzip = shell.define(\"gzip <out|out_file:application/gzip> <in_files+>\")\n",
+    "gzip = Gzip(in_files=File.mock(\"/a/file.txt\"))\n",
+    "print(gzip.cmdline)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "However, if this needs to be specified it can be by using the `$` operator, e.g."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Gzip = shell.define(\"gzip <out|out_file:application/gzip$zipped.gz> <in_files+>\")\n",
+    "gzip = Gzip(in_files=File.mock(\"/a/file.txt\"))\n",
+    "print(gzip.cmdline)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To give the field a path_template of `archive.gz` when it is written on the command line.\n",
+    "Note that this value can always be overridden when the task is initialised, e.g."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gzip = Gzip(in_files=File.mock(\"/a/file.txt\"), out_file=\"/path/to/archive.gz\")\n",
+    "print(gzip.cmdline)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

pydra/design/boutiques.py

@@ -6,8 +6,8 @@
 from functools import reduce
 from fileformats.generic import File
 from pydra.engine.specs import ShellDef
-from .base import make_task_def
-from . import shell
+from pydra.design.base import make_task_def
+from pydra.design import shell
 
 
 class arg(shell.arg):

pydra/design/python.py

@@ -2,7 +2,7 @@
 import inspect
 from typing import dataclass_transform
 import attrs
-from .base import (
+from pydra.design.base import (
     Arg,
     Out,
     ensure_field_objects,