Merge pull request #78 from UCLOrengoGroup/dev

Merging dev features before release

Author: sillitoe

.github/workflows/build-test-push.yml

@@ -1,5 +1,4 @@
 # build docker images from docker compose and push to docker hub
-
 name: Build, Test, and Push Docker Compose Images
 on:
   push:
@@ -9,6 +8,7 @@ on:
   pull_request:
     branches:
       - main
+      - dev
   release:
     types:
       - published
@@ -17,7 +17,70 @@ permissions:
   packages: write
 
 jobs:
-  build:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Java (for Nextflow)
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "21"
+
+      - name: Install Nextflow
+        run: |
+          curl -s https://get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/nextflow
+          nextflow -version
+
+      - name: Install nf-test
+        run: |
+          curl -fsSL https://get.nf-test.com | bash
+          sudo mv nf-test /usr/local/bin/nf-test
+          nf-test version
+
+      - name: Build docker images for tests (amd64 only)
+        run: |
+          docker compose version
+          docker compose build
+
+      - name: Tag local images for Nextflow test run
+        run: |
+          set -euxo pipefail
+          TEST_TAG=ci-local
+          docker tag domain-annotation-pipeline-cath-af-cli ghcr.io/uclorengogroup/domain-annotation-pipeline-cath-af-cli:${TEST_TAG}
+          docker tag domain-annotation-pipeline-script ghcr.io/uclorengogroup/domain-annotation-pipeline-script:${TEST_TAG}
+          docker tag domain-annotation-pipeline-ted-tools ghcr.io/uclorengogroup/domain-annotation-pipeline-ted-tools:${TEST_TAG}
+          docker tag domain-annotation-pipeline-foldseek ghcr.io/uclorengogroup/domain-annotation-pipeline-foldseek:${TEST_TAG}
+
+      - name: Run test
+        env:
+          NXF_ANSI_LOG: "false"
+          CONTAINER_TAG_NAME: ci-local
+          NFT_DIFF_ARGS: "-u -U 5"
+        run: |
+          set -euxo pipefail
+          nf-test test --profile +docker --ci
+
+      - name: Upload nf-test artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nf-test-failure-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .nf-test/tests/*/output
+            .nf-test/tests/*/reports
+            .nf-test/tests/*/meta
+            .nextflow.log
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 7
+
+  push-images:
+    needs: test
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -36,7 +99,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push SGE-compatible images to GHCR (linux/amd64)
+      - name: Build and push amd64 and arm64 images to GHCR
         env:
           GH_OWNER: ${{ github.repository_owner }}
           GIT_SHA: ${{ github.sha }}
@@ -54,23 +117,39 @@ jobs:
             echo "Building and pushing ${service} for linux/amd64..."
             IMAGE_BASE=ghcr.io/${OWNER_LOWER}/domain-annotation-pipeline-${service}
 
-            TAG_ARGS=(
+            AMD64_TAG_ARGS=(
               --tag ${IMAGE_BASE}:sha-${GIT_SHA:0:8}
               --tag ${IMAGE_BASE}:linux-amd64-${DATE_TAG}
               --tag ${IMAGE_BASE}:${BRANCH_TAG}-latest
               --tag ${IMAGE_BASE}:linux-amd64-latest
             )
 
+            ARM64_TAG_ARGS=(
+              --tag ${IMAGE_BASE}:sha-${GIT_SHA:0:8}-arm64
+              --tag ${IMAGE_BASE}:linux-arm64-${DATE_TAG}
+              --tag ${IMAGE_BASE}:${BRANCH_TAG}-arm64-latest
+              --tag ${IMAGE_BASE}:linux-arm64-latest
+            )
+
             if [[ "${GITHUB_EVENT_NAME}" == "release" && -n "${RELEASE_TAG:-}" ]]; then
               RELEASE_TAG_SANITIZED=$(echo "${RELEASE_TAG}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g')
-              TAG_ARGS+=(--tag ${IMAGE_BASE}:${RELEASE_TAG_SANITIZED})
+              AMD64_TAG_ARGS+=(--tag ${IMAGE_BASE}:${RELEASE_TAG_SANITIZED})
+              ARM64_TAG_ARGS+=(--tag ${IMAGE_BASE}:${RELEASE_TAG_SANITIZED}-arm64)
             fi
 
             docker buildx build \
               --platform linux/amd64 \
-              "${TAG_ARGS[@]}" \
+              "${AMD64_TAG_ARGS[@]}" \
               --push \
               docker/${service}
+
+            echo "Building and pushing ${service} for linux/arm64..."
+            docker buildx build \
+              --platform linux/arm64 \
+              "${ARM64_TAG_ARGS[@]}" \
+              --push \
+              docker/${service}
+
             docker buildx rm || true
           done
 

.github/workflows/test-pipeline.yml

@@ -2,7 +2,7 @@
 name: Pipeline test (30 TED test IDs)
 
 on:
-  push:
+  workflow_dispatch:
 
 jobs:
   test:
@@ -22,24 +22,40 @@ jobs:
         run: |
           curl -s https://get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/nextflow
-          nextflow -version
+          nextflow version
+      
+      - name: Install nf-test
+        run: |
+          curl -fsSL https://get.nf-test.com | bash
+          sudo mv nf-test /usr/local/bin/nf-test
+          nf-test -version
 
       - name: Build docker images for tests (temporary until GHCR default)
         run: |
           docker compose version
           docker compose build
 
-      - name: Run pipeline (docker,git_actions_test)
+      - name: Run test
         env:
           NXF_ANSI_LOG: "false"
+          NFT_DIFF_ARGS: "-u -U 5"
         run: |
           set -euxo pipefail
-          rm -rf work .nextflow* results/git_actions_test || true
-          nextflow run workflows/annotate.nf -profile docker,git_actions_test
+          nf-test test --profile +docker --ci
+
+      - name: Upload nf-test artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nf-test-failure-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .nf-test/tests/*/output
+            .nf-test/tests/*/reports
+            .nf-test/tests/*/meta
+            .nextflow.log
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 7
 
-      - name: Compare final_results.tsv to expected
-        run: |
-          set -euxo pipefail
-          diff -U 0 <(sort assets/test_ids/final_results.tsv) <(sort results/git_actions_test/final_results.tsv)
 
 

.gitignore

@@ -2,6 +2,7 @@
 tools/
 domain_assignments*.tsv
 results/
+reports/
 # nextflow
 .nextflow/
 work/
@@ -117,6 +118,8 @@ repo-key.pub
 .nextflow/
 work/
 .nextflow.log*
+.nf-test/
+nf-test
 
 # local build dir
 domain_assignments*.tsv
@@ -126,3 +129,5 @@ assets/uniprot_ids_100.csv
 assets/uniprot_1000.txt
 assets/bfvd.zip
 foldseek/assets/
+
+

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 This file records major behavioural and configuration changes.
 
+## 2026-07-10 (Chris Wyatt) — Config restructure & composable profiles
+- New `conf/base.config` (always loaded): a universal per-process resource floor (`cpus=1`, `memory=2.GB*attempt`) so every module has a baseline, plus a correctness fix pinning the chunk producers (`chunk_ids_by_zip`/`light_chunk_consensus_by_zip`) to `scratch=false` — node-local scratch was deleting their mapping-file outputs and breaking `filter_pdb_from_zip`/`chop_pdb_from_zip`.
+- Per-process HPC resource ladders / retries remain in `conf/singularity.config` (HPC runs use singularity).
+- Profiles are now composable: added a standalone `container` profile; `cs_cluster` is generic + label-aware (requests `gpu=true` for `sge_gpu_high`, `avx2=yes` for foldseek) and runs on shared `/SAN` (`scratch=false`); `orengo` restored as a composable profile for backward compatibility. Compose e.g. `-profile <data>,singularity,cs_cluster`.
+- `cs_cluster` Singularity image cache now honours `NXF_SINGULARITY_CACHEDIR` (falls back to `$HOME/.apptainer/pull`); removed the unused `singularity_image_dir` param.
+- Added `test_154` profile: 154-id TED test set read from a public S3 bucket `s3://excon/protein-domain-annotation/zip/` using anonymous access (`aws.client.anonymous`) — no AWS login needed.
+
+## 2026-06-22 (Chris Wyatt) — TED segmentation concurrency
+- Split `run_ted_segmentation` into three modules so Chainsaw runs concurrently with the Merizo→UniDoc chain (Chainsaw is independent; UniDoc inherits Merizo's chopping). New modules: `run_ted_merizo_unidoc` and `run_ted_chainsaw` (both GPU), and `run_ted_consensus` (CPU, `sge_low`) which joins the three choppings by `chunk_id` and computes consensus.
+- Cuts this step's wall time to roughly `max(merizo+unidoc, chainsaw) + consensus`, at the cost of two GPU slots per chunk. Pipeline outputs are unchanged; the original `modules/run_ted_segmentation.nf` is retained but no longer used.
+
+## 2026-06-22 (Chris Wyatt)
+- Updated for Nextflow 25.10+/26 (see README). Fixed `--chunk_size` which broke under v26 (now coerced to an integer before validation).
+- Execution reports (timeline, report, trace, DAG) are now generated automatically into a timestamped `reports/` folder — no need to pass `-with-timeline`/`-with-report`/`-with-trace`.
+
 ## 2025-10-31 (Nick Edmunds)
 - Added this changelog to track key updates.
 - To run on the server must include a link to: -c /SAN/orengolab/bfvd/code/domain-annotation-pipeline/nextflow.config

README.md

@@ -16,6 +16,8 @@ pip install --upgrade pip
 pip install nextflow
 ```
 
+**Nextflow version:** use **25.10 or newer** (tested with 25.10.3 and 26.04.4). These releases use Nextflow's strict (v2) config parser, which `nextflow.config` is written for. If `pip install nextflow` gives you an older release, set the version explicitly, e.g. `NXF_VER=26.04.4 nextflow run ...`.
+
 Install Docker
 https://docs.docker.com/compose/install/
 
@@ -34,6 +36,46 @@ Note: either docker or singularity must be supplied as one the the profile argum
 nextflow run workflows/annotate.nf -profile debug,docker
 ```
 
+### Profiles
+
+Runs are configured by composing profiles with `-profile a,b,c`: pick **one container engine**, add a **cluster** profile on HPC, and a **data/mode** profile for the inputs.
+
+| Group | Profiles | Sets |
+|-------|----------|------|
+| Container engine *(pick one)* | `docker`, `singularity` | how containers run (also loads the `/app` script paths) |
+| Cluster *(optional, HPC/SGE)* | `cs_cluster`, `myriad_cluster`, `orengo` | executor + submit options (scratch, GPU, avx2) |
+| Data / mode | `debug`, `benchmark_test`, `stub_run`, `test_154`, … | test inputs / run parameters |
+
+A basic resource floor for every process lives in `conf/base.config` (always applied). The fuller per-process memory/retry ladders are HPC settings in `conf/singularity.config`, so they apply when you include the `singularity` profile — hence `-profile singularity,<cluster>` for real HPC jobs. (`container` is also available as a standalone profile: script paths only, for composing with a self-contained cluster profile such as `orengo`.)
+
+```bash
+# Local, Docker, bundled test data
+nextflow run workflows/annotate.nf -profile debug,docker
+
+# UCL CS cluster — Singularity, runs on shared /SAN (no node-local scratch)
+nextflow run workflows/annotate.nf -profile singularity,cs_cluster --input_zip_dir <dir>
+
+# UCL Myriad cluster
+nextflow run workflows/annotate.nf -profile singularity,myriad_cluster --input_zip_dir <dir>
+
+# Orengo-lab CS cluster — node-local scratch, project/avx2/GPU
+nextflow run workflows/annotate.nf -profile singularity,orengo --input_zip_dir <dir>
+
+# 154-id TED test set read from a public S3 bucket (no AWS login needed)
+nextflow run workflows/annotate.nf -profile test_154,singularity,cs_cluster
+```
+
+### Execution reports
+
+Execution timeline, report, trace and DAG files are generated **automatically** on every run — you do **not** need to pass `-with-timeline`, `-with-report`, `-with-trace` or `-with-dag`. They are written to the `reports/` folder (or under `--results_dir` if you set it) and the filenames include a per-launch timestamp, so successive runs never overwrite each other:
+
+```
+reports/execution_timeline_<timestamp>.html
+reports/execution_report_<timestamp>.html
+reports/execution_trace_<timestamp>.txt
+reports/pipeline_dag_<timestamp>.html
+```
+
 ## Preparing data
 
 The pipeline expects two inputs:
@@ -152,6 +194,34 @@ ted_stub_chain_ids.zip
 
 The default files are currently set up to run a test set of 50 chain ids, producing a final results output of 100 domains.
 
+## Automated tests (nf-test)
+
+The pipeline has an [nf-test](https://www.nf-test.com/) (`tests/`) that runs the full `annotate.nf` workflow against the test data located in `./assets/test_ids` and checks the results against a pre-generated snapshot.
+
+Install nf-test:
+
+```bash
+curl -fsSL https://get.nf-test.com | bash
+```
+
+Move `nf-test` to the `bin` folder in you `$PATH`.
+
+Run the test locally:
+
+```bash
+nf-test test --profile +docker
+```
+
+If the outputs are consistent with the pre-existing snapshot, the test will succeed.
+
+If a change intentionally alters pipeline outputs, generate a new snapshot with:
+
+```bash
+nf-test test --update-snapshot
+```
+
+Note: nf-test isn't wired into CI yet — `test-pipeline.yml` and `build-test-push.yml` run the pipeline directly with `-profile docker,git_actions_test` rather than through nf-test. The idea is to replace it the nf-test at some point.
+
 ## Running on HPC
 
 ## Install (with singularity)
@@ -209,9 +279,10 @@ singularity pull domain-annotation-pipeline-ted-tools_latest.sif docker://ghcr.i
 singularity pull domain-annotation-pipeline-foldseek_latest.sif docker://ghcr.io/uclorengogroup/domain-annotation-pipeline-foldseek:main-latest
 ```
 
-The directory containing these singularity images can be added to your config file, or passed directly to nextflow:
+You normally **don't** need to pull these manually — with `-profile singularity`, Nextflow pulls each image into its cache on first run. Control **where** they're stored (and share them across runs and users) with the `NXF_SINGULARITY_CACHEDIR` environment variable:
 
 ```bash
-nextflow run workflows/annotate -profile singularity \
-    --singularity_image_dir "/path/to/singularity_images"
+export NXF_SINGULARITY_CACHEDIR=/path/to/shared/singularity_cache
 ```
+
+The `cs_cluster` profile (see below) honours this variable, falling back to `$HOME/.apptainer/pull` when it is unset. (The old `--singularity_image_dir` flag is no longer wired up — set the cache via the environment variable instead.)

assets/test_ids/30-ted-ids.zip

@@ -46,3 +46,10 @@ A0A008_01	4579eb972b3c1762bd1d1fec24cfa006	high	57-320	264	1	40	8	9	17	23	11.41
 A0A002_01	fa6ce13b0cef05776b985792f9b1b804	med	325-410_462-563	188	2	40	7	10	17	23	11.387	0.285	89.9639	proteome-tax_id-67581-0_v4		Streptomyces viridosporus	cellular organisms (no rank), Bacteria (domain), Bacillati (kingdom), Actinomycetota (phylum), Actinomycetes (class), Kitasatosporales (order), Streptomycetaceae (family), Streptomyces (genus)	0.65248	False	2ff7A00	4.1e-19	0.8901	3.40.50.300	H	0.979	0.971	91.05
 A0A000_01	150d4a3366dff240048b39a8901f229d	high	54-288	235	1	39	13	7	20	19	12.397	0.316	96.88	proteome-tax_id-67581-0_v4		Streptomyces viridosporus	cellular organisms (no rank), Bacteria (domain), Bacillati (kingdom), Actinomycetota (phylum), Actinomycetes (class), Kitasatosporales (order), Streptomycetaceae (family), Streptomyces (genus)	0.86546	True	6hrhA01	4.089e-34	0.9761	3.40.640.10	H	0.991	0.951	97.16
 A0A000_02	e4bc15472c520f667a60762ee3a889a1	high	290-389	100	1	12	3	3	6	6	11.604	0.296	95.586	proteome-tax_id-67581-0_v4		Streptomyces viridosporus	cellular organisms (no rank), Bacteria (domain), Bacillati (kingdom), Actinomycetota (phylum), Actinomycetes (class), Kitasatosporales (order), Streptomycetaceae (family), Streptomyces (genus)	0.9507	True	2bwnB01	7.664e-08	0.8815	3.90.1150.10	H	0.91	0.572	78.05
+A0A009E3I5_01	a99a762ce7240ff800352b0b7313200e	high	6-30_38-70	58	2	9	1	4	5	4	9.808	0.349	81.7419					0.32405	True								28.97
+A0A009E8H7_01	78df653bd15aaf1a1482d4d5dd2291a8	high	9-56	48	1	5	3	0	3	2	10.292	0.309	59.7269					0.63986	True								26.22
+A0A009E8H8_01	aefdd5d66a782edd91eca69333dac6e1	high	21-67	47	1	3	2	0	2	1	9.233	0.397	68.0209					0.080558	False								27.25
+A0A009E9Q4_01	ea2e0d063dcd92eda724f53107932ee0	high	22-61	40	1	4	3	0	3	1	10.524	0.275	84.638					0.36831	True	2w25A01	0.4087	0.662	1.10.10	T	0.95	0.75	79.33
+A0A009E9Q4_02	295fcf62d6729ac7c2531e29752557cb	high	93-127	35	1	6	2	2	4	2	9.1	0.312	73.0903					0.032186	True								27.89
+A0A009ECM9_01	8609fefdf4edba2fef3cc4d3de0636c5	high	8-66	59	1	9	2	3	5	4	11.552	0.332	90.0792					0.56246	True	4py5A01	0.2214	0.64	3.30.310	T	0.898	0.833	84.16
+A0A009ED70_01	04f5a1c5b737ceab8de2af812661cf34	med	2-47	46	1	9	1	3	4	5	9.067	0.324	81.2983					0.52	True	1twfI02	0.3513	0.5461		N	0.913	0.618	53.56