Merge pull request #74 from UCLOrengoGroup/nf_test

Added nf-test

Author: sillitoe

.github/workflows/build-test-push.yml

@@ -1,5 +1,4 @@
 # build docker images from docker compose and push to docker hub
-
 name: Build, Test, and Push Docker Compose Images
 on:
   push:
@@ -36,6 +35,12 @@ jobs:
           sudo mv nextflow /usr/local/bin/nextflow
           nextflow -version
 
+      - name: Install nf-test
+        run: |
+          curl -fsSL https://get.nf-test.com | bash
+          sudo mv nf-test /usr/local/bin/nf-test
+          nf-test version
+
       - name: Build docker images for tests (amd64 only)
         run: |
           docker compose version
@@ -50,18 +55,28 @@ jobs:
           docker tag domain-annotation-pipeline-ted-tools ghcr.io/uclorengogroup/domain-annotation-pipeline-ted-tools:${TEST_TAG}
           docker tag domain-annotation-pipeline-foldseek ghcr.io/uclorengogroup/domain-annotation-pipeline-foldseek:${TEST_TAG}
 
-      - name: Run pipeline (docker,git_actions_test)
+      - name: Run test
         env:
           NXF_ANSI_LOG: "false"
+          CONTAINER_TAG_NAME: ci-local
+          NFT_DIFF_ARGS: "-u -U 5"
         run: |
           set -euxo pipefail
-          rm -rf work .nextflow* results/git_actions_test || true
-          nextflow run workflows/annotate.nf -profile docker,git_actions_test --container_tag_name ci-local
+          nf-test test --profile +docker --ci
 
-      - name: Compare final_results.tsv to expected
-        run: |
-          set -euxo pipefail
-          diff -U 0 <(sort assets/test_ids/final_results.tsv) <(sort results/git_actions_test/final_results.tsv)
+      - name: Upload nf-test artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nf-test-failure-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .nf-test/tests/*/output
+            .nf-test/tests/*/reports
+            .nf-test/tests/*/meta
+            .nextflow.log
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 7
 
   push-images:
     needs: test

.github/workflows/test-pipeline.yml

@@ -22,24 +22,40 @@ jobs:
         run: |
           curl -s https://get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/nextflow
-          nextflow -version
+          nextflow version
+      
+      - name: Install nf-test
+        run: |
+          curl -fsSL https://get.nf-test.com | bash
+          sudo mv nf-test /usr/local/bin/nf-test
+          nf-test -version
 
       - name: Build docker images for tests (temporary until GHCR default)
         run: |
           docker compose version
           docker compose build
 
-      - name: Run pipeline (docker,git_actions_test)
+      - name: Run test
         env:
           NXF_ANSI_LOG: "false"
+          NFT_DIFF_ARGS: "-u -U 5"
         run: |
           set -euxo pipefail
-          rm -rf work .nextflow* results/git_actions_test || true
-          nextflow run workflows/annotate.nf -profile docker,git_actions_test
+          nf-test test --profile +docker --ci
+
+      - name: Upload nf-test artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nf-test-failure-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .nf-test/tests/*/output
+            .nf-test/tests/*/reports
+            .nf-test/tests/*/meta
+            .nextflow.log
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 7
 
-      - name: Compare final_results.tsv to expected
-        run: |
-          set -euxo pipefail
-          diff -U 0 <(sort assets/test_ids/final_results.tsv) <(sort results/git_actions_test/final_results.tsv)
 
 

.gitignore

@@ -118,6 +118,8 @@ repo-key.pub
 .nextflow/
 work/
 .nextflow.log*
+.nf-test/
+nf-test
 
 # local build dir
 domain_assignments*.tsv
@@ -127,3 +129,5 @@ assets/uniprot_ids_100.csv
 assets/uniprot_1000.txt
 assets/bfvd.zip
 foldseek/assets/
+
+

README.md

@@ -194,6 +194,34 @@ ted_stub_chain_ids.zip
 
 The default files are currently set up to run a test set of 50 chain ids, producing a final results output of 100 domains.
 
+## Automated tests (nf-test)
+
+The pipeline has an [nf-test](https://www.nf-test.com/) (`tests/`) that runs the full `annotate.nf` workflow against the test data located in `./assets/test_ids` and checks the results against a pre-generated snapshot.
+
+Install nf-test:
+
+```bash
+curl -fsSL https://get.nf-test.com | bash
+```
+
+Move `nf-test` to the `bin` folder in you `$PATH`.
+
+Run the test locally:
+
+```bash
+nf-test test --profile +docker
+```
+
+If the outputs are consistent with the pre-existing snapshot, the test will succeed.
+
+If a change intentionally alters pipeline outputs, generate a new snapshot with:
+
+```bash
+nf-test test --update-snapshot
+```
+
+Note: nf-test isn't wired into CI yet — `test-pipeline.yml` and `build-test-push.yml` run the pipeline directly with `-profile docker,git_actions_test` rather than through nf-test. The idea is to replace it the nf-test at some point.
+
 ## Running on HPC
 
 ## Install (with singularity)

conf/test.config

@@ -0,0 +1,38 @@
+/*
+========================================================================================
+    Nextflow config file for running minimal tests
+========================================================================================
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run workflows/annotate.nf -profile test,singularity,myriad_cluster
+----------------------------------------------------------------------------------------
+*/
+
+process {
+    resourceLimits = [
+        cpus: 1,
+        memory: '8.GB',
+        time: '10m'
+    ]
+}
+
+params {
+    config_profile_name        = 'Test profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+
+    // Input data
+    input_zip_dir     = "${baseDir}/../assets/test_ids"
+
+    // Chunking
+    chunk_size        = 25
+    light_chunk_size  = 50
+    heavy_chunk_size  = 10
+
+    // Compute
+    no_gpu            = true
+
+    // GH Actions runners lack the CPU features the default foldseek build needs
+    foldseek_exec     = 'foldseek_avx2'
+    ci_mode           = true
+}

nextflow.config

@@ -217,6 +217,10 @@ profiles {
         }
     }
 
+    test {
+        includeConfig "${baseDir}/../conf/test.config"
+    }
+    
     container {
         includeConfig "${baseDir}/../conf/container.config"
     }

nf-test.config

@@ -0,0 +1,12 @@
+config {
+
+    testsDir "tests"
+    workDir ".nf-test"
+    configFile "tests/nextflow.config"
+    profile "test"
+
+    plugins {
+        load "nft-utils@0.0.3"
+    }
+
+}

tests/.nftignore

@@ -0,0 +1,7 @@
+final_results.tsv
+domain_assignments.merizo.tsv
+all_domain_quality.csv
+*.html
+**/*.html
+WORKFLOW_COMPLETED
+foldseek_parsed_results.tsv

tests/nextflow.config

@@ -0,0 +1,5 @@
+/*
+========================================================================================
+    Nextflow config file for running tests
+========================================================================================
+*/

tests/workflows/annotate.nf.test

@@ -0,0 +1,57 @@
+nextflow_pipeline {
+
+    name "Test Workflow annotate.nf"
+    script "workflows/annotate.nf"
+    tag "workflow"
+
+    test("-profile test") {
+
+        when {
+            params {
+                project_name       = "nf_test_annotate"
+                results_dir        = "$outputDir"
+                if (System.getenv('CONTAINER_TAG_NAME')) {
+                    container_tag_name = System.getenv('CONTAINER_TAG_NAME')
+                } else if (System.getenv('CONTAINER_TAG_BASE')) {
+                    container_tag_name = "${System.getenv('CONTAINER_TAG_BASE')}${params.container_arch == 'arm64' ? '-arm64-latest' : '-latest'}"
+                }
+            }
+        }
+
+        then {
+            assert workflow.success
+
+            // Rounds floats to `decimals` places. Little groovy magic to avoid floating-point differences in test comparisons
+            // e.g. with decimals=3, "1.2345" -> "1.235" (HALF_UP rounds the .5 tie up)
+            // Only whole fields matching the float pattern are rounded (via ==~, a full-string match) -
+            // multi-dot values like a cath_label "3.40.50.720" are left untouched instead of being
+            // mangled by matching "3.40" as a substring.
+            def roundFloats = { String path, String delimiter = '\t', int decimals = 3 ->
+                file(path).readLines().collect { line ->
+                    line.split(delimiter, -1).collect { field ->
+                        field ==~ /-?\d+\.\d+/ ? new BigDecimal(field).setScale(decimals, java.math.RoundingMode.HALF_UP).toString() : field
+                    }.join(delimiter)
+                }
+            }
+
+            // stable_path: all files/folders under the results dir with a stable path (incl. file name)
+            def stable_path = getAllFilesFromDir(params.results_dir, relative: true, includeDir: true, ignore: ['*.html', 'WORKFLOW_COMPLETED'])
+            // stable_content: all files under the results dir with stable content (volatile files excluded via .nftignore)
+            def stable_content = getAllFilesFromDir(params.results_dir, ignoreFile: 'tests/.nftignore')
+
+            // Files with known floating-point jitter: compare rounded content instead of raw bytes
+            def rounded_final_results  = roundFloats("${params.results_dir}/final_results.tsv")
+            def rounded_merizo         = roundFloats("${params.results_dir}/domain_assignments.merizo.tsv")
+            def rounded_domain_quality = roundFloats("${params.results_dir}/all_domain_quality.csv", ',')
+
+            assertAll(
+                { assert snapshot(
+                    stable_path, stable_content,
+                    rounded_final_results, rounded_merizo, rounded_domain_quality
+                ).match() }
+            )
+        }
+
+    }
+
+}