refactor and more tests for nf-test

jagedn · jagedn · commit e3612d3e470e · 2025-10-20T20:50:11.000+02:00
Signed-off-by: Jorge Aguilera &lt;jagedn@gmail.com&gt;
diff --git a/README.md b/README.md
@@ -84,6 +84,17 @@ To run and test the plugin in a development environment, configure a local Nextf
     ./launch.sh run main.nf -plugins nf-nomad
     ```
 
+## End 2 End test (nf-test)
+
+Project uses `nf-test` to run end to end integration tests running "real" pipelines against a local nomad
+
+- compile and install a 99.99.99 version (`./gradlew clean installPlugin -Pversion=99.99.99)
+- open a terminal at `src/e2e/nomad` and execute `sudo ./start-nomad.sh` (this command will create a nomad server+client in the /tmp/nomad folder)
+- follow terminal instructions to get the NOMAD_TOKEN (`source /mtp/nomad/nomad_temp/.env`)
+- run all tests (`nf-test test`)
+
+
+
 ## Package, upload and publish
 
 The project should be hosted in a GitHub repository whose name should match the name of the plugin, that is the name of the directory in the `plugins` folder (e.g. `nf-nomad`).
diff --git a/src/e2e/nomad/client.conf b/src/e2e/nomad/client.conf
diff --git a/src/e2e/nomad/global-acl.hcl b/src/e2e/nomad/global-acl.hcl
@@ -0,0 +1,20 @@
+job "*/" {
+  policy = "write"
+}
+
+namespace "*" {
+  policy = "write"
+
+  # this policy can write, read, or destroy any variable in any namespace
+  variables {
+    path "*" {
+      capabilities = ["write", "read", "list", "destroy"]
+    }
+  }
+}
+
+
+host_volume "scratchdir" {
+  policy = "write"
+}
+
diff --git a/src/e2e/nomad/install-nomad.sh b/src/e2e/nomad/install-nomad.sh
diff --git a/src/e2e/nomad/nf-nomad-acl.hcl b/src/e2e/nomad/nf-nomad-acl.hcl
@@ -0,0 +1,20 @@
+job "*/" {
+  policy = "write"
+}
+
+namespace "*" {
+  policy = "write"
+
+  # this policy can write, read, or destroy any variable in any namespace
+  variables {
+    path "*" {
+      capabilities = ["write", "read", "list", "destroy"]
+    }
+  }
+}
+
+
+host_volume "scratchdir" {
+  policy = "write"
+}
+
diff --git a/src/e2e/nomad/server.conf b/src/e2e/nomad/server.conf
@@ -6,3 +6,6 @@ server {
   bootstrap_expect = 1
 }
 
+acl {
+  enabled = true
+}
diff --git a/src/e2e/nomad/start-nomad.sh b/src/e2e/nomad/start-nomad.sh
@@ -8,6 +8,7 @@ cp stop-nomad.sh $TMP_NOMAD
 cp server.conf $TMP_NOMAD
 cp client.conf $TMP_NOMAD
 cp wait-nomad.sh $TMP_NOMAD
+cp *.hcl $TMP_NOMAD
 cd $TMP_NOMAD
 ./install-nomad.sh
 
@@ -24,12 +25,6 @@ cat >server-custom.conf <<EOL
 data_dir  = "${CURRENT_DIR}/server"
 EOL
 
-cat >>server-custom.conf <<EOL
-acl {
-  enabled = true
-}
-EOL
-
 rm -f client-custom.conf
 cat >client-custom.conf <<EOL
 data_dir  = "${CURRENT_DIR}/client"
@@ -46,15 +41,23 @@ cp ../server.conf .
 cp ../client.conf .
 
 # secured nomad cluster
-../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
+../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf&
 cd ..
 ./wait-nomad.sh
 
 NOMAD_TOKEN=$(./nomad acl bootstrap | awk '/^Secret ID/ {print $4}')
-export NOMAD_TOKEN
-echo New super token generated.
-echo export NOMAD_TOKEN=$NOMAD_TOKEN
+export NOMAD_TOKEN=$NOMAD_TOKEN
+echo New super token generated: $NOMAD_TOKEN
 
 ./nomad namespace apply -description "local-nomadlab" nf-nomad
-./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_ACCESS_KEY MY_ACCESS_KEY=TheAccessKey
-./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_SECRET_KEY MY_SECRET_KEY=TheSecretKey
+./nomad var put -namespace nf-nomad secrets/nf-nomad/MY_ACCESS_KEY MY_ACCESS_KEY=TheAccessKey
+./nomad var put -namespace nf-nomad secrets/nf-nomad/MY_SECRET_KEY MY_SECRET_KEY=TheSecretKey
+./nomad acl policy apply -description "nf-nomad policy" nf-nomad-ops global-acl.hcl
+./nomad acl policy apply -description "nf-nomad policy" -namespace nf-nomad -job 'MAIN:sayHello (3)' var-ops nf-nomad-acl.hcl
+
+CLI_NOMAD_TOKEN=$(./nomad acl token create -name="nf-nomad" -policy="nf-nomad-ops" -policy="var-ops" -type=client | awk '/^Secret ID/ {print $4}')
+echo export NOMAD_TOKEN=$CLI_NOMAD_TOKEN >> ${CURRENT_DIR}/.env
+
+echo run the following in the terminal:
+echo "    source ${CURRENT_DIR}/.env"
+echo "    ./nf-test test"
diff --git a/src/e2e/nomad/stop-nomad.sh b/src/e2e/nomad/stop-nomad.sh
@@ -8,8 +8,9 @@ cd $TMP_NOMAD
 
 ./nomad node drain -enable $(./nomad node status -quiet)
 ./nomad system gc
+kill $(ps aux | grep '../nomad agent' | awk '{print $2}')
 sleep 1
 df -h --output=target | grep nf-task | xargs sudo umount
-kill $(ps aux | grep '../nomad agent' | awk '{print $2}')
 sleep 1
-rm -rf nomad_temp
+cd $TMP_NOMAD
+rm -rf *
diff --git a/src/e2e/nomad/wait-nomad.sh b/src/e2e/nomad/wait-nomad.sh
diff --git a/src/e2e/tests/basic.nf.test b/src/e2e/tests/basic.nf.test
@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Basic"
-    script "tests/basic/main.nf"
+    script "workflows/hello.nf"
     workflow "MAIN"
 
     test("Should run without failures") {
diff --git a/src/e2e/tests/basic.nf.test.snap b/src/e2e/tests/basic.nf.test.snap
@@ -14,6 +14,6 @@
             "nf-test": "0.9.3",
             "nextflow": "25.04.7"
         },
-        "timestamp": "2025-10-16T18:27:07.576429197"
+        "timestamp": "2025-10-18T12:56:29.780860052"
     }
 }
diff --git a/src/e2e/tests/constraints.config b/src/e2e/tests/constraints.config
@@ -0,0 +1,27 @@
+plugins {
+    id "nf-nomad@99.99.99"
+}
+
+process {
+    executor = "nomad"
+}
+
+nomad {
+
+    client {
+        address = "http://127.0.0.1:4646"
+    }
+
+    jobs {
+        deleteOnCompletion = true
+        volumes = [
+            { type "host" name "scratchdir" path '/tmp/nomad/nomad_temp/scratchdir/'},
+        ]
+    }
+
+    namespace = 'nf-nomad'
+
+    secrets {
+        enabled = true
+    }
+}
diff --git a/src/e2e/tests/constraints.nf.test b/src/e2e/tests/constraints.nf.test
@@ -1,7 +1,8 @@
 nextflow_workflow {
 
     name "Test Constraints"
-    script "tests/constraints/main.nf"
+    script "workflows/hello.nf"
+    config "tests/constraints.config"
     workflow "MAIN"
 
     test("Should run without failures") {
diff --git a/src/e2e/tests/constraints.nf.test.snap b/src/e2e/tests/constraints.nf.test.snap
@@ -14,6 +14,6 @@
             "nf-test": "0.9.3",
             "nextflow": "25.04.7"
         },
-        "timestamp": "2025-10-15T17:13:14.073637126"
+        "timestamp": "2025-10-18T12:56:41.772090797"
     }
 }
diff --git a/src/e2e/tests/constraints/main.nf b/src/e2e/tests/constraints/main.nf
diff --git a/src/e2e/tests/multiple-vols.config b/src/e2e/tests/multiple-vols.config
@@ -0,0 +1,25 @@
+plugins {
+    id "nf-nomad@99.99.99"
+}
+
+process {
+    executor = "nomad"
+}
+
+nomad {
+
+    client {
+        address = "http://127.0.0.1:4646"
+    }
+
+    jobs {
+        deleteOnCompletion = false
+
+        volumes = [
+            { type "host" name "scratchdir" path '/tmp/nomad/nomad_temp/scratchdir/'},
+            { type "host" name "scratchdir" path "/var/data1/" },
+            { type "host" name "scratchdir" path "/var/data2/" }
+        ]
+    }
+
+}
diff --git a/src/e2e/tests/multiple-vols.nf.test b/src/e2e/tests/multiple-vols.nf.test
@@ -0,0 +1,24 @@
+nextflow_workflow {
+
+    name "Test Multiple vols"
+    script "workflows/multiple-vols.nf"
+    config "tests/multiple-vols.config"
+    workflow "MAIN"
+
+    test("Should run without failures") {
+
+        when {
+            params {
+            }
+            workflow {
+            }
+        }
+
+        then {
+            assert workflow.success
+            assert snapshot(workflow.out).match()
+        }
+
+    }
+
+}
diff --git a/src/e2e/tests/multiple-vols.nf.test.snap b/src/e2e/tests/multiple-vols.nf.test.snap
@@ -0,0 +1,19 @@
+{
+    "Should run without failures": {
+        "content": [
+            {
+                "0": [
+                    "Bonjour\n",
+                    "Ciao\n",
+                    "Hello\n",
+                    "Hola\n"
+                ]
+            }
+        ],
+        "meta": {
+            "nf-test": "0.9.3",
+            "nextflow": "25.04.7"
+        },
+        "timestamp": "2025-10-20T20:44:12.406843585"
+    }
+}
diff --git a/src/e2e/tests/spread.config b/src/e2e/tests/spread.config
diff --git a/src/e2e/tests/spread.nf.test b/src/e2e/tests/spread.nf.test
@@ -1,9 +1,9 @@
 nextflow_workflow {
 
     name "Test Spread"
-    script "tests/spread/main.nf"
+    script "workflows/hello.nf"
+    config "tests/nextflow.config"
     workflow "MAIN"
-    config "tests/spread/nextflow.config"
 
     test("Should run without failures") {
 
diff --git a/src/e2e/tests/spread.nf.test.snap b/src/e2e/tests/spread.nf.test.snap
@@ -14,6 +14,6 @@
             "nf-test": "0.9.3",
             "nextflow": "25.04.7"
         },
-        "timestamp": "2025-10-15T17:25:07.048607826"
+        "timestamp": "2025-10-18T12:56:56.005109774"
     }
 }
diff --git a/src/e2e/tests/wrong-spread-nextflow.config b/src/e2e/tests/wrong-spread-nextflow.config
diff --git a/src/e2e/tests/wrong-spread.nf.test b/src/e2e/tests/wrong-spread.nf.test
@@ -1,9 +1,9 @@
 nextflow_workflow {
 
     name "Test Wrong Spread"
-    script "tests/spread/main.nf"
+    script "workflows/hello.nf"
+    config "tests/wrong-spread-nextflow.config"
     workflow "MAIN"
-    config "tests/spread/wrong-spread-nextflow.config"
 
     test("Should run without failures") {
 
diff --git a/src/e2e/workflows/hello.nf b/src/e2e/workflows/hello.nf
diff --git a/src/e2e/workflows/multiple-vols.nf b/src/e2e/workflows/multiple-vols.nf
@@ -9,7 +9,9 @@ process sayHello {
     stdout
     script:
     """
-    echo '$x world!'
+    echo '$x world!' > /var/data1/hello
+    cp /var/data1/hello /var/data2/hello2
+    echo '$x'
     """
 }
 

Original file line number	Diff line number	Diff line change
`@@ -6,3 +6,6 @@ server {`
`6`	`6`	`bootstrap_expect = 1`
`7`	`7`	`}`
`8`	`8`
	`9`	`+acl {`
	`10`	`+ enabled = true`
	`11`	`+}`