Skip to content

Commit 4b1d010

Browse files
jagednjagedn
authored
E2e test (#109)
* add nf-test as end to end validation Signed-off-by: Jorge Aguilera <[email protected]> * add spread validation to nf-test Signed-off-by: Jorge Aguilera <[email protected]> * refactor and more tests for nf-test Signed-off-by: Jorge Aguilera <[email protected]> --------- Signed-off-by: Jorge Aguilera <[email protected]>
1 parent c67cec2 commit 4b1d010

29 files changed

+661
-2
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,5 @@ oracle-nomad-cluster
1616
.settings
1717
/validation/nomad_temp/**
1818
/validation/nomad
19-
**/*.tsv
19+
**/*.tsv
20+
/src/e2e/.nf-test.log

‎README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,17 @@ To run and test the plugin in a development environment, configure a local Nextf
8484
./launch.sh run main.nf -plugins nf-nomad
8585
```
8686

87+
## End 2 End test (nf-test)
88+
89+
Project uses `nf-test` to run end to end integration tests running "real" pipelines against a local nomad
90+
91+
- compile and install a 99.99.99 version (`./gradlew clean installPlugin -Pversion=99.99.99)
92+
- open a terminal at `src/e2e/nomad` and execute `sudo ./start-nomad.sh` (this command will create a nomad server+client in the /tmp/nomad folder)
93+
- follow terminal instructions to get the NOMAD_TOKEN (`source /mtp/nomad/nomad_temp/.env`)
94+
- run all tests (`nf-test test`)
95+
96+
97+
8798
## Package, upload and publish
8899
89100
The project should be hosted in a GitHub repository whose name should match the name of the plugin, that is the name of the directory in the `plugins` folder (e.g. `nf-nomad`).

‎src/e2e/nf-test‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
APP_HOME="$HOME/.nf-test"
3+
APP_JAR="nf-test.jar"
4+
APP_UPDATE_URL="https://code.askimed.com/install/nf-test"
5+
6+
set -e
7+
8+
FOLDER=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
9+
10+
update() {
11+
if command -v curl > /dev/null 2>&1; then
12+
curl -fsSL ${APP_UPDATE_URL} | bash
13+
else
14+
wget -qO- ${APP_UPDATE_URL} | bash
15+
fi
16+
}
17+
18+
# nf-test uses the same logic as Nextflow to ensure to pick up the same jvm.
19+
# https://github.com/nextflow-io/nextflow/blob/master/nextflow#L263
20+
if [[ "$NXF_JAVA_HOME" ]]; then
21+
JAVA_HOME="$NXF_JAVA_HOME"
22+
unset JAVA_CMD
23+
fi
24+
# Determine the Java command to use to start the JVM.
25+
if [ ! -x "$JAVA_CMD" ] ; then
26+
if [ -d "$JAVA_HOME" ] ; then
27+
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
28+
# IBM's JDK on AIX uses strange locations for the executables
29+
JAVA_CMD="$JAVA_HOME/jre/sh/java"
30+
else
31+
JAVA_CMD="$JAVA_HOME/bin/java"
32+
fi
33+
elif [ -x /usr/libexec/java_home ]; then
34+
JAVA_CMD="$(/usr/libexec/java_home -v 1.8+)/bin/java"
35+
else
36+
JAVA_CMD="$(which java)" || JAVA_CMD=java
37+
fi
38+
fi
39+
40+
if test -f "${FOLDER}/${APP_JAR}"; then
41+
FILE_PATH_JAR=${FOLDER}/${APP_JAR}
42+
else
43+
FILE_PATH_JAR=${APP_HOME}/${APP_JAR}
44+
fi
45+
46+
JAVA_ARGS="-Xmx10G"
47+
if [[ "$NFT_JAVA_ARGS" ]]; then
48+
JAVA_ARGS="$NFT_JAVA_ARGS"
49+
fi
50+
51+
export JAVA_PROGRAM_ARGS=`echo "$@"`
52+
53+
if [ "${JAVA_PROGRAM_ARGS}" = "update" ]; then
54+
echo "Updating application..."
55+
cd "${FOLDER}"
56+
update
57+
else
58+
exec ${JAVA_CMD} ${JAVA_ARGS} -jar "${FILE_PATH_JAR}" "$@"
59+
fi

‎src/e2e/nf-test.config‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
config {
2+
3+
testsDir "tests"
4+
workDir "/tmp/nomad/nomad_temp/scratchdir/"
5+
configFile "tests/nextflow.config"
6+
profile ""
7+
8+
}

‎src/e2e/nomad/client.conf‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
2+
bind_addr = "0.0.0.0" # the default
3+
4+
datacenter = "test-datacenter"
5+
6+
plugin "docker" {
7+
config {
8+
allow_privileged = true
9+
volumes{
10+
enabled = true
11+
}
12+
gc {
13+
image = false
14+
image_delay = "1h"
15+
}
16+
}
17+
}
18+
19+
client {
20+
enabled = true
21+
node_class = "node"
22+
server_join {
23+
retry_join = [
24+
"localhost"
25+
]
26+
retry_max = 3
27+
retry_interval = "15s"
28+
}
29+
}

‎src/e2e/nomad/global-acl.hcl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
job "*/" {
2+
policy = "write"
3+
}
4+
5+
namespace "*" {
6+
policy = "write"
7+
8+
# this policy can write, read, or destroy any variable in any namespace
9+
variables {
10+
path "*" {
11+
capabilities = ["write", "read", "list", "destroy"]
12+
}
13+
}
14+
}
15+
16+
17+
host_volume "scratchdir" {
18+
policy = "write"
19+
}
20+

‎src/e2e/nomad/install-nomad.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
set -ue
3+
4+
NOMAD_VERSION="1.8.1"
5+
NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64}
6+
7+
## Available platforms
8+
#- "linux_amd64"
9+
#- "linux_arm64"
10+
#- "darwin_amd64"
11+
#- "darwin_arm64"
12+
#- "windows_amd64"
13+
14+
15+
if [ ! -f ./nomad ]; then
16+
curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip"
17+
unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip
18+
rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt
19+
chmod +x ./nomad
20+
fi

‎src/e2e/nomad/nf-nomad-acl.hcl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
job "*/" {
2+
policy = "write"
3+
}
4+
5+
namespace "*" {
6+
policy = "write"
7+
8+
# this policy can write, read, or destroy any variable in any namespace
9+
variables {
10+
path "*" {
11+
capabilities = ["write", "read", "list", "destroy"]
12+
}
13+
}
14+
}
15+
16+
17+
host_volume "scratchdir" {
18+
policy = "write"
19+
}
20+

‎src/e2e/nomad/server.conf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
2+
bind_addr = "0.0.0.0" # the default
3+
4+
server {
5+
enabled = true
6+
bootstrap_expect = 1
7+
}
8+
9+
acl {
10+
enabled = true
11+
}

‎src/e2e/nomad/start-nomad.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
set -ue
3+
4+
TMP_NOMAD=/tmp/nomad
5+
mkdir -p $TMP_NOMAD
6+
cp install-nomad.sh $TMP_NOMAD
7+
cp stop-nomad.sh $TMP_NOMAD
8+
cp server.conf $TMP_NOMAD
9+
cp client.conf $TMP_NOMAD
10+
cp wait-nomad.sh $TMP_NOMAD
11+
cp *.hcl $TMP_NOMAD
12+
cd $TMP_NOMAD
13+
./install-nomad.sh
14+
15+
mkdir -p nomad_temp
16+
cd nomad_temp
17+
18+
CURRENT_DIR=$(pwd)
19+
20+
mkdir -p client -p server -p scratchdir
21+
chmod ugoa+rw -R scratchdir
22+
23+
rm -f server-custom.conf
24+
cat >server-custom.conf <<EOL
25+
data_dir = "${CURRENT_DIR}/server"
26+
EOL
27+
28+
rm -f client-custom.conf
29+
cat >client-custom.conf <<EOL
30+
data_dir = "${CURRENT_DIR}/client"
31+
32+
client{
33+
host_volume "scratchdir" {
34+
path = "${CURRENT_DIR}/scratchdir"
35+
read_only = false
36+
}
37+
}
38+
EOL
39+
40+
cp ../server.conf .
41+
cp ../client.conf .
42+
43+
# secured nomad cluster
44+
../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf&
45+
cd ..
46+
./wait-nomad.sh
47+
48+
NOMAD_TOKEN=$(./nomad acl bootstrap | awk '/^Secret ID/ {print $4}')
49+
export NOMAD_TOKEN=$NOMAD_TOKEN
50+
echo New super token generated: $NOMAD_TOKEN
51+
52+
./nomad namespace apply -description "local-nomadlab" nf-nomad
53+
./nomad var put -namespace nf-nomad secrets/nf-nomad/MY_ACCESS_KEY MY_ACCESS_KEY=TheAccessKey
54+
./nomad var put -namespace nf-nomad secrets/nf-nomad/MY_SECRET_KEY MY_SECRET_KEY=TheSecretKey
55+
./nomad acl policy apply -description "nf-nomad policy" nf-nomad-ops global-acl.hcl
56+
./nomad acl policy apply -description "nf-nomad policy" -namespace nf-nomad -job 'MAIN:sayHello (3)' var-ops nf-nomad-acl.hcl
57+
58+
CLI_NOMAD_TOKEN=$(./nomad acl token create -name="nf-nomad" -policy="nf-nomad-ops" -policy="var-ops" -type=client | awk '/^Secret ID/ {print $4}')
59+
echo export NOMAD_TOKEN=$CLI_NOMAD_TOKEN >> ${CURRENT_DIR}/.env
60+
61+
echo run the following in the terminal:
62+
echo " source ${CURRENT_DIR}/.env"
63+
echo " ./nf-test test"

0 commit comments

Comments
 (0)