improve secrets provider (#81)

Signed-off-by: Jorge Aguilera <[email protected]>
Co-authored-by: Abhinav Sharma <[email protected]>

Author: jagedn

plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy

@@ -417,7 +417,8 @@ class NomadService implements Closeable{
                 config.jobOpts().region,
                 config.jobOpts().namespace,
                 null, null, null, null, null, null, null)
-        listRequest.collect{ it.path}
+        String path = (config.jobOpts().secretOpts?.path ?: '')+"/"
+        listRequest.collect{ it.path - path}
     }
 
     void deleteVariable(String key){

plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy

@@ -26,6 +26,16 @@ class NomadSecretProvider extends LocalSecretsProvider implements SecretsProvide
         return super.load()
     }
 
+    @Override
+    protected List<Secret> loadSecrets() {
+        Set<String> names = listSecretsNames()
+        List<Secret> ret = names.collect{ name->
+            String value = getSecret(name)
+            new SecretImpl(name, value)
+        }
+        ret
+    }
+
     protected boolean isEnabled(){
         if( !config ){
             config = new NomadConfig(Global.config?.nomad as Map ?: Map.of())

validation/install-nomad.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -ue
+
+NOMAD_VERSION="1.8.1"
+NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64}
+
+## Available platforms
+#- "linux_amd64"
+#- "linux_arm64"
+#- "darwin_amd64"
+#- "darwin_arm64"
+#- "windows_amd64"
+
+
+if [ ! -f ./nomad ]; then
+  curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip"
+  unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip
+  rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt
+  chmod +x ./nomad
+fi

validation/run-all.sh

@@ -46,6 +46,8 @@ if [ "$SKIPLOCAL" == 0 ]; then
     -profile test,docker --outdir $(pwd)/nomad_temp/scratchdir/bactopia/outdir \
     --datasets_cache $(pwd)/nomad_temp/scratchdir/bactopia/datasets
 
+  ./run-pipeline.sh -c secrets/nextflow.config secrets/main.nf
+
 else
   echo "skip local"
 fi

validation/secrets/nextflow.config

@@ -6,6 +6,11 @@ process {
     executor = "nomad"
 }
 
+aws {
+    accessKey = secrets.MY_ACCESS_KEY
+    secretKey = secrets.MY_SECRET_KEY
+}
+
 nomad {
 
     client {

validation/start-nomad.sh

@@ -1,26 +1,11 @@
 #!/bin/bash
 set -ue
 
-NOMAD_VERSION="1.8.1"
-NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64}
-
-## Available platforms
-#- "linux_amd64"
-#- "linux_arm64"
-#- "darwin_amd64"
-#- "darwin_arm64"
-#- "windows_amd64"
+./install-nomad.sh
 
 SECURE=0
 [[ "$@" =~ '--secure' ]] && SECURE=1
 
-if [ ! -f ./nomad ]; then
-  curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip"
-  unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip
-  rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt
-  chmod +x ./nomad
-fi
-
 mkdir -p nomad_temp
 cd nomad_temp
 
@@ -59,16 +44,19 @@ cp ../client.conf .
 
 if [ "$SECURE" == 0 ]; then
   # basic nomad cluter
-  ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf
+  ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
 else
-# secured nomad cluster
-../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
+  # secured nomad cluster
+  ../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
+  sleep 3
+  NOMAD_TOKEN=$(../nomad acl bootstrap | awk '/^Secret ID/ {print $4}')
+  export NOMAD_TOKEN
+  echo New super token generated.
+  echo export NOMAD_TOKEN=$NOMAD_TOKEN
+fi
+
 cd ..
-#./nomad namespace apply -description "local-nomadlab" nf-nomad
 ./wait-nomad.sh
-sleep 3
-NOMAD_TOKEN=$(nomad acl bootstrap | awk '/^Secret ID/ {print $4}')
-export NOMAD_TOKEN
-echo New super token generated.
-echo export NOMAD_TOKEN=$NOMAD_TOKEN
-fi
+./nomad namespace apply -description "local-nomadlab" nf-nomad
+./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_ACCESS_KEY MY_ACCESS_KEY=TheAccessKey
+./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_SECRET_KEY MY_SECRET_KEY=TheSecretKey

validation/stop-nomad.sh

@@ -3,7 +3,6 @@
 ./nomad system gc
 sleep 1
 df -h --output=target | grep nf-task | xargs sudo umount
-pkill -9 nomad
+kill $(ps aux | grep '../nomad agent' | awk '{print $2}')
 sleep 1
 rm -rf nomad_temp
-rm ./nomad