Skip to content

Commit 0180d47

Browse files
jagednjagedn
committed
improve secrets provider
Signed-off-by: Jorge Aguilera <[email protected]>
1 parent 885c620 commit 0180d47

File tree

7 files changed

+54
-29
lines changed

7 files changed

+54
-29
lines changed

‎plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,8 @@ class NomadService implements Closeable{
417417
config.jobOpts().region,
418418
config.jobOpts().namespace,
419419
null, null, null, null, null, null, null)
420-
listRequest.collect{ it.path}
420+
String path = (config.jobOpts().secretOpts?.path ?: '')+"/"
421+
listRequest.collect{ it.path - path}
421422
}
422423

423424
void deleteVariable(String key){

‎plugins/nf-nomad/src/main/nextflow/nomad/secrets/NomadSecretProvider.groovy‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,16 @@ class NomadSecretProvider extends LocalSecretsProvider implements SecretsProvide
2626
return super.load()
2727
}
2828

29+
@Override
30+
protected List<Secret> loadSecrets() {
31+
Set<String> names = listSecretsNames()
32+
List<Secret> ret = names.collect{ name->
33+
String value = getSecret(name)
34+
new SecretImpl(name, value)
35+
}
36+
ret
37+
}
38+
2939
protected boolean isEnabled(){
3040
if( !config ){
3141
config = new NomadConfig(Global.config?.nomad as Map ?: Map.of())

‎validation/install-nomad.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
set -ue
3+
4+
NOMAD_VERSION="1.8.1"
5+
NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64}
6+
7+
## Available platforms
8+
#- "linux_amd64"
9+
#- "linux_arm64"
10+
#- "darwin_amd64"
11+
#- "darwin_arm64"
12+
#- "windows_amd64"
13+
14+
15+
if [ ! -f ./nomad ]; then
16+
curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip"
17+
unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip
18+
rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt
19+
chmod +x ./nomad
20+
fi

‎validation/run-all.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ if [ "$SKIPLOCAL" == 0 ]; then
4646
-profile test,docker --outdir $(pwd)/nomad_temp/scratchdir/bactopia/outdir \
4747
--datasets_cache $(pwd)/nomad_temp/scratchdir/bactopia/datasets
4848

49+
./run-pipeline.sh -c secrets/nextflow.config secrets/main.nf
50+
4951
else
5052
echo "skip local"
5153
fi

‎validation/secrets/nextflow.config‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ process {
66
executor = "nomad"
77
}
88

9+
aws {
10+
accessKey = secrets.MY_ACCESS_KEY
11+
secretKey = secrets.MY_SECRET_KEY
12+
}
13+
914
nomad {
1015

1116
client {

‎validation/start-nomad.sh‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,11 @@
11
#!/bin/bash
22
set -ue
33

4-
NOMAD_VERSION="1.8.1"
5-
NOMAD_PLATFORM=${NOMAD_PLATFORM:-linux_amd64}
6-
7-
## Available platforms
8-
#- "linux_amd64"
9-
#- "linux_arm64"
10-
#- "darwin_amd64"
11-
#- "darwin_arm64"
12-
#- "windows_amd64"
4+
./install-nomad.sh
135

146
SECURE=0
157
[[ "$@" =~ '--secure' ]] && SECURE=1
168

17-
if [ ! -f ./nomad ]; then
18-
curl -O "https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip"
19-
unzip nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip
20-
rm -f nomad_${NOMAD_VERSION}_${NOMAD_PLATFORM}.zip LICENSE.txt
21-
chmod +x ./nomad
22-
fi
23-
249
mkdir -p nomad_temp
2510
cd nomad_temp
2611

@@ -59,16 +44,19 @@ cp ../client.conf .
5944

6045
if [ "$SECURE" == 0 ]; then
6146
# basic nomad cluter
62-
../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf
47+
../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
6348
else
64-
# secured nomad cluster
65-
../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
49+
# secured nomad cluster
50+
../nomad agent -config server.conf -config client.conf -config server-custom.conf -config client-custom.conf &
51+
sleep 3
52+
NOMAD_TOKEN=$(../nomad acl bootstrap | awk '/^Secret ID/ {print $4}')
53+
export NOMAD_TOKEN
54+
echo New super token generated.
55+
echo export NOMAD_TOKEN=$NOMAD_TOKEN
56+
fi
57+
6658
cd ..
67-
#./nomad namespace apply -description "local-nomadlab" nf-nomad
6859
./wait-nomad.sh
69-
sleep 3
70-
NOMAD_TOKEN=$(nomad acl bootstrap | awk '/^Secret ID/ {print $4}')
71-
export NOMAD_TOKEN
72-
echo New super token generated.
73-
echo export NOMAD_TOKEN=$NOMAD_TOKEN
74-
fi
60+
./nomad namespace apply -description "local-nomadlab" nf-nomad
61+
./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_ACCESS_KEY MY_ACCESS_KEY=TheAccessKey
62+
./nomad var put -namespace=nf-nomad secrets/nf-nomad/MY_SECRET_KEY MY_SECRET_KEY=TheSecretKey

‎validation/stop-nomad.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
./nomad system gc
44
sleep 1
55
df -h --output=target | grep nf-task | xargs sudo umount
6-
pkill -9 nomad
6+
kill $(ps aux | grep '../nomad agent' | awk '{print $2}')
77
sleep 1
88
rm -rf nomad_temp
9-
rm ./nomad

0 commit comments

Comments
 (0)