Skip to content

Commit 4685214

Browse files
GlassOfWhiskeyGlassOfWhiskey
committed
Improve StreamFlow on Kubernetes
This commit heavily refactors the StreamFlow Helm chart to simplify its deployment on top of Kubernetes clusters. In addition, this commit adds a `networkPolicy` flag to control the behaviour of CWL `DockerRequirement` objects into Kubernetes `Pod` items. Normally, the CWL `NetworkAccess` requirement is enforced through Kubernetes `NetworkPolicy` objects. However, `NetworkPolicy` objects regulate the network security inside a cluster, and giving the StreamFlow `Pod` permissions to create/delete them may result in unwanted security flaws. The `networkPolicy` option can be set to `False` to ignore the CWL `NetworkAccess` enforcement in such cases.
1 parent e86d21c commit 4685214

File tree

15 files changed

+354
-51
lines changed

15 files changed

+354
-51
lines changed

‎.github/workflows/ci-tests.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ jobs:
157157
python -m pip install -r docs/requirements.txt
158158
- name: "Build documentation and check for consistency"
159159
env:
160-
CHECKSUM: "b59239241d3529a179df6158271dd00ba7a86e807a37a11ac8e078ad9c377f94"
160+
CHECKSUM: "6fd3864fc99b49ced645b526f9d18ac3355ba30a94cc9a3bcb9286e6430afb28"
161161
run: |
162162
cd docs
163163
HASH="$(make checksum | tail -n1)"

‎helm/chart/Chart.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
apiVersion: v2
22
name: streamflow
3-
description: A Helm chart for StreamFlow
3+
description: A Helm chart for the StreamFlow workflow management system
44
type: application
55
version: 0.2.0
6-
appVersion: latest
6+
appVersion: 0.2.0.dev11

‎helm/chart/templates/_helpers.tpl‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{{/* vim: set filetype=mustache: */}}
22
{{/*
3-
Expand the name of the chart.
3+
Expand the name of the chart
44
*/}}
55
{{- define "streamflow.name" -}}
66
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
@@ -9,7 +9,7 @@ Expand the name of the chart.
99
{{/*
1010
Create a default fully qualified app name.
1111
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
12-
If release name contains chart name it will be used as a full name.
12+
If release name contains chart name it will be used as a full name
1313
*/}}
1414
{{- define "streamflow.fullname" -}}
1515
{{- if .Values.fullnameOverride -}}
@@ -25,12 +25,49 @@ If release name contains chart name it will be used as a full name.
2525
{{- end -}}
2626

2727
{{/*
28-
Create chart name and version as used by the chart label.
28+
Create chart name and version as used by the chart label
2929
*/}}
3030
{{- define "streamflow.chart" -}}
3131
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
3232
{{- end -}}
3333

34+
{{/*
35+
Return the proper StreamFlow image name
36+
*/}}
37+
{{- define "streamflow.image" -}}
38+
{{- $registryName := default .Values.image.registry -}}
39+
{{- $repositoryName := .Values.image.repository -}}
40+
{{- $separator := ":" -}}
41+
{{- $termination := default .Chart.AppVersion .Values.image.tag | toString -}}
42+
43+
{{- if not .Values.image.tag }}
44+
{{- if .Chart }}
45+
{{- $termination = .Chart.AppVersion | toString -}}
46+
{{- end -}}
47+
{{- end -}}
48+
{{- if .Values.image.digest }}
49+
{{- $separator = "@" -}}
50+
{{- $termination = .Values.image.digest | toString -}}
51+
{{- end -}}
52+
{{- if $registryName }}
53+
{{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}}
54+
{{- else -}}
55+
{{- printf "%s%s%s" $repositoryName $separator $termination -}}
56+
{{- end -}}
57+
{{- end -}}
58+
59+
{{/*
60+
Return the proper Docker Image Registry Secret Names evaluating values as templates
61+
*/}}
62+
{{- define "streamflow.imagePullSecrets" -}}
63+
{{- if (not (empty .Values.image.pullSecrets)) -}}
64+
imagePullSecrets:
65+
{{- range .Values.image.pullSecrets | uniq }}
66+
- name: {{ . }}
67+
{{- end }}
68+
{{- end }}
69+
{{- end }}
70+
3471
{{/*
3572
Common labels
3673
*/}}

‎helm/chart/templates/configmap.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: {{ include "streamflow.fullname" . }}
5+
namespace: {{ .Release.Namespace }}
6+
labels:
7+
{{- include "streamflow.labels" . | nindent 4 }}
8+
data:
9+
streamflow.yml: |-
10+
version: v1.0
11+
workflows:
12+
{{ .Values.streamflow.workflow.name | default uuidv4 }}:
13+
type: {{ .Values.streamflow.workflow.type }}
14+
{{- if .Values.streamflow.workflow.bindings }}
15+
{{- with .Values.streamflow.workflow.bindings }}
16+
bindings:
17+
{{- toYaml . | nindent 10 }}
18+
{{- end }}
19+
{{- end }}
20+
config:
21+
{{- if eq .Values.streamflow.workflow.type "cwl" }}
22+
file: {{ required "CWL processfile is mandatory" .Values.streamflow.workflow.cwl.processfile }}
23+
{{- if .Values.streamflow.workflow.cwl.jobfile }}
24+
settings: {{ .Values.streamflow.workflow.cwl.jobfile }}
25+
{{- end }}
26+
docker:
27+
- step: /
28+
deployment:
29+
type: kubernetes
30+
config:
31+
inCluster: true
32+
networkPolicy: {{ .Values.streamflow.workflow.cwl.restrictNetworkAccess }}
33+
{{- end }}
34+
{{- if .Values.streamflow.config }}
35+
{{- toYaml .Values.streamflow.config | nindent 4 }}
36+
{{- end }}

‎helm/chart/templates/job.yaml‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,32 +13,42 @@ spec:
1313
labels:
1414
{{- include "streamflow.selectorLabels" . | nindent 8 }}
1515
spec:
16-
{{- with .Values.imagePullSecrets }}
17-
imagePullSecrets:
18-
{{- toYaml . | nindent 8 }}
19-
{{- end }}
2016
serviceAccountName: {{ include "streamflow.serviceAccountName" . }}
17+
{{- include "streamflow.imagePullSecrets" . | nindent 6 }}
18+
{{- if .Values.podSecurityContext.enabled }}
2119
securityContext:
22-
{{- toYaml .Values.podSecurityContext | nindent 8 }}
20+
{{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
21+
{{- end }}
2322
containers:
24-
- name: {{ .Chart.Name }}
23+
- name: {{ include "streamflow.fullname" . }}
24+
{{- if .Values.containerSecurityContext.enabled }}
2525
securityContext:
26-
{{- toYaml .Values.securityContext | nindent 12 }}
27-
image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
26+
{{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
27+
{{- end }}
28+
image: {{ include "streamflow.image" . }}
29+
{{- if .Values.command }}
30+
command: {{ .Values.command }}
31+
{{- end }}
32+
{{- if .Values.args }}
2833
args: {{ .Values.args }}
34+
{{- end }}
2935
imagePullPolicy: {{ .Values.image.pullPolicy }}
36+
{{- if .Values.resources }}
3037
resources:
3138
{{- toYaml .Values.resources | nindent 12 }}
39+
{{- end }}
40+
volumeMounts:
41+
- name: streamflow-config
42+
mountPath: /streamflow/results/streamflow.yml
43+
subPath: streamflow.yml
44+
{{ if .Values.restartPolicy }}
3245
restartPolicy: {{ .Values.restartPolicy }}
46+
{{- end }}
3347
{{- with .Values.nodeSelector }}
3448
nodeSelector:
3549
{{- toYaml . | nindent 8 }}
3650
{{- end }}
37-
{{- with .Values.affinity }}
38-
affinity:
39-
{{- toYaml . | nindent 8 }}
40-
{{- end }}
41-
{{- with .Values.tolerations }}
51+
{{- with .Values.tolerations }}
4252
tolerations:
4353
{{- toYaml . | nindent 8 }}
44-
{{- end }}
54+
{{- end }}

‎helm/chart/templates/role.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{{- if .Values.rbac.create }}
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: Role
4+
metadata:
5+
name: {{ include "streamflow.fullname" . }}
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{- include "streamflow.labels" . | nindent 4 }}
9+
rules:
10+
- verbs:
11+
- get
12+
- watch
13+
- list
14+
- create
15+
- delete
16+
apiGroups:
17+
- ''
18+
resources:
19+
- pods
20+
- pods/exec
21+
{{- if eq .Values.streamflow.workflow.type "cwl" }}
22+
{{- if .Values.streamflow.workflow.restrictNetworkAccess }}
23+
- verbs:
24+
- get
25+
- list
26+
- create
27+
- delete
28+
apiGroups:
29+
- networking.k8s.io
30+
resources:
31+
- networkpolicies
32+
{{- end }}
33+
{{- end }}
34+
{{- end }}

‎helm/chart/templates/rolebinding.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{{- if .Values.rbac.create }}
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: RoleBinding
4+
metadata:
5+
name: {{ include "streamflow.fullname" . }}
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{- include "streamflow.labels" . | nindent 4 }}
9+
roleRef:
10+
kind: Role
11+
name: {{ include "streamflow.fullname" . }}
12+
apiGroup: rbac.authorization.k8s.io
13+
subjects:
14+
- kind: ServiceAccount
15+
name: {{ include "streamflow.serviceAccountName" . }}
16+
namespace: {{ .Release.Namespace }}
17+
{{- end }}

‎helm/chart/templates/serviceaccount.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ metadata:
99
annotations:
1010
{{- toYaml . | nindent 4 }}
1111
{{- end }}
12+
automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
1213
{{- end -}}

0 commit comments

Comments
 (0)