Skip to content

Commit 87d694a

Browse files
mskwierczynski982mskwierczynski982
authored
Merge pull request #147 from EcovadisCode/buildAgentsRootlessRefactor
Refactored buildkit rootless privileges
2 parents 75c521e + a575042 commit 87d694a

File tree

3 files changed

+67
-41
lines changed

3 files changed

+67
-41
lines changed

‎charts/ado-build-agents/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
apiVersion: v2
22
description: A Helm chart with Keda scalable Azure Devops build agent for Kubernetes
33
name: charts-ado-build-agents
4-
version: 1.1.0
4+
version: 1.2.0
55
appVersion: "1.0"

‎charts/ado-build-agents/templates/scalejob-staging.yaml‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,24 @@ metadata:
66
namespace: {{ .Values.aks.namespace }}
77
spec:
88
jobTargetRef:
9+
ttlSecondsAfterFinished: 30
10+
backoffLimit: 0
911
template:
1012
metadata:
1113
annotations:
12-
io.kubernetes.cri-o.userns-mode: "auto:size=65536"
14+
container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
1315
labels:
1416
azure.workload.identity/use: "true"
1517
spec:
1618
shareProcessNamespace: true
19+
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
1720
restartPolicy: Never
1821
terminationGracePeriodSeconds: 3600
1922
volumes:
2023
- name: buildkitd-certs
2124
emptyDir: {}
2225
- name: buildkitd-workspace
2326
emptyDir: {}
24-
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
2527
nodeSelector:
2628
pool: "{{ .Values.aks.agentPool }}"
2729
{{- if .Values.aks.sysbox.enabled }}
@@ -51,7 +53,12 @@ spec:
5153
image: {{ .Values.devops.ACR_NAME }}/{{ .Values.staging.stagingImageName }}:{{ .Values.staging.stagingImageVersion }}
5254
imagePullPolicy: IfNotPresent
5355
securityContext:
54-
privileged: true
56+
allowPrivilegeEscalation: true
57+
privileged: false
58+
capabilities:
59+
add:
60+
- SYS_PTRACE
61+
- KILL
5562
resources:
5663
requests:
5764
memory: {{ .Values.aks.memoryRequest }}
@@ -78,23 +85,17 @@ spec:
7885
mountPath: /certs
7986
readOnly: true
8087
- name: buildkitd
81-
image: cicdcr01weuy01.azurecr.io/dockerhub/moby/buildkit:v0.21.1-rootless
88+
image: {{ .Values.devops.ACR_NAME }}/dockerhub/moby/buildkit:v0.21.1-rootless
8289
imagePullPolicy: IfNotPresent
8390
securityContext:
84-
privileged: true
8591
runAsUser: 1000
8692
runAsGroup: 1000
87-
livenessProbe:
88-
exec:
89-
command:
90-
- buildctl
91-
- debug
92-
- workers
93-
failureThreshold: 3
94-
initialDelaySeconds: 3
95-
periodSeconds: 5
96-
successThreshold: 1
97-
timeoutSeconds: 1
93+
capabilities:
94+
add:
95+
- SYS_PTRACE
96+
- KILL
97+
seccompProfile:
98+
type: Unconfined
9899
readinessProbe:
99100
exec:
100101
command:
@@ -109,17 +110,29 @@ spec:
109110
command: ["/bin/sh", "-c"]
110111
args:
111112
- |
112-
trap 'echo "[CMD] Caught SIGTERM. Stopping buildkitd..."; kill -TERM "$BKD_PID"; wait "$BKD_PID"; exit 0' TERM INT;
113113
echo "[CMD] Starting buildkitd...";
114114
rootlesskit buildkitd \
115+
--oci-worker-no-process-sandbox \
115116
--addr tcp://0.0.0.0:1234 \
116117
--addr unix:///run/user/1000/buildkit/buildkitd.sock \
117118
--tlscacert /certs/server/ca.pem \
118119
--tlscert /certs/server/cert.pem \
119120
--tlskey /certs/server/key.pem &
120-
BKD_PID=$!;
121-
wait "$BKD_PID";
122-
echo "[CMD] buildkitd exited. Exiting cleanly...";
121+
attempts=0
122+
max_attempts=30
123+
while ! pgrep start.sh > /dev/null; do
124+
attempts=$((attempts + 1))
125+
if [ $attempts -ge $max_attempts ]; then
126+
echo "[CMD] Failed to find start.sh after $max_attempts attempts. Exiting with status 1."
127+
exit 1
128+
fi
129+
echo "[CMD] Failed to find process start.sh from agent container, retrying..."
130+
sleep 10
131+
done
132+
echo "[CMD] start.sh process found."
133+
while pgrep start.sh > /dev/null; do
134+
sleep 1
135+
done
123136
exit 0
124137
volumeMounts:
125138
- name: buildkitd-certs

‎charts/ado-build-agents/templates/scalejob.yaml‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,24 @@ metadata:
55
namespace: {{ .Values.aks.namespace }}
66
spec:
77
jobTargetRef:
8+
ttlSecondsAfterFinished: 30
9+
backoffLimit: 0
810
template:
911
metadata:
1012
annotations:
11-
io.kubernetes.cri-o.userns-mode: "auto:size=65536"
13+
container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
1214
labels:
1315
azure.workload.identity/use: "true"
1416
spec:
1517
shareProcessNamespace: true
18+
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
1619
restartPolicy: Never
1720
terminationGracePeriodSeconds: 3600
1821
volumes:
1922
- name: buildkitd-certs
2023
emptyDir: {}
2124
- name: buildkitd-workspace
2225
emptyDir: {}
23-
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
2426
nodeSelector:
2527
pool: "{{ .Values.aks.agentPool }}"
2628
{{- if .Values.aks.sysbox.enabled }}
@@ -50,7 +52,12 @@ spec:
5052
image: {{ .Values.devops.ACR_NAME }}/{{ .Values.image.imageName }}:{{ .Values.image.version }}
5153
imagePullPolicy: IfNotPresent
5254
securityContext:
53-
privileged: true
55+
allowPrivilegeEscalation: true
56+
privileged: false
57+
capabilities:
58+
add:
59+
- SYS_PTRACE
60+
- KILL
5461
resources:
5562
requests:
5663
memory: {{ .Values.aks.memoryRequest }}
@@ -77,23 +84,17 @@ spec:
7784
mountPath: /certs
7885
readOnly: true
7986
- name: buildkitd
80-
image: cicdcr01weuy01.azurecr.io/dockerhub/moby/buildkit:v0.21.1-rootless
87+
image: {{ .Values.devops.ACR_NAME }}/dockerhub/moby/buildkit:v0.21.1-rootless
8188
imagePullPolicy: IfNotPresent
8289
securityContext:
83-
privileged: true
8490
runAsUser: 1000
8591
runAsGroup: 1000
86-
livenessProbe:
87-
exec:
88-
command:
89-
- buildctl
90-
- debug
91-
- workers
92-
failureThreshold: 3
93-
initialDelaySeconds: 3
94-
periodSeconds: 5
95-
successThreshold: 1
96-
timeoutSeconds: 1
92+
capabilities:
93+
add:
94+
- SYS_PTRACE
95+
- KILL
96+
seccompProfile:
97+
type: Unconfined
9798
readinessProbe:
9899
exec:
99100
command:
@@ -108,17 +109,29 @@ spec:
108109
command: ["/bin/sh", "-c"]
109110
args:
110111
- |
111-
trap 'echo "[CMD] Caught SIGTERM. Stopping buildkitd..."; kill -TERM "$BKD_PID"; wait "$BKD_PID"; exit 0' TERM INT;
112112
echo "[CMD] Starting buildkitd...";
113113
rootlesskit buildkitd \
114+
--oci-worker-no-process-sandbox \
114115
--addr tcp://0.0.0.0:1234 \
115116
--addr unix:///run/user/1000/buildkit/buildkitd.sock \
116117
--tlscacert /certs/server/ca.pem \
117118
--tlscert /certs/server/cert.pem \
118119
--tlskey /certs/server/key.pem &
119-
BKD_PID=$!;
120-
wait "$BKD_PID";
121-
echo "[CMD] buildkitd exited. Exiting cleanly...";
120+
attempts=0
121+
max_attempts=30
122+
while ! pgrep start.sh > /dev/null; do
123+
attempts=$((attempts + 1))
124+
if [ $attempts -ge $max_attempts ]; then
125+
echo "[CMD] Failed to find start.sh after $max_attempts attempts. Exiting with status 1."
126+
exit 1
127+
fi
128+
echo "[CMD] Failed to find process start.sh from agent container, retrying..."
129+
sleep 10
130+
done
131+
echo "[CMD] start.sh process found."
132+
while pgrep start.sh > /dev/null; do
133+
sleep 1
134+
done
122135
exit 0
123136
volumeMounts:
124137
- name: buildkitd-certs

0 commit comments

Comments
 (0)