Skip to content

Refactored buildkit rootless privileges #147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mskwierczynski982 merged 2 commits into from
May 15, 2025
Merged

Refactored buildkit rootless privileges #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a761e21
Refactored buildkit rootless privileges
mskwierczynski982 May 14, 2025
a575042
acr name from var
mskwierczynski982 May 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/ado-build-agents/Chart.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
apiVersion: v2
description: A Helm chart with Keda scalable Azure Devops build agent for Kubernetes
name: charts-ado-build-agents
version: 1.1.0
version: 1.2.0
appVersion: "1.0"
53 changes: 33 additions & 20 deletions charts/ado-build-agents/templates/scalejob-staging.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,24 @@ metadata:
namespace: {{ .Values.aks.namespace }}
spec:
jobTargetRef:
ttlSecondsAfterFinished: 30
backoffLimit: 0
template:
metadata:
annotations:
io.kubernetes.cri-o.userns-mode: "auto:size=65536"
container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
labels:
azure.workload.identity/use: "true"
spec:
shareProcessNamespace: true
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
restartPolicy: Never
terminationGracePeriodSeconds: 3600
volumes:
- name: buildkitd-certs
emptyDir: {}
- name: buildkitd-workspace
emptyDir: {}
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
nodeSelector:
pool: "{{ .Values.aks.agentPool }}"
{{- if .Values.aks.sysbox.enabled }}
Expand Down Expand Up @@ -51,7 +53,12 @@ spec:
image: {{ .Values.devops.ACR_NAME }}/{{ .Values.staging.stagingImageName }}:{{ .Values.staging.stagingImageVersion }}
imagePullPolicy: IfNotPresent
securityContext:
privileged: true
allowPrivilegeEscalation: true
privileged: false
capabilities:
add:
- SYS_PTRACE
- KILL
resources:
requests:
memory: {{ .Values.aks.memoryRequest }}
Expand All @@ -78,23 +85,17 @@ spec:
mountPath: /certs
readOnly: true
- name: buildkitd
image: cicdcr01weuy01.azurecr.io/dockerhub/moby/buildkit:v0.21.1-rootless
image: {{ .Values.devops.ACR_NAME }}/dockerhub/moby/buildkit:v0.21.1-rootless
imagePullPolicy: IfNotPresent
securityContext:
privileged: true
runAsUser: 1000
runAsGroup: 1000
livenessProbe:
exec:
command:
- buildctl
- debug
- workers
failureThreshold: 3
initialDelaySeconds: 3
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
capabilities:
add:
- SYS_PTRACE
- KILL
seccompProfile:
type: Unconfined
readinessProbe:
exec:
command:
Expand All @@ -109,17 +110,29 @@ spec:
command: ["/bin/sh", "-c"]
args:
- |
trap 'echo "[CMD] Caught SIGTERM. Stopping buildkitd..."; kill -TERM "$BKD_PID"; wait "$BKD_PID"; exit 0' TERM INT;
echo "[CMD] Starting buildkitd...";
rootlesskit buildkitd \
--oci-worker-no-process-sandbox \
--addr tcp://0.0.0.0:1234 \
--addr unix:///run/user/1000/buildkit/buildkitd.sock \
--tlscacert /certs/server/ca.pem \
--tlscert /certs/server/cert.pem \
--tlskey /certs/server/key.pem &
BKD_PID=$!;
wait "$BKD_PID";
echo "[CMD] buildkitd exited. Exiting cleanly...";
attempts=0
max_attempts=30
while ! pgrep start.sh > /dev/null; do
attempts=$((attempts + 1))
if [ $attempts -ge $max_attempts ]; then
echo "[CMD] Failed to find start.sh after $max_attempts attempts. Exiting with status 1."
exit 1
fi
echo "[CMD] Failed to find process start.sh from agent container, retrying..."
sleep 10
done
echo "[CMD] start.sh process found."
while pgrep start.sh > /dev/null; do
sleep 1
done
exit 0
volumeMounts:
- name: buildkitd-certs
Expand Down
53 changes: 33 additions & 20 deletions charts/ado-build-agents/templates/scalejob.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,24 @@ metadata:
namespace: {{ .Values.aks.namespace }}
spec:
jobTargetRef:
ttlSecondsAfterFinished: 30
backoffLimit: 0
template:
metadata:
annotations:
io.kubernetes.cri-o.userns-mode: "auto:size=65536"
container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
labels:
azure.workload.identity/use: "true"
spec:
shareProcessNamespace: true
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
restartPolicy: Never
terminationGracePeriodSeconds: 3600
volumes:
- name: buildkitd-certs
emptyDir: {}
- name: buildkitd-workspace
emptyDir: {}
serviceAccountName: "svc-acc-{{ .Values.buildAgentName }}"
nodeSelector:
pool: "{{ .Values.aks.agentPool }}"
{{- if .Values.aks.sysbox.enabled }}
Expand Down Expand Up @@ -50,7 +52,12 @@ spec:
image: {{ .Values.devops.ACR_NAME }}/{{ .Values.image.imageName }}:{{ .Values.image.version }}
imagePullPolicy: IfNotPresent
securityContext:
privileged: true
allowPrivilegeEscalation: true
privileged: false
capabilities:
add:
- SYS_PTRACE
- KILL
resources:
requests:
memory: {{ .Values.aks.memoryRequest }}
Expand All @@ -77,23 +84,17 @@ spec:
mountPath: /certs
readOnly: true
- name: buildkitd
image: cicdcr01weuy01.azurecr.io/dockerhub/moby/buildkit:v0.21.1-rootless
image: {{ .Values.devops.ACR_NAME }}/dockerhub/moby/buildkit:v0.21.1-rootless
imagePullPolicy: IfNotPresent
securityContext:
privileged: true
runAsUser: 1000
runAsGroup: 1000
livenessProbe:
exec:
command:
- buildctl
- debug
- workers
failureThreshold: 3
initialDelaySeconds: 3
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 1
capabilities:
add:
- SYS_PTRACE
- KILL
seccompProfile:
type: Unconfined
readinessProbe:
exec:
command:
Expand All @@ -108,17 +109,29 @@ spec:
command: ["/bin/sh", "-c"]
args:
- |
trap 'echo "[CMD] Caught SIGTERM. Stopping buildkitd..."; kill -TERM "$BKD_PID"; wait "$BKD_PID"; exit 0' TERM INT;
echo "[CMD] Starting buildkitd...";
rootlesskit buildkitd \
--oci-worker-no-process-sandbox \
--addr tcp://0.0.0.0:1234 \
--addr unix:///run/user/1000/buildkit/buildkitd.sock \
--tlscacert /certs/server/ca.pem \
--tlscert /certs/server/cert.pem \
--tlskey /certs/server/key.pem &
BKD_PID=$!;
wait "$BKD_PID";
echo "[CMD] buildkitd exited. Exiting cleanly...";
attempts=0
max_attempts=30
while ! pgrep start.sh > /dev/null; do
attempts=$((attempts + 1))
if [ $attempts -ge $max_attempts ]; then
echo "[CMD] Failed to find start.sh after $max_attempts attempts. Exiting with status 1."
exit 1
fi
echo "[CMD] Failed to find process start.sh from agent container, retrying..."
sleep 10
done
echo "[CMD] start.sh process found."
while pgrep start.sh > /dev/null; do
sleep 1
done
exit 0
volumeMounts:
- name: buildkitd-certs
Expand Down
Loading