Skip to content

Commit 8ab052d

Browse files
authored
Merge pull request #6732 from zalando-incubator/admission-controller-psp
Implement PSP in admission-controller
2 parents b5ed7e2 + 3f4bdd9 commit 8ab052d

File tree

9 files changed

+125
-39
lines changed

9 files changed

+125
-39
lines changed

cluster/config-defaults.yaml

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,7 @@ teapot_admission_controller_namespace_delete_protection_enabled: "false"
555555
teapot_admission_controller_resolve_vanity_images: "true"
556556

557557
{{if eq .Cluster.Environment "e2e"}}
558-
teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota|limitrange)-.*)$"
558+
teapot_admission_controller_ignore_namespaces: "^kube-system|((downward-api|kubectl|projected|statefulset|pod-network|scope-selectors|resourcequota|limitrange|sysctl|node-tests|e2e-kubelet-etc-hosts)-.*)$"
559559
teapot_admission_controller_crd_ensure_no_resources_on_delete: "false"
560560
{{else}}
561561
teapot_admission_controller_ignore_namespaces: "^kube-system$"
@@ -611,6 +611,21 @@ teapot_admission_controller_configmap_deletion_protection_enabled: "false"
611611
teapot_admission_controller_configmap_deletion_protection_enabled: "true"
612612
{{end}}
613613

614+
# Enable and configure Pod Security Policy rules implemented in admission-controller.
615+
teapot_admission_controller_pod_security_policy_enabled: "true"
616+
617+
# comma separated list of service accounts that are allowed to use privileged
618+
# pod security policy rules. Format: `<namespace>_<service-account-name>`
619+
{{ if eq .Cluster.Environment "e2e" }}
620+
teapot_admission_controller_pod_security_policy_privileged_service_accounts: "psp-privileged-zalando_privileged-sa,psp-privileged-deployment-zalando_privileged-sa"
621+
{{ else }}
622+
teapot_admission_controller_pod_security_policy_privileged_service_accounts: ""
623+
{{ end }}
624+
teapot_admission_controller_pod_security_policy_privileged_allow_privilege_escalation: "false"
625+
626+
# Optionally disable PodSecurityPolicy. Make sure `teapot_admission_controller_pod_security_policy_enabled` is true if this is disabled, otherwise there are no Pod security Policy enforcement in the cluster.
627+
pod_security_policy_enabled: "false"
628+
614629
# Prevent the use of a particular AZ as much as possible
615630
blocked_availability_zone: ""
616631

cluster/manifests/01-admission-control/config.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,33 @@ data:
8282
# This setting enables and disables replacement of vanity images with ECR images during pod admission (during create and update)
8383
pod.vanity-image-replacement.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_resolve_vanity_images }}"
8484

85+
# Pod Security Policy
86+
pod.pod-security-policy.enable: "{{ .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_enabled }}"
87+
88+
# service accounts that need privileged PSP should be defined here as `<namespace>_<sa-name>`
89+
pod.pod-security-policy.privileged-service-accounts.kube-system_kube-proxy: ""
90+
pod.pod-security-policy.privileged-service-accounts.kube-system_skipper-ingress: ""
91+
pod.pod-security-policy.privileged-service-accounts.kube-system_node-monitor: ""
92+
pod.pod-security-policy.privileged-service-accounts.kube-system_nvidia: ""
93+
pod.pod-security-policy.privileged-service-accounts.kube-system_audittrail-adapter: ""
94+
pod.pod-security-policy.privileged-service-accounts.kube-system_kube-aws-iam-controller: ""
95+
pod.pod-security-policy.privileged-service-accounts.kube-system_kube2iam: ""
96+
pod.pod-security-policy.privileged-service-accounts.kube-system_ebs-csi-node-sa: ""
97+
pod.pod-security-policy.privileged-service-accounts.kube-system_flannel: ""
98+
pod.pod-security-policy.privileged-service-accounts.kube-system_etcd-backup: ""
99+
pod.pod-security-policy.privileged-service-accounts.kube-system_coredns: ""
100+
pod.pod-security-policy.privileged-service-accounts.kube-system_efs-provisioner: ""
101+
pod.pod-security-policy.privileged-service-accounts.visibility_logging-agent: ""
102+
{{- range $sa := split .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_privileged_service_accounts "," }}
103+
pod.pod-security-policy.privileged-service-accounts.{{ $sa }}: ""
104+
{{- end}}
105+
106+
{{- range $sysctl := split .Cluster.ConfigItems.allowed_unsafe_sysctls "," }}
107+
pod.pod-security-policy.allowed-unsafe-sysctls.{{ $sysctl }}: ""
108+
{{- end}}
109+
110+
pod.pod-security-policy.allow-privilege-escalation: "{{ .Cluster.ConfigItems.teapot_admission_controller_pod_security_policy_privileged_allow_privilege_escalation }}"
111+
85112
deployment.default.rolling-update-max-surge: "{{ .Cluster.ConfigItems.teapot_admission_controller_deployment_default_max_surge }}"
86113
deployment.default.rolling-update-max-unavailable: "{{ .Cluster.ConfigItems.teapot_admission_controller_deployment_default_max_unavailable }}"
87114

cluster/manifests/deletions.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,3 +271,9 @@ post_apply:
271271
kind: HorizontalPodAutoscaler
272272
namespace: kube-system
273273
{{ end }}
274+
{{- if ne .Cluster.ConfigItems.pod_security_policy_enabled "true" }}
275+
- kind: PodSecurityPolicy
276+
name: privileged
277+
- kind: PodSecurityPolicy
278+
name: restricted
279+
{{- end }}

cluster/manifests/psp/pod_security_policy.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{{- if eq .Cluster.ConfigItems.pod_security_policy_enabled "true" }}
12
apiVersion: policy/v1beta1
23
kind: PodSecurityPolicy
34
metadata:
@@ -87,3 +88,4 @@ spec:
8788
- SETUID
8889
- SYS_CHROOT
8990
- SYS_NICE
91+
{{- end }}

cluster/node-pools/master-default/userdata.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,14 +120,14 @@ write_files:
120120
- --allow-privileged=true
121121
- --service-cluster-ip-range=10.5.0.0/16
122122
- --secure-port=443
123-
- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ExtendedResourceToleration,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection,PodSecurityPolicy,Priority,NodeRestriction{{if eq .Cluster.ConfigItems.event_rate_limit_enable "true"}},EventRateLimit{{end}}
123+
- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ExtendedResourceToleration,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,StorageObjectInUseProtection,{{ if eq .Cluster.ConfigItems.pod_security_policy_enabled "true" }}PodSecurityPolicy,{{end}}Priority,NodeRestriction{{if eq .Cluster.ConfigItems.event_rate_limit_enable "true"}},EventRateLimit{{end}}
124124
{{- if eq .Cluster.ConfigItems.event_rate_limit_enable "true"}}
125125
# This file specifies the EventRateLimit admission plugin's configuration
126126
- --admission-control-config-file=/etc/kubernetes/config/admission-config.yaml
127127
{{- end }}
128128
- --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem
129129
- --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
130-
- --runtime-config=policy/v1beta1=true,authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true,autoscaling/v2beta2={{ .Cluster.ConfigItems.autoscaling_v2beta2_enabled }},autoscaling/v2beta1={{ .Cluster.ConfigItems.autoscaling_v2beta1_enabled }},batch/v1beta1={{ .Cluster.ConfigItems.batch_v1beta1_enabled }}
130+
- --runtime-config={{ if eq .Cluster.ConfigItems.pod_security_policy_enabled "true" }}policy/v1beta1=true,{{end}}authorization.k8s.io/v1beta1=true,scheduling.k8s.io/v1alpha1=true,admissionregistration.k8s.io/v1beta1=true,autoscaling/v2beta2={{ .Cluster.ConfigItems.autoscaling_v2beta2_enabled }},autoscaling/v2beta1={{ .Cluster.ConfigItems.autoscaling_v2beta1_enabled }},batch/v1beta1={{ .Cluster.ConfigItems.batch_v1beta1_enabled }}
131131
- --authentication-token-webhook-config-file=/etc/kubernetes/config/authn.yaml
132132
- --authentication-token-webhook-cache-ttl=10s
133133
- --cloud-provider=aws
@@ -205,7 +205,7 @@ write_files:
205205
limits:
206206
memory: {{ .Values.InstanceInfo.MemoryFraction (parseInt64 .Cluster.ConfigItems.apiserver_memory_limit_percent)}}
207207
{{- end }}
208-
- image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-185
208+
- image: 926694233939.dkr.ecr.eu-central-1.amazonaws.com/production_namespace/teapot/admission-controller:master-187
209209
name: admission-controller
210210
lifecycle:
211211
preStop:

test/e2e/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ require (
2424
k8s.io/client-go v0.24.17
2525
k8s.io/kubernetes v0.0.0-00010101000000-000000000000
2626
k8s.io/pod-security-admission v0.0.0
27+
k8s.io/utils v0.0.0-20240102154912-e7106e64919e
2728
)
2829

2930
require (
@@ -222,7 +223,6 @@ require (
222223
k8s.io/metrics v0.24.17 // indirect
223224
k8s.io/mount-utils v0.24.17 // indirect
224225
k8s.io/sample-apiserver v0.0.0 // indirect
225-
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
226226
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect
227227
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
228228
sigs.k8s.io/kustomize/api v0.11.4 // indirect

test/e2e/go.sum

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1315,8 +1315,9 @@ k8s.io/sample-apiserver v0.24.17 h1:l4HFFu2BJXgIPs7yzYZ3lffragka3/TNzSf+krqC+V0=
13151315
k8s.io/sample-apiserver v0.24.17/go.mod h1:0TJRSfFu6UzS5N6EVtMIlJlPa3vkiO2QW8p9utpLAfk=
13161316
k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
13171317
k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
1318-
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
13191318
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
1319+
k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
1320+
k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
13201321
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
13211322
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
13221323
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

test/e2e/psp.go

Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -32,49 +32,63 @@ import (
3232
)
3333

3434
var _ = describe("PSP use", func() {
35-
privilegedRole := "privileged-psp"
3635
privilegedSA := "privileged-sa"
3736
f := framework.NewDefaultFramework("psp")
37+
f.SkipNamespaceCreation = true
3838
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
3939
var cs kubernetes.Interface
4040

4141
BeforeEach(func() {
4242
cs = f.ClientSet
43-
saObj := createServiceAccount(f.Namespace.Name, privilegedSA)
44-
_, err := cs.CoreV1().ServiceAccounts(f.Namespace.Name).Create(context.TODO(), saObj, metav1.CreateOptions{})
43+
})
44+
45+
It("Should not create a privileged POD if restricted SA [PSP] [Zalando]", func() {
46+
defaultSA := "default"
47+
ns := "psp-restricted-zalando"
48+
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
49+
ObjectMeta: metav1.ObjectMeta{
50+
Name: ns,
51+
},
52+
}, metav1.CreateOptions{})
4553
Expect(err).NotTo(HaveOccurred())
4654

47-
_, err = cs.RbacV1().RoleBindings(f.Namespace.Name).Create(context.TODO(), createRBACRoleBindingSA(privilegedRole, f.Namespace.Name, privilegedSA), metav1.CreateOptions{})
55+
// create SA
56+
saObj := createServiceAccount(ns, privilegedSA)
57+
_, err = cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), saObj, metav1.CreateOptions{})
4858
Expect(err).NotTo(HaveOccurred())
49-
})
5059

51-
// TODO: We have to have a solution to get an unprivileged
52-
// User to check this, if not it would always create a
53-
// privileged POD for an unprivileged serviceAccount.
54-
// --
55-
// It("Should not create a POD that use privileged PSP [PSP] [Zalando]", func() {
56-
// defaultSA := "default"
57-
// ns := f.Namespace.Name
58-
// label := map[string]string{
59-
// "app": "psp",
60-
// }
61-
// msg := fmt.Sprintf("Creating a privileged POD as %s", defaultSA)
62-
// By(msg)
63-
// route := fmt.Sprintf(`* -> inlineContent("%s") -> <shunt>`, "OK")
64-
// pod := createSkipperPodWithHostNetwork("", ns, defaultSA, route, label, 80)
65-
// defer func() {
66-
// By(msg)
67-
// defer GinkgoRecover()
68-
// err := cs.CoreV1().Pods(ns).Delete(pod.Name, metav1.NewDeleteOptions(0))
69-
// Expect(err).To(HaveOccurred())
70-
// }()
71-
// _, err := cs.CoreV1().Pods(ns).Create(pod)
72-
// Expect(err).To(HaveOccurred())
73-
// framework.ExpectNoError(f.WaitForPodRunning(pod.Name))
74-
// })
60+
label := map[string]string{
61+
"app": "psp",
62+
}
63+
msg := fmt.Sprintf("Creating a privileged POD as %s", defaultSA)
64+
By(msg)
65+
route := fmt.Sprintf(`* -> inlineContent("%s") -> <shunt>`, "OK")
66+
pod := createSkipperPodWithHostNetwork("", ns, defaultSA, route, label, 80)
67+
defer func() {
68+
By(msg)
69+
defer GinkgoRecover()
70+
71+
err = cs.CoreV1().Namespaces().Delete(context.TODO(), ns, metav1.DeleteOptions{})
72+
Expect(err).NotTo(HaveOccurred())
73+
}()
74+
_, err = cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
75+
Expect(err).To(HaveOccurred())
76+
})
7577

7678
It("Should create a POD that use privileged PSP [PSP] [Zalando]", func() {
77-
ns := f.Namespace.Name
79+
ns := "psp-privileged-zalando"
80+
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
81+
ObjectMeta: metav1.ObjectMeta{
82+
Name: ns,
83+
},
84+
}, metav1.CreateOptions{})
85+
Expect(err).NotTo(HaveOccurred())
86+
87+
// create SA
88+
saObj := createServiceAccount(ns, privilegedSA)
89+
_, err = cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), saObj, metav1.CreateOptions{})
90+
Expect(err).NotTo(HaveOccurred())
91+
7892
label := map[string]string{
7993
"app": "psp",
8094
}
@@ -89,16 +103,31 @@ var _ = describe("PSP use", func() {
89103
defer GinkgoRecover()
90104
err := cs.CoreV1().Pods(ns).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
91105
Expect(err).NotTo(HaveOccurred())
106+
107+
err = cs.CoreV1().Namespaces().Delete(context.TODO(), ns, metav1.DeleteOptions{})
108+
Expect(err).NotTo(HaveOccurred())
92109
}()
93110

94-
_, err := cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
111+
_, err = cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{})
95112
Expect(err).NotTo(HaveOccurred())
96113

97114
framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace))
98115
})
99116

100117
It("Should create a POD that use privileged PSP via deployment [PSP] [Zalando]", func() {
101-
ns := f.Namespace.Name
118+
ns := "psp-privileged-deployment-zalando"
119+
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
120+
ObjectMeta: metav1.ObjectMeta{
121+
Name: ns,
122+
},
123+
}, metav1.CreateOptions{})
124+
Expect(err).NotTo(HaveOccurred())
125+
126+
// create SA
127+
saObj := createServiceAccount(ns, privilegedSA)
128+
_, err = cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), saObj, metav1.CreateOptions{})
129+
Expect(err).NotTo(HaveOccurred())
130+
102131
label := map[string]string{
103132
"app": "psp",
104133
}
@@ -117,6 +146,9 @@ var _ = describe("PSP use", func() {
117146
defer GinkgoRecover()
118147
err := cs.AppsV1().Deployments(ns).Delete(context.TODO(), d.Name, metav1.DeleteOptions{})
119148
Expect(err).NotTo(HaveOccurred())
149+
150+
err = cs.CoreV1().Namespaces().Delete(context.TODO(), ns, metav1.DeleteOptions{})
151+
Expect(err).NotTo(HaveOccurred())
120152
}()
121153

122154
deploy, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), d, metav1.CreateOptions{})
@@ -134,7 +166,7 @@ var _ = describe("PSP use", func() {
134166
Expect(err).NotTo(HaveOccurred())
135167
By(fmt.Sprintf("Got rs: %s, from deployment: %s", rs.Name, deploy.Name))
136168

137-
pods, err := e2epod.PodsCreatedByLabel(f.ClientSet, f.Namespace.Name, rs.Name, replicas, labelSelector)
169+
pods, err := e2epod.PodsCreatedByLabel(f.ClientSet, ns, rs.Name, replicas, labelSelector)
138170
Expect(err).NotTo(HaveOccurred())
139171
By(fmt.Sprintf("Ensuring each pod is running for rs: %s, pod: %s", rs.Name, pods.Items[0].Name))
140172
// Wait for the pods to enter the running state. Waiting loops until the pods

test/e2e/util.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"k8s.io/kubernetes/test/e2e/framework/config"
2828
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
2929
testutil "k8s.io/kubernetes/test/utils"
30+
"k8s.io/utils/ptr"
3031

3132
. "github.com/onsi/ginkgo"
3233
. "github.com/onsi/gomega"
@@ -251,6 +252,7 @@ func createSkipperPodWithHostNetwork(nameprefix, namespace, serviceAccount, rout
251252
pod := createSkipperPod(nameprefix, namespace, route, labels, port)
252253
pod.Spec.HostNetwork = true
253254
pod.Spec.ServiceAccountName = serviceAccount
255+
pod.Spec.TerminationGracePeriodSeconds = ptr.To(int64(0))
254256
pod.Spec.Containers[0].Ports[0].HostPort = int32(port)
255257
return pod
256258
}
@@ -272,6 +274,7 @@ func createSkipperPod(nameprefix, namespace, route string, labels map[string]str
272274

273275
func createSkipperPodSpec(route string, port int32) corev1.PodSpec {
274276
return corev1.PodSpec{
277+
TerminationGracePeriodSeconds: ptr.To(int64(0)),
275278
Containers: []corev1.Container{
276279
{
277280
Name: "skipper",

0 commit comments

Comments
 (0)