e2e: support admissionapi.LevelRestricted in test/e2e/framwork/pod

CreatePod and MakePod only accepted an `isPrivileged` boolean, which made it
impossible to write tests using those helpers which work in a default
framework.Framework, because the default there is LevelRestricted.

The simple boolean gets replaced with admissionapi.Level. Passing
LevelRestricted does the same as calling e2epod.MixinRestrictedPodSecurity.

Instead of explicitly passing a constant to these modified helpers, most tests
get updated to pass f.NamespacePodSecurityLevel. This has the advantage
that if that level gets lowered in the future, tests only need to be updated in
one place.

In some cases, helpers taking client+namespace+timeouts parameters get replaced
with passing the Framework instance to get access to
f.NamespacePodSecurityEnforceLevel. These helpers don't need separate
parameters because in practice all they ever used where the values from the
Framework instance.

Author: pohly

test/e2e/dra/dra.go

@@ -643,7 +643,7 @@ func (b *builder) parameters(kv ...string) *v1.ConfigMap {
 // makePod returns a simple pod with no resource claims.
 // The pod prints its env and waits.
 func (b *builder) pod() *v1.Pod {
-	pod := e2epod.MakePod(b.f.Namespace.Name, nil, nil, false, "env && sleep 100000")
+	pod := e2epod.MakePod(b.f.Namespace.Name, nil, nil, b.f.NamespacePodSecurityLevel, "env && sleep 100000")
 	pod.Labels = make(map[string]string)
 	pod.Spec.RestartPolicy = v1.RestartPolicyNever
 	// Let kubelet kill the pods quickly. Setting

test/e2e/framework/deployment/fixtures.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	testutils "k8s.io/kubernetes/test/utils"
+	admissionapi "k8s.io/pod-security-admission/api"
 )
 
 // UpdateDeploymentWithRetries updates the specified deployment with retries.
@@ -71,8 +72,8 @@ func NewDeployment(deploymentName string, replicas int32, podLabels map[string]s
 }
 
 // CreateDeployment creates a deployment.
-func CreateDeployment(ctx context.Context, client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*appsv1.Deployment, error) {
-	deploymentSpec := testDeployment(replicas, podLabels, nodeSelector, namespace, pvclaims, false, command)
+func CreateDeployment(ctx context.Context, client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, securityLevel admissionapi.Level, command string) (*appsv1.Deployment, error) {
+	deploymentSpec := testDeployment(replicas, podLabels, nodeSelector, namespace, pvclaims, securityLevel, command)
 	deployment, err := client.AppsV1().Deployments(namespace).Create(ctx, deploymentSpec, metav1.CreateOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("deployment %q Create API error: %w", deploymentSpec.Name, err)
@@ -175,7 +176,7 @@ func (o replicaSetsByCreationTimestamp) Less(i, j int) bool {
 
 // testDeployment creates a deployment definition based on the namespace. The deployment references the PVC's
 // name.  A slice of BASH commands can be supplied as args to be run by the pod
-func testDeployment(replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *appsv1.Deployment {
+func testDeployment(replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, securityLevel admissionapi.Level, command string) *appsv1.Deployment {
 	if len(command) == 0 {
 		command = "trap exit TERM; while true; do sleep 1; done"
 	}
@@ -202,7 +203,7 @@ func testDeployment(replicas int32, podLabels map[string]string, nodeSelector ma
 							Name:            "write-pod",
 							Image:           e2epod.GetDefaultTestImage(),
 							Command:         e2epod.GenerateScriptCmd(command),
-							SecurityContext: e2epod.GenerateContainerSecurityContext(isPrivileged),
+							SecurityContext: e2epod.GenerateContainerSecurityContext(securityLevel),
 						},
 					},
 					RestartPolicy: v1.RestartPolicyAlways,

test/e2e/framework/pod/create.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	clientset "k8s.io/client-go/kubernetes"
 	imageutils "k8s.io/kubernetes/test/utils/image"
+	admissionapi "k8s.io/pod-security-admission/api"
 )
 
 const (
@@ -40,7 +41,7 @@ type Config struct {
 	PVCs                   []*v1.PersistentVolumeClaim
 	PVCsReadOnly           bool
 	InlineVolumeSources    []*v1.VolumeSource
-	IsPrivileged           bool
+	SecurityLevel          admissionapi.Level
 	Command                string
 	HostIPC                bool
 	HostPID                bool
@@ -52,8 +53,8 @@ type Config struct {
 }
 
 // CreateUnschedulablePod with given claims based on node selector
-func CreateUnschedulablePod(ctx context.Context, client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) (*v1.Pod, error) {
-	pod := MakePod(namespace, nodeSelector, pvclaims, isPrivileged, command)
+func CreateUnschedulablePod(ctx context.Context, client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, securityLevel admissionapi.Level, command string) (*v1.Pod, error) {
+	pod := MakePod(namespace, nodeSelector, pvclaims, securityLevel, command)
 	pod, err := client.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("pod Create API error: %w", err)
@@ -73,12 +74,12 @@ func CreateUnschedulablePod(ctx context.Context, client clientset.Interface, nam
 
 // CreateClientPod defines and creates a pod with a mounted PV. Pod runs infinite loop until killed.
 func CreateClientPod(ctx context.Context, c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
-	return CreatePod(ctx, c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")
+	return CreatePod(ctx, c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, admissionapi.LevelPrivileged, "")
 }
 
 // CreatePod with given claims based on node selector
-func CreatePod(ctx context.Context, client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) (*v1.Pod, error) {
-	pod := MakePod(namespace, nodeSelector, pvclaims, isPrivileged, command)
+func CreatePod(ctx context.Context, client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, securityLevel admissionapi.Level, command string) (*v1.Pod, error) {
+	pod := MakePod(namespace, nodeSelector, pvclaims, securityLevel, command)
 	pod, err := client.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("pod Create API error: %w", err)
@@ -128,7 +129,7 @@ func CreateSecPodWithNodeSelection(ctx context.Context, client clientset.Interfa
 
 // MakePod returns a pod definition based on the namespace. The pod references the PVC's
 // name.  A slice of BASH commands can be supplied as args to be run by the pod
-func MakePod(ns string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *v1.Pod {
+func MakePod(ns string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, securityLevel admissionapi.Level, command string) *v1.Pod {
 	if len(command) == 0 {
 		command = "trap exit TERM; while true; do sleep 1; done"
 	}
@@ -147,7 +148,7 @@ func MakePod(ns string, nodeSelector map[string]string, pvclaims []*v1.Persisten
 					Name:            "write-pod",
 					Image:           GetDefaultTestImage(),
 					Command:         GenerateScriptCmd(command),
-					SecurityContext: GenerateContainerSecurityContext(isPrivileged),
+					SecurityContext: GenerateContainerSecurityContext(securityLevel),
 				},
 			},
 			RestartPolicy: v1.RestartPolicyOnFailure,
@@ -157,6 +158,10 @@ func MakePod(ns string, nodeSelector map[string]string, pvclaims []*v1.Persisten
 	if nodeSelector != nil {
 		podSpec.Spec.NodeSelector = nodeSelector
 	}
+	if securityLevel == admissionapi.LevelRestricted {
+		podSpec = MustMixinRestrictedPodSecurity(podSpec)
+	}
+
 	return podSpec
 }
 
@@ -196,6 +201,10 @@ func MakePodSpec(podConfig *Config) *v1.PodSpec {
 	if podConfig.ImageID != imageutils.None {
 		image = podConfig.ImageID
 	}
+	securityLevel := podConfig.SecurityLevel
+	if securityLevel == "" {
+		securityLevel = admissionapi.LevelBaseline
+	}
 	podSpec := &v1.PodSpec{
 		HostIPC:         podConfig.HostIPC,
 		HostPID:         podConfig.HostPID,
@@ -205,7 +214,7 @@ func MakePodSpec(podConfig *Config) *v1.PodSpec {
 				Name:            "write-pod",
 				Image:           GetTestImage(image),
 				Command:         GenerateScriptCmd(podConfig.Command),
-				SecurityContext: GenerateContainerSecurityContext(podConfig.IsPrivileged),
+				SecurityContext: GenerateContainerSecurityContext(securityLevel),
 			},
 		},
 		RestartPolicy: v1.RestartPolicyOnFailure,

test/e2e/framework/pod/utils.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 
+	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 
 	v1 "k8s.io/api/core/v1"
@@ -111,12 +112,25 @@ func GeneratePodSecurityContext(fsGroup *int64, seLinuxOptions *v1.SELinuxOption
 // GenerateContainerSecurityContext generates the corresponding container security context with the given inputs
 // If the Node OS is windows, currently we will ignore the inputs and return nil.
 // TODO: Will modify it after windows has its own security context
-func GenerateContainerSecurityContext(privileged bool) *v1.SecurityContext {
+func GenerateContainerSecurityContext(level psaapi.Level) *v1.SecurityContext {
 	if NodeOSDistroIs("windows") {
 		return nil
 	}
-	return &v1.SecurityContext{
-		Privileged: &privileged,
+
+	switch level {
+	case psaapi.LevelBaseline:
+		return &v1.SecurityContext{
+			Privileged: pointer.Bool(false),
+		}
+	case psaapi.LevelPrivileged:
+		return &v1.SecurityContext{
+			Privileged: pointer.Bool(true),
+		}
+	case psaapi.LevelRestricted:
+		return GetRestrictedContainerSecurityContext()
+	default:
+		ginkgo.Fail(fmt.Sprintf("unknown k8s.io/pod-security-admission/policy.Level %q", level))
+		panic("not reached")
 	}
 }
 

test/e2e/framework/volume/fixtures.go

@@ -59,6 +59,7 @@ import (
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
 	imageutils "k8s.io/kubernetes/test/utils/image"
+	admissionapi "k8s.io/pod-security-admission/api"
 	uexec "k8s.io/utils/exec"
 
 	"github.com/onsi/ginkgo/v2"
@@ -398,8 +399,9 @@ func runVolumeTesterPod(ctx context.Context, client clientset.Interface, timeout
 	When SELinux is enabled on the host, client-pod can not read the content, with permission denied.
 	Invoking client-pod as privileged, so that it can access the volume content, even when SELinux is enabled on the host.
 	*/
-	if config.Prefix == "hostpathsymlink" || config.Prefix == "hostpath" {
-		privileged = true
+	securityLevel := admissionapi.LevelBaseline // TODO (#118184): also support LevelRestricted
+	if privileged || config.Prefix == "hostpathsymlink" || config.Prefix == "hostpath" {
+		securityLevel = admissionapi.LevelPrivileged
 	}
 	command = "while true ; do sleep 2; done "
 	seLinuxOptions := &v1.SELinuxOptions{Level: "s0:c0,c1"}
@@ -443,9 +445,9 @@ func runVolumeTesterPod(ctx context.Context, client clientset.Interface, timeout
 		// a privileged container, so we don't go privileged for block volumes.
 		// https://github.com/moby/moby/issues/35991
 		if privileged && test.Mode == v1.PersistentVolumeBlock {
-			privileged = false
+			securityLevel = admissionapi.LevelBaseline
 		}
-		clientPod.Spec.Containers[0].SecurityContext = e2epod.GenerateContainerSecurityContext(privileged)
+		clientPod.Spec.Containers[0].SecurityContext = e2epod.GenerateContainerSecurityContext(securityLevel)
 
 		if test.Mode == v1.PersistentVolumeBlock {
 			clientPod.Spec.Containers[0].VolumeDevices = append(clientPod.Spec.Containers[0].VolumeDevices, v1.VolumeDevice{

test/e2e/storage/flexvolume_mounted_volume_resize.go

@@ -141,7 +141,7 @@ var _ = utils.SIGDescribe("[Feature:Flexvolumes] Mounted flexvolume expand[Slow]
 		framework.ExpectEqual(len(pvs), 1)
 
 		ginkgo.By("Creating a deployment with the provisioned volume")
-		deployment, err := e2edeployment.CreateDeployment(ctx, c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, "")
+		deployment, err := e2edeployment.CreateDeployment(ctx, c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, admissionapi.LevelRestricted, "")
 		framework.ExpectNoError(err, "Failed creating deployment %v", err)
 		ginkgo.DeferCleanup(c.AppsV1().Deployments(ns).Delete, deployment.Name, metav1.DeleteOptions{})
 

test/e2e/storage/mounted_volume_resize.go

@@ -111,7 +111,7 @@ var _ = utils.SIGDescribe("Mounted volume expand [Feature:StorageProvider]", fun
 		// Keeping pod on same node reproduces the scenario that volume might already be mounted when resize is attempted.
 		// We should consider adding a unit test that exercises this better.
 		ginkgo.By("Creating a deployment with selected PVC")
-		deployment, err := e2edeployment.CreateDeployment(ctx, c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, "")
+		deployment, err := e2edeployment.CreateDeployment(ctx, c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, admissionapi.LevelRestricted, "")
 		framework.ExpectNoError(err, "Failed creating deployment %v", err)
 		ginkgo.DeferCleanup(c.AppsV1().Deployments(ns).Delete, deployment.Name, metav1.DeleteOptions{})
 

test/e2e/storage/nfs_persistent_volume-disruptive.go

@@ -178,7 +178,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() {
 			framework.ExpectNoError(e2epv.WaitOnPVandPVC(ctx, c, f.Timeouts, ns, pv2, pvc2))
 
 			ginkgo.By("Attaching both PVC's to a single pod")
-			clientPod, err = e2epod.CreatePod(ctx, c, ns, nil, []*v1.PersistentVolumeClaim{pvc1, pvc2}, true, "")
+			clientPod, err = e2epod.CreatePod(ctx, c, ns, nil, []*v1.PersistentVolumeClaim{pvc1, pvc2}, f.NamespacePodSecurityLevel, "")
 			framework.ExpectNoError(err)
 		})
 
@@ -301,7 +301,7 @@ func initTestCase(ctx context.Context, f *framework.Framework, c clientset.Inter
 		}
 	}()
 	framework.ExpectNoError(err)
-	pod := e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")
+	pod := e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, f.NamespacePodSecurityLevel, "")
 	pod.Spec.NodeName = nodeName
 	framework.Logf("Creating NFS client pod.")
 	pod, err = c.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})

test/e2e/storage/persistent_volumes.go

@@ -285,7 +285,7 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() {
 			// (and test) succeed.
 			ginkgo.It("should test that a PV becomes Available and is clean after the PVC is deleted.", func(ctx context.Context) {
 				ginkgo.By("Writing to the volume.")
-				pod := e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "touch /mnt/volume1/SUCCESS && (id -G | grep -E '\\b777\\b')")
+				pod := e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, f.NamespacePodSecurityLevel, "touch /mnt/volume1/SUCCESS && (id -G | grep -E '\\b777\\b')")
 				pod, err = c.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, c, pod.Name, ns, f.Timeouts.PodStart))
@@ -303,7 +303,7 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() {
 				// If a file is detected in /mnt, fail the pod and do not restart it.
 				ginkgo.By("Verifying the mount has been cleaned.")
 				mount := pod.Spec.Containers[0].VolumeMounts[0].MountPath
-				pod = e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, fmt.Sprintf("[ $(ls -A %s | wc -l) -eq 0 ] && exit 0 || exit 1", mount))
+				pod = e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, f.NamespacePodSecurityLevel, fmt.Sprintf("[ $(ls -A %s | wc -l) -eq 0 ] && exit 0 || exit 1", mount))
 				pod, err = c.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, c, pod.Name, ns, f.Timeouts.PodStart))
@@ -447,7 +447,7 @@ func makeStatefulSetWithPVCs(ns, cmd string, mounts []v1.VolumeMount, claims []v
 //	Has been shown to be necessary using Go 1.7.
 func createWaitAndDeletePod(ctx context.Context, c clientset.Interface, t *framework.TimeoutContext, ns string, pvc *v1.PersistentVolumeClaim, command string) (err error) {
 	framework.Logf("Creating nfs test pod")
-	pod := e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, command)
+	pod := e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, admissionapi.LevelPrivileged, command)
 	runPod, err := c.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
 	if err != nil {
 		return fmt.Errorf("pod Create API error: %w", err)

test/e2e/storage/pvc_protection.go

@@ -95,7 +95,7 @@ var _ = utils.SIGDescribe("PVC Protection", func() {
 
 		ginkgo.By("Creating a Pod that becomes Running and therefore is actively using the PVC")
 		pvcClaims := []*v1.PersistentVolumeClaim{pvc}
-		pod, err = e2epod.CreatePod(ctx, client, nameSpace, nil, pvcClaims, false, "")
+		pod, err = e2epod.CreatePod(ctx, client, nameSpace, nil, pvcClaims, f.NamespacePodSecurityLevel, "")
 		framework.ExpectNoError(err, "While creating pod that uses the PVC or waiting for the Pod to become Running")
 
 		ginkgo.By("Waiting for PVC to become Bound")
@@ -156,7 +156,7 @@ var _ = utils.SIGDescribe("PVC Protection", func() {
 		framework.ExpectNotEqual(pvc.ObjectMeta.DeletionTimestamp, nil)
 
 		ginkgo.By("Creating second Pod whose scheduling fails because it uses a PVC that is being deleted")
-		secondPod, err2 := e2epod.CreateUnschedulablePod(ctx, client, nameSpace, nil, []*v1.PersistentVolumeClaim{pvc}, false, "")
+		secondPod, err2 := e2epod.CreateUnschedulablePod(ctx, client, nameSpace, nil, []*v1.PersistentVolumeClaim{pvc}, f.NamespacePodSecurityLevel, "")
 		framework.ExpectNoError(err2, "While creating second pod that uses a PVC that is being deleted and that is Unschedulable")
 
 		ginkgo.By("Deleting the second pod that uses the PVC that is being deleted")