Added network policy test setup for soak, scale and load and supporting tools

clusterloader2/cmd/clusterloader.go

@@ -49,6 +49,7 @@ import (
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/dns"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/network"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/network-policy"
+	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/network-policy/network-policy-soak"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/probes"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/slos"
 )

clusterloader2/pkg/measurement/common/network-policy/manifests/clusterrole.yaml

@@ -9,3 +9,10 @@ rules:
 - apiGroups: ["networking.k8s.io"]
   resources: ["networkpolicies"]
   verbs: ["get"]
+- apiGroups: ["cilium.io"]
+  resources: ["ciliumclusterwidenetworkpolicies"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["cilium.io"]
+  resources: ["ciliumnetworkpolicies"]
+  verbs: ["get", "list", "watch"]
+

clusterloader2/pkg/measurement/common/network-policy/manifests/dep-test-client-policy-creation.yaml

@@ -34,7 +34,8 @@ spec:
           name: npdelaymetrics
           protocol: TCP
         imagePullPolicy: Always
-        image: gcr.io/k8s-staging-perf-tests/network-policy-enforcement-latency/policy-creation-enforcement-latency:v0.0.1
+        # image: gcr.io/k8s-staging-perf-tests/network-policy-enforcement-latency/policy-creation-enforcement-latency:v0.0.1
+        image: docker.io/sanamsarath/policy-creation-enforcement-latency:v0.0.1
         command:
         - sh
         - -c
@@ -46,6 +47,8 @@ spec:
           -MaxTargets={{.MaxTargets}}
           -MetricsPort={{.MetricsPort}}
           -AllowPolicyName={{.AllowPolicyName}}
+          -np_type={{.NetworkPolicy_Type}}
+          -np_namespace={{.Namespace}} # change this value according to the namespace where the policy is created, verify policy-egress-allow-target-pods.yaml
         resources:
           requests:
             cpu: 200m

clusterloader2/pkg/measurement/common/network-policy/manifests/policy-egress-allow-target-pods.yaml

@@ -1,3 +1,61 @@
+{{ if eq .NetworkPolicy_Type "cnp" }}
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: {{.Name}}
+  namespace: {{.ClientNamespace}} # since this policy is egress, the namespace is the client namespace
+  labels:
+    type: {{.TypeLabelValue}}
+spec:
+  endpointSelector:
+    matchLabels:
+      type: {{.TypeLabelValue}}
+  egress:
+  - toEndpoints:
+    - matchLabels:
+        net-pol-test: {{.TargetLabelValue}}
+{{if .OnlyTargetNamespace}}
+        k8s:io.kubernetes.pod.namespace: {{.TargetNamespace}}
+{{end}}
+{{if .L7Enabled}}
+    toPorts:
+    - ports:
+      - port: "{{.TargetPort}}"
+        protocol: TCP
+      rules:
+        http:
+        - method: GET
+          path: /
+{{end}}
+{{else if .NetworkPolicy_Type "ccnp" }}
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: {{.Name}}
+  labels:
+    type: {{.TypeLabelValue}}
+spec:
+  endpointSelector:
+    matchLabels:
+      type: {{.TypeLabelValue}}
+  egress:
+  - toEndpoints:
+    - matchLabels:
+        net-pol-test: {{.TargetLabelValue}}
+{{if .OnlyTargetNamespace}}
+        k8s:io.kubernetes.pod.namespace: {{.TargetNamespace}}
+{{end}}
+{{if .L7Enabled}}
+    toPorts:
+    - ports:
+      - port: "{{.TargetPort}}"
+        protocol: TCP
+      rules:
+        http:
+        - method: GET
+          path: /
+{{end}}
+{{else}}
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -22,4 +80,8 @@ spec:
           kubernetes.io/metadata.name: {{.TargetNamespace}}
 {{else}}
       namespaceSelector: {}
+    ports:
+    - port: "{{.TargetPort}}"
+      protocol: TCP
 {{end}}
+{{end}}

clusterloader2/pkg/measurement/common/network-policy/network-policy-enforcement-latency.go

@@ -27,6 +27,7 @@ import (
 	"golang.org/x/time/rate"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 	"k8s.io/perf-tests/clusterloader2/pkg/framework"
@@ -111,6 +112,12 @@ type networkPolicyEnforcementMeasurement struct {
 	// testClientNodeSelectorValue is value key for the node label on which the
 	// test client pods should run.
 	testClientNodeSelectorValue string
+	// np type is the type of network policy to be created, default is k8s.
+	npType string
+	// l7Enabled is a flag to enable L7 policies.
+	l7Enabled bool
+	// l7 port
+	l7Port int
 }
 
 // Execute - Available actions:
@@ -215,6 +222,21 @@ func (nps *networkPolicyEnforcementMeasurement) initializeMeasurement(config *me
 		return fmt.Errorf("cannot initialize the %q, no namespaces with prefix %q exist", networkPolicyEnforcementName, testNamespacePrefix)
 	}
 
+	// Get the network policy type
+	if nps.npType, err = util.GetStringOrDefault(config.Params, "networkPolicyType", "k8s"); err != nil {
+		return fmt.Errorf("failed to get network policy type, error: %v", err)
+	}
+
+	// Get the L7 enabled flag
+	if nps.l7Enabled, err = util.GetBoolOrDefault(config.Params, "l7Enabled", false); err != nil {
+		klog.Infof("Failed to get L7 enabled flag, error: %v", err)
+	}
+
+	// Get the L7 port
+	if nps.l7Port, err = util.GetIntOrDefault(config.Params, "l7Port", 80); err != nil {
+		klog.Infof("Failed to get L7 port, error: %v", err)
+	}
+
 	return nil
 }
 
@@ -386,9 +408,10 @@ func (nps *networkPolicyEnforcementMeasurement) createPolicyAllowAPIServer() err
 
 func (nps *networkPolicyEnforcementMeasurement) createPolicyToTargetPods(policyName, targetNamespace, testType string, allowForTargetPods bool) error {
 	templateMap := map[string]interface{}{
-		"Name":           policyName,
-		"Namespace":      nps.testClientNamespace,
-		"TypeLabelValue": testType,
+		"Name":            policyName,
+		"Namespace":       nps.testClientNamespace,
+		"TypeLabelValue":  testType,
+		"ClientNamespace": nps.testClientNamespace,
 	}
 
 	if allowForTargetPods {
@@ -404,6 +427,14 @@ func (nps *networkPolicyEnforcementMeasurement) createPolicyToTargetPods(policyN
 		templateMap["OnlyTargetNamespace"] = false
 	}
 
+	templateMap["NetworkPolicy_Type"] = nps.npType
+
+	// if L7 enabled, then add the L7 parameters to templateMap
+	if nps.l7Enabled {
+		templateMap["L7Enabled"] = nps.l7Enabled
+		templateMap["TargetPort"] = nps.l7Port
+	}
+
 	if err := nps.framework.ApplyTemplatedManifests(manifestsFS, policyEgressTargetPodsFilePath, templateMap); err != nil {
 		return fmt.Errorf("error while creating allow egress to pods network policy: %v", err)
 	}
@@ -420,6 +451,7 @@ func (nps *networkPolicyEnforcementMeasurement) createTestClientDeployments(temp
 		templateMap["Name"] = fmt.Sprintf("%s-%s-%d", testType, netPolicyTestClientName, i)
 		templateMap["TargetNamespace"] = ns
 		templateMap["AllowPolicyName"] = fmt.Sprintf("%s-%d", allowPolicyName, i)
+		templateMap["NetworkPolicy_Type"] = nps.npType
 
 		if err := nps.framework.ApplyTemplatedManifests(manifestsFS, deploymentFilePath, templateMap); err != nil {
 			return fmt.Errorf("error while creating test client deployment: %v", err)
@@ -528,13 +560,55 @@ func (nps *networkPolicyEnforcementMeasurement) deleteClusterRoleAndBinding() er
 	return nps.k8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), netPolicyTestClientName, metav1.DeleteOptions{})
 }
 
+func (nps *networkPolicyEnforcementMeasurement) deleteNetworkPolicies() error {
+	klog.V(2).Infof("Deleting Cilium network policies for measurement %q", networkPolicyEnforcementName)
+
+	dynamicClient := nps.framework.GetDynamicClients().GetClient()
+
+	switch nps.npType {
+	case "k8s":
+		return nil
+	case "ccnp":
+		// Define the GVR for CiliumClusterwideNetworkPolicy
+		ccnpGVR := schema.GroupVersionResource{
+			Group:    "cilium.io",
+			Version:  "v2",
+			Resource: "ciliumclusterwidenetworkpolicies",
+		}
+
+		if err := dynamicClient.Resource(ccnpGVR).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: fmt.Sprintf("type=%s", policyCreationTest)}); err != nil {
+			klog.Warningf("failed to delete CiliumClusterwideNetworkPolicy of type %s, error: %v", policyCreationTest, err)
+		}
+
+		if err := dynamicClient.Resource(ccnpGVR).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: fmt.Sprintf("type=%s", podCreationTest)}); err != nil {
+			klog.Warningf("failed to delete CiliumClusterwideNetworkPolicy of type %s, error: %v", podCreationTest, err)
+		}
+	case "cnp":
+		// Define the GVR for CiliumNetworkPolicy
+		cnpGVR := schema.GroupVersionResource{
+			Group:    "cilium.io",
+			Version:  "v2",
+			Resource: "ciliumnetworkpolicies",
+		}
+
+		if err := dynamicClient.Resource(cnpGVR).Namespace(nps.testClientNamespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil {
+			klog.Warningf("failed to delete CiliumNetworkPolicy in ns:%s, error: %v", nps.testClientNamespace, err)
+		}
+	}
+	return nil
+}
+
 func (nps *networkPolicyEnforcementMeasurement) cleanUp() error {
 	if nps.k8sClient == nil {
 		return fmt.Errorf("cleanup skipped - the measurement is not running")
 	}
 
 	if err := nps.deleteClusterRoleAndBinding(); err != nil {
-		return err
+		klog.Warningf("Failed to delete ClusterRole and ClusterRoleBinding, error: %v", err)
+	}
+
+	if err := nps.deleteNetworkPolicies(); err != nil {
+		klog.Warningf("Failed to delete network policies, error: %v", err)
 	}
 
 	klog.V(2).Infof("Deleting namespace %q for measurement %q", nps.testClientNamespace, networkPolicyEnforcementName)

clusterloader2/pkg/measurement/common/network-policy/network-policy-soak/manifests/allow_apiserver_np.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{.Name}}
+  namespace: {{.Namespace}}
+  labels:
+    {{ (StructuralData .ClientLabelKey)}}: {{.ClientLabelValue}}
+spec:
+  podSelector:
+    matchLabels:
+      {{ (StructuralData .ClientLabelKey)}}: {{.ClientLabelValue}}
+  policyTypes:
+  - Egress
+  egress:
+  - ports:
+    - port: 443
+      protocol: TCP
+    - port: 80
+      protocol: TCP
+    to:
+    - ipBlock:
+        cidr: {{.KubeAPIServerIP}}/32

clusterloader2/pkg/measurement/common/network-policy/network-policy-soak/manifests/client_deploy.yaml

@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    {{ (StructuralData .ClientLabelKey)}}: {{.ClientLabelValue}}
+  name: {{.UniqueName}}
+  namespace: {{.ClientNamespace}}
+spec:
+  replicas: {{.Replicas}}
+  selector:
+    matchLabels:
+      {{ (StructuralData .ClientLabelKey)}}: {{.ClientLabelValue}}
+  template:
+    metadata:
+      labels:
+        {{ (StructuralData .ClientLabelKey)}}: {{.ClientLabelValue}}
+    spec:
+      nodeSelector:
+        node: "client"
+      serviceAccountName: {{.ClientName}}-sa
+      containers:
+      - image: acnpublic.azurecr.io/sanamsarath/netloader:0.7
+        name: netloader
+        args:
+          - "--dest_labelSelector={{.TargetLabelKey}}={{.TargetLabelValue}}"
+          - "--namespace={{.TargetNamespace}}"
+          - "--duration={{.Duration}}"
+          - "--interval=1"
+          - "--workers={{.Workers}}"
+          - "--destPort={{.TargetPort}}"
+          - "--destPath={{.TargetPath}}"
+        resources: 
+          requests: 
+             cpu: "20m"
+             memory: "40Mi"
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 900
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 900
+      - key: "slo"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+

clusterloader2/pkg/measurement/common/network-policy/network-policy-soak/manifests/clusterrole.yaml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{.Name}}-cr
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]

clusterloader2/pkg/measurement/common/network-policy/network-policy-soak/manifests/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{.Name}}-crb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{.Name}}-cr
+subjects:
+- kind: ServiceAccount
+  name: {{.Name}}-sa
+  namespace: {{.Namespace}}