scheduler network policies

This commit introduces two network policies for the scheduler:
1. A default deny-all policy that blocks all ingress and egress traffic by default. This serves as a security baseline, ensuring that no unintended connections are allowed from or to the scheduler pod.
2. A policy allowing egress traffic from the scheduler pod to the API server.

Signed-off-by: Ronny Baturov <[email protected]>

Author: rbaturov

pkg/numaresourcesscheduler/manifests/manifests.go

@@ -23,6 +23,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -117,6 +118,30 @@ func Deployment(namespace string) (*appsv1.Deployment, error) {
 	return dp, nil
 }
 
+func NetworkPolicy(policyType, namespace string) (*networkingv1.NetworkPolicy, error) {
+	var fileName string
+
+	if policyType == "" || policyType == "default" {
+		fileName = "networkpolicy.yaml"
+	} else {
+		fileName = fmt.Sprintf("networkpolicy.%s.yaml", policyType)
+	}
+
+	obj, err := loadObject(filepath.Join("yaml", fileName))
+	if err != nil {
+		return nil, err
+	}
+
+	np, ok := obj.(*networkingv1.NetworkPolicy)
+	if !ok {
+		return nil, fmt.Errorf("unexpected type, got %t", obj)
+	}
+	if namespace != "" {
+		np.Namespace = namespace
+	}
+	return np, nil
+}
+
 func deserializeObjectFromData(data []byte) (runtime.Object, error) {
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 	obj, _, err := decode(data, nil, nil)

pkg/numaresourcesscheduler/manifests/manifests_test.go

@@ -39,4 +39,10 @@ func TestLoad(t *testing.T) {
 	if obj, err := Deployment(""); obj == nil || err != nil {
 		t.Errorf("Deployment() failed: err=%v", err)
 	}
+	if obj, err := NetworkPolicy("default", ""); obj == nil || err != nil {
+		t.Errorf("NetworkPolicy() failed: err=%v", err)
+	}
+	if obj, err := NetworkPolicy("apiserver", ""); obj == nil || err != nil {
+		t.Errorf("NetworkPolicy() failed: err=%v", err)
+	}
 }

pkg/numaresourcesscheduler/manifests/sched/sched.go

@@ -19,6 +19,7 @@ package sched
 import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -29,14 +30,16 @@ import (
 )
 
 type Manifests struct {
-	ServiceAccount        *corev1.ServiceAccount
-	ConfigMap             *corev1.ConfigMap
-	ClusterRole           *rbacv1.ClusterRole
-	ClusterRoleBindingK8S *rbacv1.ClusterRoleBinding
-	ClusterRoleBindingNRT *rbacv1.ClusterRoleBinding
-	Role                  *rbacv1.Role
-	RoleBinding           *rbacv1.RoleBinding
-	Deployment            *appsv1.Deployment
+	ServiceAccount         *corev1.ServiceAccount
+	ConfigMap              *corev1.ConfigMap
+	ClusterRole            *rbacv1.ClusterRole
+	ClusterRoleBindingK8S  *rbacv1.ClusterRoleBinding
+	ClusterRoleBindingNRT  *rbacv1.ClusterRoleBinding
+	Role                   *rbacv1.Role
+	RoleBinding            *rbacv1.RoleBinding
+	Deployment             *appsv1.Deployment
+	DefaultNetworkPolicy   *networkingv1.NetworkPolicy
+	APIserverNetworkPolicy *networkingv1.NetworkPolicy
 }
 
 func (mf Manifests) ToObjects() []client.Object {
@@ -49,19 +52,23 @@ func (mf Manifests) ToObjects() []client.Object {
 		mf.Role,
 		mf.RoleBinding,
 		mf.Deployment,
+		mf.DefaultNetworkPolicy,
+		mf.APIserverNetworkPolicy,
 	}
 }
 
 func (mf Manifests) Clone() Manifests {
 	return Manifests{
-		ServiceAccount:        mf.ServiceAccount.DeepCopy(),
-		ConfigMap:             mf.ConfigMap.DeepCopy(),
-		ClusterRole:           mf.ClusterRole.DeepCopy(),
-		ClusterRoleBindingK8S: mf.ClusterRoleBindingK8S.DeepCopy(),
-		ClusterRoleBindingNRT: mf.ClusterRoleBindingNRT.DeepCopy(),
-		Role:                  mf.Role.DeepCopy(),
-		RoleBinding:           mf.RoleBinding.DeepCopy(),
-		Deployment:            mf.Deployment.DeepCopy(),
+		ServiceAccount:         mf.ServiceAccount.DeepCopy(),
+		ConfigMap:              mf.ConfigMap.DeepCopy(),
+		ClusterRole:            mf.ClusterRole.DeepCopy(),
+		ClusterRoleBindingK8S:  mf.ClusterRoleBindingK8S.DeepCopy(),
+		ClusterRoleBindingNRT:  mf.ClusterRoleBindingNRT.DeepCopy(),
+		Role:                   mf.Role.DeepCopy(),
+		RoleBinding:            mf.RoleBinding.DeepCopy(),
+		Deployment:             mf.Deployment.DeepCopy(),
+		DefaultNetworkPolicy:   mf.DefaultNetworkPolicy.DeepCopy(),
+		APIserverNetworkPolicy: mf.APIserverNetworkPolicy.DeepCopy(),
 	}
 }
 
@@ -108,5 +115,15 @@ func GetManifests(namespace string) (Manifests, error) {
 		return mf, err
 	}
 
+	mf.DefaultNetworkPolicy, err = manifests.NetworkPolicy("default", namespace)
+	if err != nil {
+		return mf, err
+	}
+
+	mf.APIserverNetworkPolicy, err = manifests.NetworkPolicy("apiserver", namespace)
+	if err != nil {
+		return mf, err
+	}
+
 	return mf, nil
 }

pkg/numaresourcesscheduler/manifests/yaml/networkpolicy.apiserver.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: scheduler-egress-to-api-server
+spec:
+  podSelector:
+    matchLabels:
+      app: secondary-scheduler 
+  egress:
+  - ports:
+    - protocol: TCP
+      port: 6443 
+  policyTypes:
+  - Egress

pkg/numaresourcesscheduler/manifests/yaml/networkpolicy.yaml

@@ -0,0 +1,11 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: scheduler-default-deny-all
+spec:
+  podSelector:
+    matchLabels:
+      app: secondary-scheduler
+  policyTypes:
+  - Ingress
+  - Egress

pkg/numaresourcesscheduler/objectstate/sched/sched.go

@@ -21,6 +21,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/klog/v2"
 
@@ -43,15 +44,17 @@ const (
 )
 
 type ExistingManifests struct {
-	Existing                   schedmanifests.Manifests
-	serviceAccountError        error
-	configMapError             error
-	clusterRoleError           error
-	clusterRoleBindingK8SError error
-	clusterRoleBindingNRTError error
-	roleError                  error
-	roleBindingError           error
-	deploymentError            error
+	Existing                    schedmanifests.Manifests
+	serviceAccountError         error
+	configMapError              error
+	clusterRoleError            error
+	clusterRoleBindingK8SError  error
+	clusterRoleBindingNRTError  error
+	roleError                   error
+	roleBindingError            error
+	deploymentError             error
+	DefaultNetworkPolicyError   error
+	APIserverNetworkPolicyError error
 }
 
 func (em ExistingManifests) State(mf schedmanifests.Manifests) []objectstate.ObjectState {
@@ -112,6 +115,20 @@ func (em ExistingManifests) State(mf schedmanifests.Manifests) []objectstate.Obj
 			Compare:  compare.Object,
 			Merge:    merge.ObjectForUpdate,
 		},
+		{
+			Existing: em.Existing.DefaultNetworkPolicy,
+			Error:    em.DefaultNetworkPolicyError,
+			Desired:  mf.DefaultNetworkPolicy.DeepCopy(),
+			Compare:  compare.Object,
+			Merge:    merge.MetadataForUpdate,
+		},
+		{
+			Existing: em.Existing.APIserverNetworkPolicy,
+			Error:    em.APIserverNetworkPolicyError,
+			Desired:  mf.APIserverNetworkPolicy.DeepCopy(),
+			Compare:  compare.Object,
+			Merge:    merge.MetadataForUpdate,
+		},
 	}
 }
 
@@ -158,6 +175,16 @@ func FromClient(ctx context.Context, cli client.Client, mf schedmanifests.Manife
 	if ret.deploymentError = cli.Get(ctx, client.ObjectKeyFromObject(mf.Deployment), dp); ret.deploymentError == nil {
 		ret.Existing.Deployment = dp
 	}
+
+	np := &networkingv1.NetworkPolicy{}
+	if ret.DefaultNetworkPolicyError = cli.Get(ctx, client.ObjectKeyFromObject(mf.DefaultNetworkPolicy), np); ret.DefaultNetworkPolicyError == nil {
+		ret.Existing.DefaultNetworkPolicy = np
+	}
+
+	npAPI := &networkingv1.NetworkPolicy{}
+	if ret.APIserverNetworkPolicyError = cli.Get(ctx, client.ObjectKeyFromObject(mf.APIserverNetworkPolicy), npAPI); ret.APIserverNetworkPolicyError == nil {
+		ret.Existing.APIserverNetworkPolicy = npAPI
+	}
 	return ret
 }