NE-1476: Added network policies for the router

Added the framework for network policies for the router.

The operator has a deny all network policy that for the openshift-ingress-operator namespace and an allow policy for egress to the apiserver and dns ports at any IP.

The operator installs a deny all network policy for the openshift-ingress namespace.

Then for each ingresscontroller that it manages it installs an allow policy for ingress for http and https traffic and metrics.

It has to allow ingress from the router pods to any IP because the route endpoints can be at any ip or pod.

It also needs access to the api server, but that is covered by the wildcard allow policy.

https://issues.redhat.com/browse/NE-1476

Author: knobunc

manifests/00-cluster-role.yaml

@@ -155,6 +155,7 @@ rules:
   - networking.k8s.io
   resources:
   - ingressclasses
+  - networkpolicies
   verbs:
   - '*'
 

manifests/01-role.yaml

@@ -37,6 +37,13 @@ rules:
   - services
   verbs:
   - "*"
+
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - "*"
 ---
 # Role for the operator to delete Role and RoleBindings
 # in the openshift-config namespace.

manifests/02-network-policy.yaml

@@ -0,0 +1,43 @@
+# We control the namespace, deny anything we do not explicitly allow
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-operator-deny-all
+  namespace: openshift-ingress-operator
+  annotations:
+    capability.openshift.io/name: Ingress
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+spec:
+  podSelector: {}
+  policyTypes:
+  - Ingress
+  - Egress
+---
+### Allow the operator needs to talk to the apiserver and dns,
+### but it also needs to be able to configure external DNS and the endpoints
+### are only known at run time.  So it needs wildcard egress.
+### Allow access to the metrics ports on the operators
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-operator-allow
+  namespace: openshift-ingress-operator
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+spec:
+  podSelector:
+    matchLabels:
+      name: ingress-operator
+  policyTypes:
+  - Egress
+  - Ingress
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 9393

manifests/02-deployment-ibm-cloud-managed.yaml renamed to manifests/03-deployment-ibm-cloud-managed.yaml

@@ -0,0 +1,25 @@
+# Router Pods
+#  Egress to api server and everything (routes set endpoints, so can be literally anything)
+#  Ingress to http on port 80 and https on port 443 (TCP) and to metrics (1936)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+# name, namespace,labels and annotations are set at runtime
+spec:
+  podSelector:
+    # matchLabels are set at runtime
+    matchLabels: {}
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 80
+    - protocol: TCP
+      port: 443
+    - protocol: TCP
+      port: 1936
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  policyTypes:
+  - Ingress
+  - Egress

manifests/02-deployment.yaml renamed to manifests/03-deployment.yaml

@@ -0,0 +1,11 @@
+# Default deny all policy for all pods in the namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+    name: openshift-ingress-deny-all
+    namespace: openshift-ingress
+spec:
+ podSelector: {}
+ policyTypes:
+ - Ingress
+ - Egress

manifests/03-cluster-operator.yaml renamed to manifests/04-cluster-operator.yaml

@@ -13,6 +13,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
@@ -24,13 +25,15 @@ import (
 )
 
 const (
-	RouterNamespaceAsset          = "assets/router/namespace.yaml"
-	RouterServiceAccountAsset     = "assets/router/service-account.yaml"
-	RouterClusterRoleAsset        = "assets/router/cluster-role.yaml"
-	RouterClusterRoleBindingAsset = "assets/router/cluster-role-binding.yaml"
-	RouterDeploymentAsset         = "assets/router/deployment.yaml"
-	RouterServiceInternalAsset    = "assets/router/service-internal.yaml"
-	RouterServiceCloudAsset       = "assets/router/service-cloud.yaml"
+	RouterNamespaceAsset            = "assets/router/namespace.yaml"
+	RouterServiceAccountAsset       = "assets/router/service-account.yaml"
+	RouterClusterRoleAsset          = "assets/router/cluster-role.yaml"
+	RouterClusterRoleBindingAsset   = "assets/router/cluster-role-binding.yaml"
+	RouterDeploymentAsset           = "assets/router/deployment.yaml"
+	RouterServiceInternalAsset      = "assets/router/service-internal.yaml"
+	RouterServiceCloudAsset         = "assets/router/service-cloud.yaml"
+	RouterNetworkPolicyDenyAllAsset = "assets/router/networkpolicy-deny-all.yaml"
+	RouterNetworkPolicyAllowAsset   = "assets/router/networkpolicy-allow.yaml"
 
 	MetricsClusterRoleAsset        = "assets/router/metrics/cluster-role.yaml"
 	MetricsClusterRoleBindingAsset = "assets/router/metrics/cluster-role-binding.yaml"
@@ -314,6 +317,22 @@ func GatewayAPIViewClusterRole() *rbacv1.ClusterRole {
 	return clusterRole
 }
 
+func RouterNetworkPolicyDenyAll() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(RouterNetworkPolicyDenyAllAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func RouterNetworkPolicyAllow() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(RouterNetworkPolicyAllowAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
 func NewServiceAccount(manifest io.Reader) (*corev1.ServiceAccount, error) {
 	sa := corev1.ServiceAccount{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&sa); err != nil {
@@ -404,6 +423,15 @@ func NewRoute(manifest io.Reader) (*routev1.Route, error) {
 	return &o, nil
 }
 
+func NewNetworkPolicy(manifest io.Reader) (*networkingv1.NetworkPolicy, error) {
+	o := networkingv1.NetworkPolicy{}
+	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil {
+		return nil, err
+	}
+
+	return &o, nil
+}
+
 func NewCustomResourceDefinition(manifest io.Reader) (*apiextensionsv1.CustomResourceDefinition, error) {
 	o := apiextensionsv1.CustomResourceDefinition{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil {

pkg/manifests/assets/router/networkpolicy-allow.yaml

@@ -24,6 +24,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 
 	"k8s.io/client-go/tools/record"
 
@@ -114,6 +115,9 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) {
 	if err := c.Watch(source.Kind[client.Object](operatorCache, &corev1.Service{}, enqueueRequestForOwningIngressController(config.Namespace))); err != nil {
 		return nil, err
 	}
+	if err := c.Watch(source.Kind[client.Object](operatorCache, &networkingv1.NetworkPolicy{}, enqueueRequestForOwningIngressController(config.Namespace))); err != nil {
+		return nil, err
+	}
 	// Add watch for deleted pods specifically for ensuring ingress deletion.
 	if err := c.Watch(source.Kind[client.Object](operatorCache, &corev1.Pod{}, enqueueRequestForOwningIngressController(config.Namespace), predicate.Funcs{
 		CreateFunc:  func(e event.CreateEvent) bool { return false },
@@ -1078,6 +1082,10 @@ func (r *reconciler) ensureIngressController(ci *operatorv1.IngressController, d
 		return fmt.Errorf("failed to ensure namespace: %v", err)
 	}
 
+	if _, _, err := r.ensureRouterNamespaceNetworkPolicy(ci); err != nil {
+		return fmt.Errorf("failed to ensure default namespacenetwork policy: %v", err)
+	}
+
 	if err := r.ensureRouterServiceAccount(); err != nil {
 		return fmt.Errorf("failed to ensure service account: %v", err)
 	}
@@ -1086,6 +1094,10 @@ func (r *reconciler) ensureIngressController(ci *operatorv1.IngressController, d
 		return fmt.Errorf("failed to ensure cluster role binding: %v", err)
 	}
 
+	if err := r.ensureRouterNetworkPolicy(ci); err != nil {
+		return fmt.Errorf("failed to ensure network policy: %v", err)
+	}
+
 	var errs []error
 	if _, _, err := r.ensureServiceCAConfigMap(); err != nil {
 		// Even if we were unable to create the configmap at this time,