NE-1476: Added network policies for the router

Added the framework for network policies for the router.

The operator has a deny all network policy that for the openshift-ingress-operator namespace and an allow policy for egress to the apiserver and dns ports at any IP.

The operator installs a deny all network policy for the openshift-ingress namespace.

Then for each ingresscontroller that it manages it installs an allow policy for ingress for http and https traffic and metrics.

It has to allow ingress from the router pods to any IP because the route endpoints can be at any ip or pod.

It also needs access to the api server, but that is covered by the wildcard allow policy.

https://issues.redhat.com/browse/NE-1476

Author: knobunc

manifests/00-cluster-role.yaml

@@ -155,6 +155,7 @@ rules:
   - networking.k8s.io
   resources:
   - ingressclasses
+  - networkpolicies
   verbs:
   - '*'
 

manifests/01-network-policy.yaml

@@ -0,0 +1,43 @@
+# We control the namespace, deny anything we do not explicitly allow
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-operator-deny-all
+  namespace: openshift-ingress-operator
+  annotations:
+    capability.openshift.io/name: Ingress
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+spec:
+  podSelector: {}
+  policyTypes:
+  - Ingress
+  - Egress
+---
+### Allow the operator needs to talk to the apiserver and dns,
+### but it also needs to be able to configure external DNS and the endpoints
+### are only known at run time.  So it needs wildcard egress.
+### Allow access to the metrics ports on the operators
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-operator-allow
+  namespace: openshift-ingress-operator
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+spec:
+  podSelector:
+    matchLabels:
+      name: ingress-operator
+  policyTypes:
+  - Egress
+  - Ingress
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 9393

manifests/01-role.yaml

@@ -37,6 +37,13 @@ rules:
   - services
   verbs:
   - "*"
+
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - "*"
 ---
 # Role for the operator to delete Role and RoleBindings
 # in the openshift-config namespace.

pkg/manifests/assets/canary/networkpolicy-allow.yaml

@@ -0,0 +1,19 @@
+# Router Pods
+#  No egress
+#  Ingress to http on port 8888 and https on port 8443 (TCP)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+# name, namespace, labels and annotations are set at runtime
+spec:
+  podSelector:
+    # matchLabels are set at runtime
+    matchLabels: {}
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 8888
+    - protocol: TCP
+      port: 8443
+  policyTypes:
+  - Ingress
+  - Egress

pkg/manifests/assets/canary/networkpolicy-deny-all.yaml

@@ -0,0 +1,11 @@
+# Default deny all policy for all pods in the namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: openshift-ingress-canary-deny-all
+  namespace: openshift-ingress-canary
+spec:
+ podSelector: {}
+ policyTypes:
+ - Ingress
+ - Egress

pkg/manifests/assets/gateway-api/gateway-networkpolicy-allow.yaml

@@ -0,0 +1,23 @@
+# Istio Pods for Gateway API
+#  Egress to api server and everything (routes set endpoints, so can be literally anything)
+#  Ingress need to be wildcarded as well since the ports are configurable
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: gateway-api-allow
+  namespace: openshift-ingress
+spec:
+  podSelector:
+    matchLabels:
+      istio.io/rev: "openshift-gateway"
+  ingress:
+  - from:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  policyTypes:
+  - Ingress
+  - Egress

pkg/manifests/assets/router/networkpolicy-allow.yaml

@@ -0,0 +1,25 @@
+# Router Pods
+#  Egress to api server and everything (routes set endpoints, so can be literally anything)
+#  Ingress to http on port 80 and https on port 443 (TCP) and to metrics (1936)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+# name, namespace,labels and annotations are set at runtime
+spec:
+  podSelector:
+    # matchLabels are set at runtime
+    matchLabels: {}
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 80
+    - protocol: TCP
+      port: 443
+    - protocol: TCP
+      port: 1936
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+  policyTypes:
+  - Ingress
+  - Egress

pkg/manifests/assets/router/networkpolicy-deny-all.yaml

@@ -0,0 +1,11 @@
+# Default deny all policy for all pods in the namespace
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+    name: openshift-ingress-deny-all
+    namespace: openshift-ingress
+spec:
+ podSelector: {}
+ policyTypes:
+ - Ingress
+ - Egress

pkg/manifests/manifests.go

@@ -13,6 +13,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
@@ -24,31 +25,36 @@ import (
 )
 
 const (
-	RouterNamespaceAsset          = "assets/router/namespace.yaml"
-	RouterServiceAccountAsset     = "assets/router/service-account.yaml"
-	RouterClusterRoleAsset        = "assets/router/cluster-role.yaml"
-	RouterClusterRoleBindingAsset = "assets/router/cluster-role-binding.yaml"
-	RouterDeploymentAsset         = "assets/router/deployment.yaml"
-	RouterServiceInternalAsset    = "assets/router/service-internal.yaml"
-	RouterServiceCloudAsset       = "assets/router/service-cloud.yaml"
+	RouterNamespaceAsset            = "assets/router/namespace.yaml"
+	RouterServiceAccountAsset       = "assets/router/service-account.yaml"
+	RouterClusterRoleAsset          = "assets/router/cluster-role.yaml"
+	RouterClusterRoleBindingAsset   = "assets/router/cluster-role-binding.yaml"
+	RouterDeploymentAsset           = "assets/router/deployment.yaml"
+	RouterServiceInternalAsset      = "assets/router/service-internal.yaml"
+	RouterServiceCloudAsset         = "assets/router/service-cloud.yaml"
+	RouterNetworkPolicyDenyAllAsset = "assets/router/networkpolicy-deny-all.yaml"
+	RouterNetworkPolicyAllowAsset   = "assets/router/networkpolicy-allow.yaml"
 
 	MetricsClusterRoleAsset        = "assets/router/metrics/cluster-role.yaml"
 	MetricsClusterRoleBindingAsset = "assets/router/metrics/cluster-role-binding.yaml"
 	MetricsRoleAsset               = "assets/router/metrics/role.yaml"
 	MetricsRoleBindingAsset        = "assets/router/metrics/role-binding.yaml"
 
-	CanaryNamespaceAsset = "assets/canary/namespace.yaml"
-	CanaryDaemonSetAsset = "assets/canary/daemonset.yaml"
-	CanaryServiceAsset   = "assets/canary/service.yaml"
-	CanaryRouteAsset     = "assets/canary/route.yaml"
-
-	GatewayClassCRDAsset            = "assets/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml"
-	GatewayCRDAsset                 = "assets/gateway-api/gateway.networking.k8s.io_gateways.yaml"
-	GRPCRouteCRDAsset               = "assets/gateway-api/gateway.networking.k8s.io_grpcroutes.yaml"
-	HTTPRouteCRDAsset               = "assets/gateway-api/gateway.networking.k8s.io_httproutes.yaml"
-	ReferenceGrantCRDAsset          = "assets/gateway-api/gateway.networking.k8s.io_referencegrants.yaml"
-	GatewayAPIAdminClusterRoleAsset = "assets/gateway-api/aggregated-cluster-roles/admin-cluster-role.yaml"
-	GatewayAPIViewClusterRoleAsset  = "assets/gateway-api/aggregated-cluster-roles/view-cluster-role.yaml"
+	CanaryNamespaceAsset            = "assets/canary/namespace.yaml"
+	CanaryDaemonSetAsset            = "assets/canary/daemonset.yaml"
+	CanaryServiceAsset              = "assets/canary/service.yaml"
+	CanaryRouteAsset                = "assets/canary/route.yaml"
+	CanaryNetworkPolicyDenyAllAsset = "assets/canary/networkpolicy-deny-all.yaml"
+	CanaryNetworkPolicyAllowAsset   = "assets/canary/networkpolicy-allow.yaml"
+
+	GatewayClassCRDAsset              = "assets/gateway-api/gateway.networking.k8s.io_gatewayclasses.yaml"
+	GatewayCRDAsset                   = "assets/gateway-api/gateway.networking.k8s.io_gateways.yaml"
+	GRPCRouteCRDAsset                 = "assets/gateway-api/gateway.networking.k8s.io_grpcroutes.yaml"
+	HTTPRouteCRDAsset                 = "assets/gateway-api/gateway.networking.k8s.io_httproutes.yaml"
+	ReferenceGrantCRDAsset            = "assets/gateway-api/gateway.networking.k8s.io_referencegrants.yaml"
+	GatewayAPIAdminClusterRoleAsset   = "assets/gateway-api/aggregated-cluster-roles/admin-cluster-role.yaml"
+	GatewayAPIViewClusterRoleAsset    = "assets/gateway-api/aggregated-cluster-roles/view-cluster-role.yaml"
+	GatewayAPIAllowNetworkPolicyAsset = "assets/gateway-api/gateway-networkpolicy-allow.yaml"
 
 	// Annotation used to inform the certificate generation service to
 	// generate a cluster-signed certificate and populate the secret.
@@ -258,6 +264,22 @@ func CanaryRoute() *routev1.Route {
 	return route
 }
 
+func CanaryNetworkPolicyDenyAll() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(CanaryNetworkPolicyDenyAllAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func CanaryNetworkPolicyAllow() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(CanaryNetworkPolicyAllowAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
 func GatewayClassCRD() *apiextensionsv1.CustomResourceDefinition {
 	crd, err := NewCustomResourceDefinition(MustAssetReader(GatewayClassCRDAsset))
 	if err != nil {
@@ -314,6 +336,30 @@ func GatewayAPIViewClusterRole() *rbacv1.ClusterRole {
 	return clusterRole
 }
 
+func GatewayAPIAllowNetworkPolicy() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(GatewayAPIAllowNetworkPolicyAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func RouterNetworkPolicyDenyAll() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(RouterNetworkPolicyDenyAllAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
+func RouterNetworkPolicyAllow() *networkingv1.NetworkPolicy {
+	networkPolicy, err := NewNetworkPolicy(MustAssetReader(RouterNetworkPolicyAllowAsset))
+	if err != nil {
+		panic(err)
+	}
+	return networkPolicy
+}
+
 func NewServiceAccount(manifest io.Reader) (*corev1.ServiceAccount, error) {
 	sa := corev1.ServiceAccount{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&sa); err != nil {
@@ -404,6 +450,15 @@ func NewRoute(manifest io.Reader) (*routev1.Route, error) {
 	return &o, nil
 }
 
+func NewNetworkPolicy(manifest io.Reader) (*networkingv1.NetworkPolicy, error) {
+	o := networkingv1.NetworkPolicy{}
+	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil {
+		return nil, err
+	}
+
+	return &o, nil
+}
+
 func NewCustomResourceDefinition(manifest io.Reader) (*apiextensionsv1.CustomResourceDefinition, error) {
 	o := apiextensionsv1.CustomResourceDefinition{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&o); err != nil {

pkg/manifests/manifests_test.go

@@ -35,7 +35,8 @@ func TestManifests(t *testing.T) {
 	RouterClusterRole()
 	RouterClusterRoleBinding()
 	RouterStatsSecret(ci)
-
+	RouterNetworkPolicyDenyAll()
+	RouterNetworkPolicyAllow()
 	MetricsClusterRole()
 	MetricsClusterRoleBinding()
 	MetricsRole()
@@ -51,6 +52,14 @@ func TestManifests(t *testing.T) {
 	GRPCRouteCRD()
 	HTTPRouteCRD()
 	ReferenceGrantCRD()
+	GatewayAPIAllowNetworkPolicy()
+
+	CanaryNamespace()
+	CanaryDaemonSet()
+	CanaryService()
+	CanaryRoute()
+	CanaryNetworkPolicyDenyAll()
+	CanaryNetworkPolicyAllow()
 
 	adminOnlyVerbs := []string{"create", "update", "patch", "delete", "deletecollection"}
 	GatewayAPIAdminClusterRole()