Automatically allow egress IPs on ManagedSeeds

Signed-off-by: Justin Lamp <[email protected]>

Author: modzilla99

charts/gardener-extension-acl/templates/rbac.yaml

@@ -144,6 +144,23 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+{{ include "labels" . | indent 4 }}
+  name: {{ include "name" . }}
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  resourceNames:
+  - shoot-info
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: {{ include "name" . }}
@@ -157,3 +174,19 @@ subjects:
 - kind: ServiceAccount
   name: {{ include "name" . }}
   namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "name" . }}
+  namespace: kube-system
+  labels:
+{{ include "labels" . | indent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "name" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "name" . }}
+  namespace: {{ .Release.Namespace }}

pkg/controller/actuator.go

@@ -36,6 +36,7 @@ import (
 	"github.com/pkg/errors"
 	istionetworkv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/client-go/rest"
@@ -48,6 +49,8 @@ import (
 	"github.com/stackitcloud/gardener-extension-acl/pkg/extensionspec"
 	"github.com/stackitcloud/gardener-extension-acl/pkg/helper"
 	"github.com/stackitcloud/gardener-extension-acl/pkg/imagevector"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
 const (
@@ -147,6 +150,14 @@ func (a *actuator) Reconcile(ctx context.Context, log logr.Logger, ex *extension
 
 	alwaysAllowedCIDRs = append(alwaysAllowedCIDRs, helper.GetSeedSpecificAllowedCIDRs(cluster.Seed)...)
 
+	// On ManagedSeeds the shoot-info ConfigMap contains the egress CIDRs of the cluster
+	// These CIDRs need to be allowed in order for the external API reachability check to work
+	if egressCIDRs, err := a.getSeedEgressIPOnManagedSeeds(ctx); err != nil {
+		return err
+	} else if len(egressCIDRs) > 0 {
+		alwaysAllowedCIDRs = append(alwaysAllowedCIDRs, egressCIDRs...)
+	}
+
 	if len(a.extensionConfig.AdditionalAllowedCIDRs) >= 1 {
 		alwaysAllowedCIDRs = append(alwaysAllowedCIDRs, a.extensionConfig.AdditionalAllowedCIDRs...)
 	}
@@ -446,3 +457,25 @@ func (a *actuator) findDefaultIstioLabels(
 
 	return gw.Spec.Selector, nil
 }
+
+// getSeedEgressIPOnManagedSeeds returns the egressIP CIDRs of the ManagedSeed, if the
+// Seed is not a shoot, it will return an empty list
+func (a *actuator) getSeedEgressIPOnManagedSeeds(ctx context.Context) ([]string, error) {
+	cm := corev1.ConfigMap{}
+	if err := a.client.Get(ctx, client.ObjectKey{
+		Name:      "shoot-info",
+		Namespace: "kube-system"},
+		&cm); err != nil {
+		if apierrors.IsNotFound(err) {
+			return []string{}, nil
+		}
+		return nil, err
+	}
+
+	cidrs, ok := cm.Data["egressCIDRs"]
+	if !ok {
+		return nil, errors.New("unable to get egress CIDRs from shoot-info ConfigMap")
+	}
+
+	return strings.Split(cidrs, ","), nil
+}

pkg/controller/actuator_test.go

@@ -222,6 +222,40 @@ var _ = Describe("actuator test", func() {
 		})
 	})
 
+	Describe("reconciliation of an extension object running on a managedSeed", func() {
+		BeforeEach(func() {
+			createShootInfo([]string{"1.1.1.1/32", "1.1.1.2/32"})
+		})
+
+		AfterEach(func() {
+			deleteShootInfo()
+		})
+
+		It("should create ACLs including egressIPs of managedSeed", func() {
+			extSpec := extensionspec.ExtensionSpec{
+				Rule: &envoyfilters.ACLRule{
+					Cidrs:  []string{"1.2.3.4/24"},
+					Action: "ALLOW",
+					Type:   "remote_ip",
+				},
+			}
+			extSpecJSON, err := json.Marshal(extSpec)
+			Expect(err).NotTo(HaveOccurred())
+			ext := createNewExtension(shootNamespace1, extSpecJSON)
+			Expect(ext).To(Not(BeNil()))
+
+			Expect(a.Reconcile(ctx, logger, ext)).To(Succeed())
+
+			mr := &v1alpha1.ManagedResource{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: ResourceNameSeed, Namespace: shootNamespace1}, mr)).To(Succeed())
+			secret := &corev1.Secret{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: mr.Spec.SecretRefs[0].Name, Namespace: shootNamespace1}, secret)).To(Succeed())
+			Expect(secret.Data["seed"]).To(ContainSubstring("1.2.3.4"))
+			Expect(secret.Data["seed"]).To(ContainSubstring("1.1.1.1"))
+			Expect(secret.Data["seed"]).To(ContainSubstring("1.1.1.2"))
+		})
+	})
+
 	Describe("a shoot switching the istio namespace (e.g. when being migrated to HA)", func() {
 		It("should modify the EnvoyFilter objects accordingly", func() {
 			By("1) creating the EnvoyFilter object correctly in the ORIGINAL namespace")

pkg/controller/suite_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 
 	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
@@ -18,6 +19,7 @@ import (
 	istionetworkingv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -247,6 +249,29 @@ func createNewCluster(shootNamespace string) {
 	Expect(k8sClient.Create(ctx, cluster)).ShouldNot(HaveOccurred())
 }
 
+func createShootInfo(cidrs []string) {
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "shoot-info",
+			Namespace: "kube-system",
+		},
+		Data: map[string]string{
+			"egressCIDRs": strings.Join(cidrs, ","),
+		},
+	}
+	Expect(k8sClient.Create(ctx, cm)).ShouldNot(HaveOccurred())
+}
+
+func deleteShootInfo() {
+	cm := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "shoot-info",
+			Namespace: "kube-system",
+		},
+	}
+	Expect(k8sClient.Delete(ctx, cm)).ShouldNot(HaveOccurred())
+}
+
 func deleteNamespace(name string) {
 	namespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{