Gardener operator (#239)

Author: Gerrit91

Makefile

@@ -40,7 +40,7 @@ VRF=Vrf20
 else ifeq ($(MINI_LAB_FLAVOR),gardener)
 GARDENER_ENABLED=true
 # usually gardener restricts the maximum version for k8s:
-K8S_VERSION=1.30.8
+K8S_VERSION=1.32.5
 LAB_TOPOLOGY=mini-lab.sonic.yaml
 VRF=Vrf20
 else
@@ -375,15 +375,13 @@ dev-env:
 
 .PHONY: fetch-virtual-kubeconfig
 fetch-virtual-kubeconfig:
-	kubectl config unset users.virtual-garden
-	kubectl config unset contexts.virtual-garden
-	kubectl config unset clusters.virtual-garden
-	kubectl get secret -n garden garden-kubeconfig-for-admin -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	kubectl --kubeconfig=.virtual-kubeconfig config rename-context garden virtual-garden
-	sed -i 's/name: garden/name: virtual-garden/g' .virtual-kubeconfig
-	sed -i 's/name: admin/name: virtual-garden/g' .virtual-kubeconfig
-	kubectl --kubeconfig=.virtual-kubeconfig config set contexts.virtual-garden.cluster virtual-garden
-	kubectl --kubeconfig=.virtual-kubeconfig config set contexts.virtual-garden.user virtual-garden
-	KUBECONFIG=$$KUBECONFIG:.virtual-kubeconfig kubectl config view --flatten > .merged-kubeconfig
-	rm .virtual-kubeconfig
-	mv .merged-kubeconfig .kubeconfig
+	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
+	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
+	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
+	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden
+	@kubectl --kubeconfig=$(KUBECONFIG) config unset clusters.garden
+	@KUBECONFIG=$(KUBECONFIG):.virtual-kubeconfig kubectl config view --flatten > .merged-kubeconfig
+	@rm .virtual-kubeconfig
+	@mv .merged-kubeconfig $(KUBECONFIG)

compose.yaml

@@ -89,7 +89,7 @@ services:
     environment:
       - REGISTRY_PROXY_REMOTEURL="https://registry-1.docker.io"
       - REGISTRY_PROXY_TTL=168h
-      - REGISTRY_STORAGE_DELETE_ENABLED=true 
+      - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
   proxy-gcr:
     image: registry:3
@@ -102,7 +102,7 @@ services:
     environment:
       - REGISTRY_PROXY_REMOTEURL="https://gcr.io"
       - REGISTRY_PROXY_TTL=168h
-      - REGISTRY_STORAGE_DELETE_ENABLED=true 
+      - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
   proxy-ghcr:
     image: registry:3
@@ -115,7 +115,7 @@ services:
     environment:
       - REGISTRY_PROXY_REMOTEURL="https://ghcr.io"
       - REGISTRY_PROXY_TTL=168h
-      - REGISTRY_STORAGE_DELETE_ENABLED=true 
+      - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
   proxy-k8s:
     image: registry:3
@@ -128,7 +128,7 @@ services:
     environment:
       - REGISTRY_PROXY_REMOTEURL="https://registry.k8s.io"
       - REGISTRY_PROXY_TTL=168h
-      - REGISTRY_STORAGE_DELETE_ENABLED=true 
+      - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
   proxy-quay:
     image: registry:3
@@ -141,7 +141,7 @@ services:
     environment:
       - REGISTRY_PROXY_REMOTEURL="https://quay.io"
       - REGISTRY_PROXY_TTL=168h
-      - REGISTRY_STORAGE_DELETE_ENABLED=true 
+      - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
 volumes:
   proxy-docker:

control-plane/kind.yaml

@@ -21,6 +21,13 @@ nodes:
       - containerPort: 50051
         hostPort: 50051
         listenAddress: 0.0.0.0
+    # if you want to run gardener operator + metal-stack, you need more pods
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            max-pods: "256"
 containerdConfigPatches:
   - |-
     [plugins."io.containerd.grpc.v1.cri".registry]

deploy_gardener.yaml

@@ -16,67 +16,125 @@
             nodeNetwork: 172.18.0.0/16
             podNetwork: 10.244.0.0/24
             serviceNetwork: 10.96.0.0/16
+      tags: gardener
 
-    - name: Create garden namespace
-      k8s:
-        definition:
-          apiVersion: v1
-          kind: Namespace
-          metadata:
-            name: garden
-
-    # our current state in metal-roles/gardener does not support network policies from gardenlet <-> virtual garden
-    # this should be possible to resolve when we use the Gardener Operator
-    - name: Deploy allow all network policy
-      k8s:
-        definition: "{{ lookup('file', 'netpol-allow-all.yaml') }}"
-        namespace: garden
-        apply: yes
   roles:
     - name: ansible-common
       tags: always
     - name: minio
+      tags: minio
     - name: powerdns
       tags: powerdns
-    - name: metal-roles/control-plane/roles/gardener
+    - name: metal-roles/control-plane/roles/gardener-operator
+      tags: gardener
+    - name: metal-roles/control-plane/roles/gardener-extensions
+      tags: gardener
+    - name: metal-roles/control-plane/roles/gardener-virtual-garden-access
+      tags: gardener
+    - name: metal-roles/control-plane/roles/gardener-cloud-profile
+      tags: gardener
+    - name: metal-roles/control-plane/roles/gardener-gardenlet
       tags: gardener
   vars:
     metal_control_plane_host_provider: metal
 
   post_tasks:
-    # gardener exposes the istio ingress gateway through service type load balancer
-    # we can fake the exposal by patching the status field, which is also what's
-    # done in the gardener local environment
-    - name: Wait for istio ingress gateway service
-      kubernetes.core.k8s_info:
-        api_version: v1
-        kind: Service
-        name: istio-ingressgateway
-        namespace: istio-ingress
-      register: result
-      until: result.resources
-      retries: 30
-      delay: 10
+      - name: Get kubeconfig for virtual garden access
+        virtual_garden_kubeconfig:
+          garden_name: "{{ metal_control_plane_stage_name }}"
+        tags: gardener
+
+      - name: Wait for Gardenlet to be reconciled
+        kubernetes.core.k8s_info:
+          api_version: seedmanagement.gardener.cloud/v1alpha1
+          kind: Gardenlet
+          name: "local"
+          namespace: garden
+          kubeconfig: "{{ virtual_garden_kubeconfig }}"
+          wait: yes
+          wait_condition:
+            reason: Reconciled
+            status: "True"
+            type: GardenletReconciled
+          wait_timeout: 900
+        tags: gardener
+
+      - name: Wait for istio ingress gateway service
+        kubernetes.core.k8s_info:
+          api_version: v1
+          kind: Service
+          name: istio-ingressgateway
+          namespace: istio-ingress
+        register: result
+        until: result.resources
+        retries: 30
+        delay: 10
+        tags: gardener
+
+      - name: Patch istio ingress gateway service status
+        patch_service_status_k8s:
+          name: istio-ingressgateway
+          namespace: istio-ingress
+          body:
+            status:
+              loadBalancer:
+                ingress:
+                  - ip: "172.17.0.1"
+        tags: gardener
+
+      - name: Expose istio gateway through ingress-nginx (for local environments)
+        k8s:
+          definition:
+            apiVersion: networking.k8s.io/v1
+            kind: Ingress
+            metadata:
+              annotations:
+                nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+              name: apiserver-ingress
+              namespace: istio-ingress
+            spec:
+              ingressClassName: nginx
+              rules:
+              - host: "{{ metal_control_plane_stage_name }}.{{ gardener_gardenlet_default_dns_domain }}"
+                http:
+                  paths:
+                  - path: /
+                    pathType: Prefix
+                    backend:
+                      service:
+                        name: istio-ingressgateway
+                        port:
+                          number: 443
+              tls:
+              - hosts:
+                - "{{ metal_control_plane_stage_name }}.{{ gardener_gardenlet_default_dns_domain }}"
+        tags: gardener
 
-    - name: Patch ingress status of istio ingress gateway to allow seed to get ready
-      patch_service_status_k8s:
-        name: istio-ingressgateway
-        namespace: istio-ingress
-        body:
-          status:
-            loadBalancer:
-              ingress:
-                - ip: "172.17.0.1"
+      - name: Wait until Garden is ready
+        kubernetes.core.k8s_info:
+          api_version: "operator.gardener.cloud/v1alpha1"
+          kind: Garden
+          name: "{{ metal_control_plane_stage_name }}"
+          wait: yes
+          wait_condition:
+            status: "True"
+            type: "{{ item }}"
+          wait_timeout: 300
+        loop:
+          - VirtualComponentsHealthy
+          - RuntimeComponentsHealthy
+        tags: gardener
 
-    - name: Wait until seed is ready
-      kubernetes.core.k8s_info:
-        api_version: "core.gardener.cloud/v1beta1"
-        kind: Seed
-        name: "{{ metal_control_plane_stage_name }}"
-        kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}"
-        wait: yes
-        wait_condition:
-          reason: GardenletReady
-          status: "True"
-          type: GardenletReady
-        wait_timeout: 300
+      - name: Wait until seed is ready
+        kubernetes.core.k8s_info:
+          api_version: "core.gardener.cloud/v1beta1"
+          kind: Seed
+          name: "{{ metal_control_plane_stage_name }}"
+          kubeconfig: "{{ virtual_garden_kubeconfig }}"
+          wait: yes
+          wait_condition:
+            reason: GardenletReady
+            status: "True"
+            type: GardenletReady
+          wait_timeout: 300
+        tags: gardener