Deploy envoy-gateway in prow clusters (gardener#5626)

* Deploy `envoy-gateway`

* Adapt `oauth2_proxy`

* Adapt `monitoring`

* Adapt `prow`

* Create dependencies to `envoy-gateway` for kustomizations which use `Gateway` and `envoy-gateway` objects

* Address PR review feedback

* Remove port 80 from trusted monitoring

Author: oliver-goetz

clusters/base/envoy-gateway-kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: envoy-gateway
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./deploy/envoy-gateway
+  prune: true
+  retryInterval: 2m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: ci-infra
+  wait: true

clusters/base/monitoring-kustomization.yaml

@@ -4,6 +4,8 @@ metadata:
   name: monitoring
   namespace: flux-system
 spec:
+  dependsOn:
+  - name: envoy-gateway
   interval: 30m
   prune: true
   retryInterval: 2m

clusters/prow-trusted/oauth2-proxy-kustomization.yaml

@@ -4,6 +4,8 @@ metadata:
   name: oauth2-proxy
   namespace: flux-system
 spec:
+  dependsOn:
+  - name: envoy-gateway
   interval: 30m
   path: deploy/oauth2-proxy
   prune: true

clusters/prow-trusted/prow-kustomization.yaml

@@ -4,6 +4,8 @@ metadata:
   name: prow
   namespace: flux-system
 spec:
+  dependsOn:
+  - name: envoy-gateway
   interval: 30m
   path: ./deploy/prow
   prune: true

deploy/envoy-gateway/envoy-gateway-helmrepo.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: envoy-gateway
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 30m
+  url: oci://docker.io/envoyproxy

deploy/envoy-gateway/envoy-gateway-namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: envoy-gateway-system

deploy/envoy-gateway/envoy-gateway-vpa.yaml

@@ -0,0 +1,37 @@
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  labels:
+    app.kubernetes.io/name: envoy-gateway
+  name: envoy-gateway
+  namespace: envoy-gateway-system
+spec:
+  resourcePolicy:
+    containerPolicies:
+    - containerName: '*'
+      controlledValues: RequestsOnly
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: envoy-gateway
+  updatePolicy:
+    updateMode: InPlaceOrRecreate
+---
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  labels:
+    app.kubernetes.io/name: envoy-gateway
+  name: envoy-prow
+  namespace: envoy-gateway-system
+spec:
+  resourcePolicy:
+    containerPolicies:
+    - containerName: '*'
+      controlledValues: RequestsOnly
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: envoy-prow
+  updatePolicy:
+    updateMode: InPlaceOrRecreate

deploy/envoy-gateway/envoy-gateway.yaml

@@ -0,0 +1,49 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: envoy-gateway
+  namespace: flux-system
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: gateway-helm
+      version: 'v1.6.x'
+      sourceRef:
+        kind: HelmRepository
+        name: envoy-gateway
+        namespace: flux-system
+      interval: 1m
+  releaseName: envoy-gateway
+  targetNamespace: envoy-gateway-system
+  values:
+    # Check https://gateway.envoyproxy.io/docs/install/gateway-helm-api/ for all available values.
+    config:
+      envoyGateway:
+        extensionApis:
+          enableBackend: true
+          enableEnvoyPatchPolicy: true
+
+    deployment:
+      replicas: 2
+      pod:
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: worker.gardener.cloud/system-components
+                  operator: In
+                  values:
+                  - "true"
+        topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: envoy-gateway
+              app.kubernetes.io/instance: envoy-gateway
+
+    podDisruptionBudget:
+      maxUnavailable: 1

deploy/envoy-gateway/envoy-proxy-configuration.yaml

@@ -0,0 +1,28 @@
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: high-availability-config
+  namespace: envoy-gateway-system
+spec:
+  ipFamily: DualStack
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        replicas: 3
+        pod:
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchExpressions:
+                  - key: worker.gardener.cloud/system-components
+                    operator: In
+                    values:
+                    - "true"
+          topologySpreadConstraints:
+          - maxSkew: 1
+            topologyKey: topology.kubernetes.io/zone
+            whenUnsatisfiable: DoNotSchedule
+      envoyPDB:
+        maxUnavailable: 1

deploy/envoy-gateway/gatewayclass.yaml

@@ -0,0 +1,11 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  name: prow
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: high-availability-config
+    namespace: envoy-gateway-system