Migrate Istio to Flux managed Helm releases

- replace Istio operator with Flux managed Helm releases
- install Istio in stages using Flux dependencies between istio-base, istiod and istio-gateway charts
- use Flux variable substitutions to set the version for Istio control plane upgrades
- replace Istio Prometheus manifests with Flagger Prometheus
- adapt the e2e tests and set the Istio Gateway to NodePort, so it can run in Kubernetes Kind

Signed-off-by: Stefan Prodan <[email protected]>

Author: stefanprodan

.github/workflows/e2e.yaml

@@ -20,6 +20,9 @@ jobs:
           version: v0.11.1
       - name: Install Flux in Kubernetes Kind
         run: flux install
+      - name: Set Istio Gateway service type
+        run: |
+          kubectl -n flux-system create cm istio-version --from-literal=service=NodePort
       - name: Setup cluster reconciliation
         run: |
           flux create source git flux-system \
@@ -30,8 +33,8 @@ jobs:
           --path=./clusters/my-cluster
       - name: Verify cluster reconciliation
         run: |
-          kubectl -n flux-system wait kustomization/istio-operator --for=condition=ready --timeout=2m
           kubectl -n flux-system wait kustomization/istio-system --for=condition=ready --timeout=2m
+          kubectl -n flux-system wait kustomization/istio-gateway --for=condition=ready --timeout=2m
           kubectl -n flux-system wait kustomization/apps --for=condition=ready --timeout=2m
           kubectl -n prod wait canary/frontend --for=condition=promoted --timeout=1m
           kubectl -n prod rollout status deployment/frontend --timeout=1m

apps/backend/hpa.yaml

@@ -8,7 +8,7 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: backend
-  minReplicas: 2
+  minReplicas: 1
   maxReplicas: 4
   metrics:
   - type: Resource

apps/frontend/canary.yaml

@@ -22,10 +22,10 @@ spec:
     port: 9898
     # Istio gateways (optional)
     gateways:
-    - istio-system/public-gateway
+      - istio-system/public-gateway
     # Istio virtual service host names (optional)
     hosts:
-    - "*"
+      - "*"
   analysis:
     # schedule interval (default 60s)
     interval: 15s
@@ -72,4 +72,4 @@ spec:
         timeout: 15s
         url: http://flagger-loadtester.prod/
         metadata:
-          cmd: "hey -z 1m -q 10 -c 2 -H 'Cookie: type=insider' http://frontend.prod:9898/"
+          cmd: "hey -z 1m -q 10 -c 2 -H 'Cookie: type=insider' http://istio-gateway.istio-system/"

apps/frontend/deployment.yaml

@@ -63,5 +63,5 @@ spec:
             cpu: 2000m
             memory: 512Mi
           requests:
-            cpu: 100m
-            memory: 64Mi
+            cpu: 10m
+            memory: 16Mi

apps/frontend/hpa.yaml

@@ -8,7 +8,7 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: frontend
-  minReplicas: 2
+  minReplicas: 1
   maxReplicas: 4
   metrics:
   - type: Resource

apps/loadtest/deployment.yaml

@@ -19,7 +19,7 @@ spec:
     spec:
       containers:
       - name: loadtester
-        image: weaveworks/flagger-loadtester:0.13.0
+        image: ghcr.io/fluxcd/flagger-loadtester:0.22.0
         imagePullPolicy: IfNotPresent
         ports:
         - name: http

clusters/my-cluster/istio-version.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: istio-version
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: merge
+data:
+  version: 1.13.3

clusters/my-cluster/istio.yaml

@@ -1,52 +1,22 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
-kind: Kustomization
-metadata:
-  name: istio-operator
-  namespace: flux-system
-spec:
-  interval: 10m0s
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./istio/operator
-  prune: true
-  validation: client
-  healthChecks:
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: istio-operator
-      namespace: istio-operator
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
 kind: Kustomization
 metadata:
   name: istio-system
   namespace: flux-system
 spec:
-  dependsOn:
-    - name: istio-operator
   interval: 10m0s
+  path: ./istio/system
+  prune: true
+  wait: true
   sourceRef:
     kind: GitRepository
     name: flux-system
-  path: ./istio/system
-  prune: true
-  validation: client
-  healthChecks:
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: istiod
-      namespace: istio-system
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: prometheus
-      namespace: istio-system
-    - apiVersion: apps/v1
-      kind: Deployment
-      name: flagger
-      namespace: istio-system
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: istio-version
 ---
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
 kind: Kustomization
 metadata:
   name: istio-gateway
@@ -55,9 +25,9 @@ spec:
   dependsOn:
     - name: istio-system
   interval: 10m0s
+  path: ./istio/gateway
+  prune: true
+  wait: true
   sourceRef:
     kind: GitRepository
     name: flux-system
-  path: ./istio/gateway
-  prune: true
-  validation: client

istio/gateway/flagger-metrics.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   provider:
     type: prometheus
-    address: http://prometheus.istio-system:9090
+    address: http://flagger-prometheus.istio-system:9090
   query: |
     100 - sum(
         rate(
@@ -38,7 +38,7 @@ metadata:
 spec:
   provider:
     type: prometheus
-    address: http://prometheus.istio-system:9090
+    address: http://flagger-prometheus.istio-system:9090
   query: |
     histogram_quantile(
         0.99,

istio/gateway/public-gateway.yaml

@@ -1,11 +1,11 @@
-apiVersion: networking.istio.io/v1alpha3
+apiVersion: networking.istio.io/v1beta1
 kind: Gateway
 metadata:
   name: public-gateway
   namespace: istio-system
 spec:
   selector:
-    istio: ingressgateway
+    istio: gateway
   servers:
     - port:
         number: 80