Add canary release test to e2e

Signed-off-by: Stefan Prodan <[email protected]>

Author: stefanprodan

.github/workflows/e2e.yaml

@@ -31,21 +31,38 @@ jobs:
           --path=./clusters/my-cluster
       - name: Verify cluster reconciliation
         run: |
-          kubectl -n flux-system wait kustomization/istio-operator --for=condition=ready --timeout=1m
-          kubectl -n flux-system wait kustomization/istio-system --for=condition=ready --timeout=1m
-          kubectl -n flux-system wait kustomization/apps --for=condition=ready --timeout=1m
-          kubectl -n prod wait canary/frontend --for=condition=promoted
-          kubectl -n prod wait canary/backend --for=condition=promoted
-      - name: List canaries
+          kubectl -n flux-system wait kustomization/istio-operator --for=condition=ready --timeout=2m
+          kubectl -n flux-system wait kustomization/istio-system --for=condition=ready --timeout=2m
+          kubectl -n flux-system wait kustomization/apps --for=condition=ready --timeout=2m
+          kubectl -n prod wait canary/frontend --for=condition=promoted --timeout=1m
+          kubectl -n prod rollout status deployment/frontend --timeout=1m
+          kubectl -n prod wait canary/backend --for=condition=promoted --timeout=1m
+          kubectl -n prod rollout status deployment/backend --timeout=1m
+      - name: Test canary release
         run: |
-          kubectl -n prod get canaries
+          kubectl -n prod set image deployment/backend backend=stefanprodan/podinfo:5.0.1
+          echo '>>> Waiting for canary finalization'
+          retries=25
+          count=0
+          ok=false
+          until ${ok}; do
+              kubectl -n prod get canary/backend | grep 'Succeeded' && ok=true || ok=false
+              sleep 20
+              kubectl -n istio-system logs deployment/flagger --tail 1
+              count=$(($count + 1))
+              if [[ ${count} -eq ${retries} ]]; then
+                  echo "No more retries left"
+                  exit 1
+              fi
+          done
       - name: Debug failure
         if: failure()
         run: |
           kubectl -n flux-system get all
           kubectl -n flux-system logs deploy/source-controller
           kubectl -n flux-system logs deploy/kustomize-controller
           kubectl -n flux-system logs deploy/helm-controller
+          kubectl -n istio-system logs deployment/flagger
           kubectl -n istio-operator get all
           kubectl -n istio-system get all
           kubectl -n prod get all

apps/backend/canary.yaml

@@ -11,7 +11,7 @@ spec:
     name: backend
   # the maximum time in seconds for the canary deployment
   # to make progress before it is rollback (default 600s)
-  progressDeadlineSeconds: 60
+  progressDeadlineSeconds: 120
   # HPA reference (optional)
   autoscalerRef:
     apiVersion: autoscaling/v2beta1

apps/backend/deployment.yaml

@@ -62,5 +62,5 @@ spec:
             cpu: 2000m
             memory: 512Mi
           requests:
-            cpu: 100m
-            memory: 64Mi
+            cpu: 10m
+            memory: 16Mi