Added observability tests.

Author: pantierra

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
 
   integration-tests:
     name: Integration tests
-    needs: fast-checks
+    # needs: fast-checks
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     steps:
@@ -89,6 +89,7 @@ jobs:
           echo "=== eoAPI Deployment ==="
           export RELEASE_NAME="${RELEASE_NAME}"
           export PGO_VERSION="${{ env.PGO_VERSION }}"
+          export GITHUB_SHA="${{ github.sha }}"
           export CI_MODE=true
 
           # Deploy using consolidated script with CI mode
@@ -109,7 +110,7 @@ jobs:
         run: |
           ./scripts/debug-deployment.sh
 
-      - name: Cleanup
+      - name: Cleanup integration test
         if: always()
         run: |
           helm uninstall "$RELEASE_NAME" -n eoapi || true
@@ -158,3 +159,122 @@ jobs:
             head -1 "$file" | grep -q "^---$" || { echo "❌ Missing frontmatter: $file"; missing=1; }
           done
           exit $missing
+
+  observability-tests:
+    name: Observability tests
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    # needs: integration-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Start K3s cluster
+        uses: jupyterhub/action-k3s-helm@v4
+        with:
+          k3s-channel: latest
+          helm-version: ${{ env.HELM_VERSION }}
+          metrics-enabled: false
+          docker-enabled: true
+
+      - name: Set release name
+        run: echo "RELEASE_NAME=eoapi-$(echo "${{ github.sha }}" | cut -c1-8)" >> "$GITHUB_ENV"
+
+      - name: Wait for K3s readiness
+        run: |
+          echo "=== Waiting for K3s cluster to be ready ==="
+
+          # The action already sets up kubectl context, just verify it works
+          kubectl cluster-info
+          kubectl get nodes
+
+          # Wait for core components
+          kubectl wait --for=condition=Ready pod -l k8s-app=kube-dns -n kube-system --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=traefik -n kube-system --timeout=300s
+
+          # Verify Traefik CRDs
+          timeout=300; counter=0
+          for crd in "middlewares.traefik.io" "ingressroutes.traefik.io"; do
+            while [ $counter -lt $timeout ] && ! kubectl get crd "$crd" &>/dev/null; do
+              sleep 3; counter=$((counter + 3))
+            done
+            [ $counter -ge $timeout ] && { echo "❌ Timeout waiting for $crd"; exit 1; }
+          done
+
+          echo "✅ K3s cluster ready"
+
+      - name: Deploy eoAPI with monitoring
+        run: |
+          echo "=== Deploying eoAPI with monitoring stack ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          export PGO_VERSION="${{ env.PGO_VERSION }}"
+          export GITHUB_SHA="${{ github.sha }}"
+          export CI_MODE=true
+          export OBSERVABILITY_MODE=true
+
+          # Deploy using consolidated script with observability mode enabled
+          ./scripts/deploy.sh --ci
+
+      - name: Wait for monitoring stack to be ready
+        run: |
+          echo "=== Waiting for monitoring components ==="
+
+          # Wait for metrics-server first (required for HPA)
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=metrics-server -n eoapi --timeout=300s || echo "metrics-server not ready"
+
+          # Wait for Prometheus server
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/component=server,app.kubernetes.io/name=prometheus -n eoapi --timeout=300s || echo "Prometheus server not ready"
+
+          # Wait for Grafana
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=grafana -n eoapi --timeout=300s || echo "Grafana not ready"
+
+          # Wait for prometheus-adapter
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-adapter -n eoapi --timeout=300s || echo "prometheus-adapter not ready"
+
+          # Give time for HPA to be created and configured
+          echo "=== Waiting for HPA creation ==="
+          sleep 60
+
+          echo "=== Final monitoring stack status ==="
+          kubectl get pods -n eoapi | grep -E "(prometheus|grafana|metrics-server)" || true
+          kubectl get hpa -n eoapi || echo "No HPA resources found yet"
+
+      - name: Validate core eoAPI services
+        run: |
+          echo "=== Validating core eoAPI services ==="
+
+          # Wait for core application pods to be ready
+          kubectl wait --for=condition=Ready pod -l app="$RELEASE_NAME"-stac -n eoapi --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app="$RELEASE_NAME"-raster -n eoapi --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app="$RELEASE_NAME"-vector -n eoapi --timeout=300s
+
+          echo "✅ Core eoAPI services are ready"
+
+      - name: Run observability tests
+        run: |
+          echo "=== Running observability test suite ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          export NAMESPACE="eoapi"
+
+          # Install python dependencies for testing
+          python -m pip install --upgrade pip
+          pip install pytest requests psycopg2-binary
+
+          # Run observability tests
+          python -m pytest .github/workflows/tests/test_observability.py -v --tb=short
+
+          # Run autoscaling tests
+          python -m pytest .github/workflows/tests/test_autoscaling.py -v --tb=short -m "not slow"
+
+      - name: Debug observability stack on failure
+        if: failure()
+        run: |
+          ./scripts/debug-deployment.sh
+
+      - name: Cleanup observability test
+        if: always()
+        run: |
+          helm uninstall "$RELEASE_NAME" -n eoapi || true
+          kubectl delete namespace eoapi || true