Merge branch 'main' into pipeline/Argocd

Author: USHER-PB

.github/scripts/import-existing-resources.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
-set -euo pipefail
+set -uo pipefail
 
 # Import existing Kubernetes resources into Terraform state
 # This prevents conflicts when deploying to an existing cluster
 
 NAMESPACE="${NAMESPACE:-observability}"
 REPORT_FILE="import-report.json"
+FAILURE_COUNT=0
+FAILED_OPERATIONS=()
 
 echo "🔍 Scanning for existing resources to import..."
 
@@ -31,7 +33,8 @@ cleanup_conflicting_resources() {
 
   # Get all resources of this type that match any of the keywords
   local pattern=$(echo "$keywords" | sed 's/,/|/g')
-  local resources=$(kubectl get "$resource_type" -o name | grep -E "$pattern" || true)
+  local resources
+  resources=$(kubectl get "$resource_type" -o name 2>/dev/null | grep -E "$pattern" || true)
 
   for res in $resources; do
     # Get owner using multiple methods
@@ -44,20 +47,28 @@ cleanup_conflicting_resources() {
 
     if [ -n "$ns_owner" ] && [ "$ns_owner" != "$NAMESPACE" ]; then
       echo "    ⚠️  CONFLICT: $res owned by '$ns_owner'. Deleting..."
-      kubectl delete "$res" --ignore-not-found --timeout=30s
+      if ! kubectl delete "$res" --ignore-not-found --timeout=30s 2>/dev/null; then
+        echo "    ❌ Failed to delete $res (continuing anyway)"
+        FAILURE_COUNT=$((FAILURE_COUNT+1))
+        FAILED_OPERATIONS+=("cleanup: kubectl delete $res")
+      fi
       deleted_any=true
     elif [ -z "$ns_owner" ]; then
       # Special case: Resource exists but no Helm owner. 
       # If it matches our exact release names, it's a "zombie" resource from a failed/partial manual install
       if [[ "$res" =~ monitoring-loki|monitoring-mimir|monitoring-tempo|monitoring-grafana|monitoring-prometheus ]]; then
         echo "    ⚠️  ZOMBIE RESOURCE: $res has no owner but matches stack pattern. Deleting to ensure clean install..."
-        kubectl delete "$res" --ignore-not-found --timeout=30s
+        if ! kubectl delete "$res" --ignore-not-found --timeout=30s 2>/dev/null; then
+          echo "    ❌ Failed to delete zombie resource $res (continuing anyway)"
+          FAILURE_COUNT=$((FAILURE_COUNT+1))
+          FAILED_OPERATIONS+=("cleanup: kubectl delete zombie $res")
+        fi
         deleted_any=true
       fi
     fi
   done
 
-  [ "$deleted_any" = true ] && sleep 5
+  [ "$deleted_any" = true ] && sleep 5 || true
 }
 
 # Deep Scan for all LGTM related cluster-scoped components
@@ -84,27 +95,34 @@ import_resource() {
     return 0
   else
     if grep -q "Resource already managed" /tmp/import.log; then
-      echo "    ℹ️  Already managed by Terraform"
+      echo "    ✅ Already managed by Terraform"
       jq --arg addr "$tf_address" --arg reason "already_managed" \
-        '.skipped += [{"address": $addr, "reason": $reason}]' \
+        '.imports += [{"address": $addr, "reason": $reason}]' \
         "$REPORT_FILE" > /tmp/report.tmp && mv /tmp/report.tmp "$REPORT_FILE"
+      return 0  # Return success instead of failure
     else
       echo "    ⚠️  Import failed (resource may not exist)"
       jq --arg addr "$tf_address" --arg error "$(cat /tmp/import.log | tail -5)" \
         '.errors += [{"address": $addr, "error": $error}]' \
         "$REPORT_FILE" > /tmp/report.tmp && mv /tmp/report.tmp "$REPORT_FILE"
+      return 1
     fi
-    return 1
   fi
 }
 
 # Check if namespace exists
 if kubectl get namespace "$NAMESPACE" &>/dev/null; then
   echo "📂 Found existing namespace: $NAMESPACE"
-  import_resource \
-    "kubernetes_namespace.observability" \
-    "$NAMESPACE" \
-    "Namespace: $NAMESPACE"
+  
+  # Check if already in Terraform state
+  if terraform state list | grep -q "kubernetes_namespace.observability"; then
+    echo "  ℹ️  Namespace already managed by Terraform - skipping import"
+  else
+    import_resource \
+      "kubernetes_namespace.observability" \
+      "$NAMESPACE" \
+      "Namespace: $NAMESPACE"
+  fi
 else
   echo "  ℹ️  Namespace $NAMESPACE does not exist (will be created)"
 fi
@@ -175,6 +193,30 @@ if [ "${CLOUD_PROVIDER:-}" == "gke" ]; then
   fi
 fi
 
+# ── Grafana Imports ─────────────────────────────────────────────────
+# Global datasources and other core Grafana resources would be imported here.
+# NOTE: Tenant teams, datasources, and folders are managed dynamically
+# by the grafana-team-sync CronJob, so they are NOT imported into Terraform.
+# ─────────────────────────────────────────────────────────────────────
+
+echo "📈 Scanning for existing Grafana resources to import into state..."
+
+if [ -n "${GRAFANA_URL:-}" ] && [ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]; then
+  GRAFANA_AUTH="admin:${GRAFANA_ADMIN_PASSWORD}"
+  
+  # Test Grafana connectivity first
+  if ! curl -sf --user "$GRAFANA_AUTH" "${GRAFANA_URL}/api/health" >/dev/null 2>&1; then
+    echo "    ⚠️  Cannot reach Grafana API at ${GRAFANA_URL} - skipping Grafana imports"
+    FAILURE_COUNT=$((FAILURE_COUNT+1))
+    FAILED_OPERATIONS+=("grafana: API unreachable at ${GRAFANA_URL}")
+  else
+    echo "  ℹ️  Grafana API is reachable at ${GRAFANA_URL}"
+    # Future global Grafana resource imports can be added here
+  fi
+else
+  echo "⏭️  Skipping Grafana imports: GRAFANA_URL or GRAFANA_ADMIN_PASSWORD not set"
+fi
+
 # Summary
 echo ""
 echo "📊 Import Summary:"
@@ -186,6 +228,19 @@ echo "  ✅ Imported: $IMPORTED"
 echo "  ⏭️  Skipped: $SKIPPED"
 echo "  ❌ Errors: $ERRORS"
 
+# Report any failures that occurred during execution
+if [ "$FAILURE_COUNT" -gt 0 ]; then
+  echo ""
+  echo "⚠️  Failure Summary: $FAILURE_COUNT operation(s) failed but script continued"
+  echo "Failed operations:"
+  for op in "${FAILED_OPERATIONS[@]}"; do
+    echo "  - $op"
+  done
+  echo ""
+  echo "ℹ️  These failures were logged but did not prevent other imports from executing."
+  echo "ℹ️  Terraform apply will proceed and create any resources that failed to import."
+fi
+
 echo ""
 echo "📄 Full report saved to: $REPORT_FILE"
 cat "$REPORT_FILE" | jq '.'

.github/scripts/smoke-tests.sh

@@ -102,6 +102,7 @@ TIMESTAMP=$(date +%s)000000000
 TRACE_ID=$(uuidgen | tr -d '-')
 
 LOKI_PUSH_RESPONSE=$(curl -s -X POST "$LOKI_ENDPOINT/loki/api/v1/push" \
+  -H "X-Scope-OrgID: default" \
   -H "Content-Type: application/json" \
   -d '{
     "streams": [
@@ -140,6 +141,7 @@ START_TIME_LOKI=$(($(date +%s) - 300))
 for i in {1..6}; do
   sleep 5
   LOKI_QUERY_RESPONSE=$(curl -s -G "$LOKI_ENDPOINT/loki/api/v1/query_range" \
+    -H "X-Scope-OrgID: default" \
     --data-urlencode 'query={job="smoke-test"}' \
     --data-urlencode "start=$START_TIME_LOKI" \
     --data-urlencode 'limit=10' || echo "FAILED")
@@ -185,6 +187,7 @@ EOF
 )
 
 MIMIR_PUSH_RESPONSE=$(echo "$MIMIR_PUSH" | curl -s -X POST "$MIMIR_ENDPOINT/api/v1/push" \
+  -H "X-Scope-OrgID: default" \
   -H "Content-Type: application/x-protobuf" \
   -H "X-Prometheus-Remote-Write-Version: 0.1.0" \
   --data-binary @- || echo "FAILED")
@@ -207,6 +210,7 @@ for i in {1..12}; do
   # Instead of a manual push which is hard with curl, we verify that Prometheus is successfully
   # remote-writing its own metrics to Mimir. We look for any metric starting with 'prometheus_'
   MIMIR_QUERY_RESPONSE=$(curl -s -G "$MIMIR_ENDPOINT/prometheus/api/v1/query" \
+    -H "X-Scope-OrgID: default" \
     --data-urlencode 'query={__name__=~"prometheus_.*"}' || echo "FAILED")
 
   if [[ "$MIMIR_QUERY_RESPONSE" != "FAILED" ]] && echo "$MIMIR_QUERY_RESPONSE" | jq -e '.data.result | length > 0' >/dev/null 2>&1; then
@@ -306,6 +310,7 @@ EOF
 )
 
 TEMPO_PUSH_RESPONSE=$(echo "$TEMPO_TRACE" | curl -s -X POST "$TEMPO_INGEST_ENDPOINT/v1/traces" \
+  -H "X-Scope-OrgID: default" \
   -H "Content-Type: application/json" \
   -d @- || echo "FAILED")
 
@@ -324,7 +329,7 @@ TEMPO_QUERY_SUCCESS=false
 # Try for up to 60 seconds for Tempo as tracing can be slower to index
 for i in {1..12}; do
   sleep 5
-  TEMPO_QUERY=$(curl -s "$TEMPO_QUERY_ENDPOINT/api/traces/${TRACE_ID}" || echo "FAILED")
+  TEMPO_QUERY=$(curl -s -H "X-Scope-OrgID: default" "$TEMPO_QUERY_ENDPOINT/api/traces/${TRACE_ID}" || echo "FAILED")
 
   if [[ "$TEMPO_QUERY" != "FAILED" ]] && echo "$TEMPO_QUERY" | jq -e '.batches | length > 0' >/dev/null 2>&1; then
     record_test "tempo" "query_trace" "PASS" "Successfully retrieved trace"
@@ -395,6 +400,87 @@ done
 
 
 
+
+#=============================================================================
+# MULTI-TENANCY ISOLATION TESTS
+# Validates that tenant data is fully isolated — webank cannot see default
+# data, and default cannot see webank data.
+# Acceptance Criteria: "Team A cannot view Team B's logs in Loki/Mimir/Tempo"
+#=============================================================================
+echo ""
+echo "🔒 Testing Multi-Tenancy Isolation..."
+
+ISOLATION_OK=true
+
+# --- Loki Isolation ---
+echo "  📝 [Loki] Pushing a secret log to 'webank' tenant..."
+ISOLATION_TIMESTAMP=$(date +%s)000000000
+WEBANK_SECRET="WEBANK-ONLY-SECRET-$(uuidgen)"
+
+curl -s -X POST "$LOKI_ENDPOINT/loki/api/v1/push" \
+  -H "X-Scope-OrgID: webank" \
+  -H "Content-Type: application/json" \
+  -d "{\"streams\":[{\"stream\":{\"job\":\"isolation-test\"},\"values\":[[\"$ISOLATION_TIMESTAMP\",\"$WEBANK_SECRET\"]]}" \
+  > /dev/null
+
+sleep 6
+
+# Query as 'default' — must NOT see the webank secret
+ISOLATION_AS_DEFAULT=$(curl -s -G "$LOKI_ENDPOINT/loki/api/v1/query_range" \
+  -H "X-Scope-OrgID: default" \
+  --data-urlencode 'query={job="isolation-test"}' \
+  --data-urlencode "start=$(($(date +%s) - 60))" \
+  --data-urlencode 'limit=10' || echo "FAILED")
+
+if echo "$ISOLATION_AS_DEFAULT" | grep -q "$WEBANK_SECRET"; then
+  record_test "isolation" "loki_cross_tenant_leak" "FAIL" "CRITICAL: default tenant CAN see webank data — isolation is BROKEN"
+  echo "    ❌ CRITICAL: Loki isolation FAILED — default tenant sees webank data!"
+  ISOLATION_OK=false
+else
+  record_test "isolation" "loki_cross_tenant_leak" "PASS" "default tenant cannot see webank data"
+  echo "    ✅ Loki: default tenant cannot see webank data"
+fi
+
+# Query as 'webank' — MUST see its own secret
+ISOLATION_AS_WEBANK=$(curl -s -G "$LOKI_ENDPOINT/loki/api/v1/query_range" \
+  -H "X-Scope-OrgID: webank" \
+  --data-urlencode 'query={job="isolation-test"}' \
+  --data-urlencode "start=$(($(date +%s) - 60))" \
+  --data-urlencode 'limit=10' || echo "FAILED")
+
+if echo "$ISOLATION_AS_WEBANK" | grep -q "$WEBANK_SECRET"; then
+  record_test "isolation" "loki_tenant_reads_own" "PASS" "webank tenant can read its own logs"
+  echo "    ✅ Loki: webank tenant can read its own logs"
+else
+  record_test "isolation" "loki_tenant_reads_own" "FAIL" "webank tenant cannot read its own logs"
+  echo "    ❌ Loki: webank tenant cannot read its own logs"
+  ISOLATION_OK=false
+fi
+
+# --- Mimir Isolation ---
+echo "  📊 [Mimir] Checking metric namespace isolation..."
+
+# Query a metric as 'webank' — it must not see 'prometheus_*' metrics
+# (those are shipped by Prometheus under the 'default' tenant)
+WEBANK_SEES_DEFAULT=$(curl -s -G "$MIMIR_ENDPOINT/prometheus/api/v1/query" \
+  -H "X-Scope-OrgID: webank" \
+  --data-urlencode 'query={__name__=~"prometheus_.*"}' || echo "FAILED")
+
+if [[ "$WEBANK_SEES_DEFAULT" != "FAILED" ]] && echo "$WEBANK_SEES_DEFAULT" | jq -e '.data.result | length > 0' > /dev/null 2>&1; then
+  record_test "isolation" "mimir_cross_tenant_leak" "FAIL" "CRITICAL: webank tenant sees prometheus_* metrics from 'default' tenant"
+  echo "    ❌ CRITICAL: Mimir isolation FAILED — webank sees default tenant metrics!"
+  ISOLATION_OK=false
+else
+  record_test "isolation" "mimir_cross_tenant_leak" "PASS" "webank tenant cannot see default tenant metrics"
+  echo "    ✅ Mimir: webank tenant cannot see default tenant metrics"
+fi
+
+if [ "$ISOLATION_OK" = true ]; then
+  echo "  🎉 All isolation tests PASSED — multi-tenancy is working correctly"
+else
+  echo "  ❌ Isolation tests FAILED — multi-tenancy is NOT properly enforced"
+fi
+
 #=============================================================================
 # INTEGRATION TEST
 #=============================================================================