Adding connection checks and unit tests (#14)

Author: deep1725

docs/Agents.md

@@ -252,6 +252,61 @@ GITHUB_ACTIONS=true       # Simulate GitHub Actions
 - Shared utilities for log streaming and file operations
 - Centralized error state detection logic
 
+---
+
+## Session: 2026-02-24 — Connectivity Checks Overhaul
+
+### KUBECONFIG env var support
+- `LoadK8sConfig()` now uses `clientcmd.NewDefaultClientConfigLoadingRules()` instead of hardcoding `~/.kube/config`
+- Respects `KUBECONFIG` env var and colon-separated paths (same resolution order as `kubectl`)
+- File: `pkg/utils/kubeconfig.go`
+
+### Connectivity script extracted to `connectivity_check.py`
+- Python script moved out of the Go string literal into `pkg/debug/connectivity_check.py`
+- Embedded at build time via `//go:embed connectivity_check.py`
+- Namespace substituted by Go: `strings.ReplaceAll(script, "{ns}", namespace)`
+- `{ns}` in plain strings → replaced by Go; in f-strings → use `ns = '{ns}'` Python variable trick
+- File: `pkg/debug/connections.go`, `pkg/debug/connectivity_check.py`
+
+### Refactored connectivity checks with shared data structure
+Eliminated per-service copy-paste. Two helper functions cover all patterns:
+- `parse_url_service(name, env_var, default_port)` — single URL env var (Postgres, ClickHouse)
+- `parse_host_port_service(name, host_env, port_env, note)` — split HOST+PORT env vars (Redis, RabbitMQ)
+
+Adding a new service = one line in the `services` list.
+
+### Services now tested from inside the API pod
+
+| Service | Method | Source |
+|---|---|---|
+| Authz | TCP `:3592` | hardcoded service name |
+| Runners API | TCP `:8090` | hardcoded service name |
+| UI | TCP `:3000` | hardcoded service name |
+| Minio | HTTP GET `/minio/health/ready` | `minio-client.{ns}.svc.cluster.local:9000` |
+| ClickHouse replica-N | HTTP GET `/ping` | `chi-clickhouse-cluster-0-N.{ns}.svc.cluster.local:8123` |
+| PostgreSQL | TCP | `GALILEO_DATABASE_URL_READ` |
+| ClickHouse | TCP | `GALILEO_CLICKHOUSE_URL_READ_WRITE` |
+| Redis | TCP | `GALILEO_REDIS_HOST` + `GALILEO_REDIS_PORT` |
+| RabbitMQ | TCP | `GALILEO_RABBITMQ_HOST` + `GALILEO_RABBITMQ_PORT` |
+
+ClickHouse replicas are dynamically generated from `GALILEO_CLICKHOUSE_EXPECTED_REPLICAS` — mirrors what the `wait-deps` init container checks.
+
+### Infrastructure notes discovered
+- **Redis**: Azure Managed Redis (external) — env vars: `GALILEO_REDIS_HOST`, `GALILEO_REDIS_PORT` (10000), `GALILEO_REDIS_PROTOCOL` (`rediss+poll`), `GALILEO_REDIS_PASSWORD`, `GALILEO_REDIS_REQUIRE`
+- **RabbitMQ**: env vars `GALILEO_RABBITMQ_HOST`, `GALILEO_RABBITMQ_PORT` (5671 TLS / 5672 plain), `GALILEO_RABBITMQ_TLS_ENABLED`
+- **Minio**: PVC-based on Azure — check via HTTP health endpoint, not pod readiness
+- **ClickHouse Keeper**: API pod does not talk to Keeper directly. Keeper is checked via Go-side `checkDependencyHealth` only
+- **runner-large, runners-metrics-controller**: Worker pods with no Kubernetes Service — cannot be TCP-checked
+
+### `wait-deps` init container
+Waits for (in order):
+1. `minio-client.{ns}:9000/minio/health/ready` → HTTP 200
+2. `chi-clickhouse-cluster-0-{i}.{ns}:8123/ping` → HTTP 200 (for each replica)
+
+If stuck in `PodInitializing`, either Minio or ClickHouse (which depends on Keeper) is not ready.
+
+---
+
 ## Quick Reference
 
 ### Common Commands

pkg/debug/connections.go

@@ -1,6 +1,8 @@
 package debug
 
 import (
+	_ "embed"
+
 	"bufio"
 	"context"
 	"errors"
@@ -18,6 +20,9 @@ import (
 	"galileoctl/pkg/utils"
 )
 
+//go:embed connectivity_check.py
+var connectivityCheckScript string
+
 // ApiConnections runs the connectivity checks from an api pod inside the given namespace.
 func ApiConnections(namespace string, tailLines int) error {
 	if namespace == "" {
@@ -49,64 +54,8 @@ func ApiConnections(namespace string, tailLines int) error {
 
 	fmt.Printf("✅ Using pod: %s\n", pod)
 
-	// Run the same Python connectivity checks inside the pod
-	py := `import socket
-import os
-from urllib.parse import urlparse
-
-services = [
-    'rabbitmq-cluster.%s.svc.cluster.local',
-    'authz.%s.svc.cluster.local'
-]
-
-print('Starting connectivity tests...')
-print('')
-
-for svc in services:
-    print(f'Testing connectivity to {svc}...')
-    try:
-        socket.setdefaulttimeout(5)
-        result = socket.getaddrinfo(svc, None)
-        if result:
-            print(f'✅ SUCCESS: {svc} is reachable (DNS resolved)')
-        else:
-            print(f'❌ FAILED: {svc} DNS resolution failed')
-    except Exception as e:
-        print(f'❌ FAILED: {svc} is not reachable ({str(e)})')
-    print('')
-
-env_vars = [
-    'GALILEO_DATABASE_URL_READ',
-    'GALILEO_CLICKHOUSE_URL_READ_WRITE'
-]
-
-for env_var in env_vars:
-    url = os.getenv(env_var)
-    if url:
-        print(f'Testing connectivity to {env_var}...')
-        try:
-            parsed = urlparse(url)
-            host = parsed.hostname
-            port = parsed.port or (5432 if 'postgresql' in url else 9000)
-            print(f'  Connecting to {host}:{port}...')
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            sock.settimeout(5)
-            result = sock.connect_ex((host, port))
-            sock.close()
-            if result == 0:
-                print(f'✅ SUCCESS: {env_var} ({host}:{port}) is reachable')
-            else:
-                print(f'❌ FAILED: {env_var} ({host}:{port}) connection refused')
-        except Exception as e:
-            print(f'❌ FAILED: {env_var} connectivity test failed ({str(e)})')
-    else:
-        print(f'⚠️  WARNING: {env_var} environment variable not found')
-    print('')
-
-print('Connectivity tests completed.')
-`
-
-	py = fmt.Sprintf(py, namespace, namespace)
+	// Run connectivity checks inside the pod (script embedded from connectivity_check.py)
+	py := strings.ReplaceAll(connectivityCheckScript, "{ns}", namespace)
 
 	// Load Kubernetes config
 	cfg, client, err := utils.LoadK8sConfig()
@@ -328,6 +277,13 @@ func diagnoseLogs(logLines []string) string {
 		issues = append(issues, "❌ ClickHouse connection failed (check if ClickHouse pods are running)")
 	}
 
+	// Redis connection issues
+	if strings.Contains(allLogs, "redis") && (strings.Contains(allLogs, "connection refused") ||
+		strings.Contains(allLogs, "failed to connect") ||
+		strings.Contains(allLogs, "10000")) {
+		issues = append(issues, "❌ Redis connection failed (check Azure Managed Redis reachability)")
+	}
+
 	// RabbitMQ connection issues
 	if strings.Contains(allLogs, "rabbitmq") && (strings.Contains(allLogs, "connection refused") ||
 		strings.Contains(allLogs, "failed to connect") ||