chore: harden deployment and tracing

haasonsaas · haasonsaas · commit 0fcc5b477bac · 2025-10-18T10:40:59.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,11 +22,13 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
+          cache: true
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
 
       - name: Install Go tools
         run: |
@@ -35,6 +37,7 @@ jobs:
 
       - name: Install Python tools
         run: |
+          pip install --upgrade pip
           pip install black flake8 isort mypy
 
       - name: Check Go formatting
@@ -84,14 +87,17 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
+          cache: true
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
 
       - name: Install Python dependencies
         run: |
+          pip install --upgrade pip
           pip install -r app/requirements.txt
           pip install pytest pytest-cov
 
diff --git a/cmd/inventory/main.go b/cmd/inventory/main.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/EvalOps/keep/pkg/secrets"
+	"github.com/EvalOps/keep/pkg/telemetry"
 	serverpkg "github.com/EvalOps/keep/services/inventory/server"
 )
 
@@ -33,12 +34,22 @@ func main() {
 		RequireMTLS: envOrDefault("INVENTORY_REQUIRE_MTLS", "false") == "true",
 	}
 
+	ctx := context.Background()
+	if err := telemetry.Init(ctx, telemetry.Config{
+		Endpoint:    os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),
+		Insecure:    os.Getenv("OTEL_EXPORTER_OTLP_INSECURE") == "true",
+		ServiceName: "inventory",
+		Environment: envOrDefault("APP_ENV", "development"),
+	}); err != nil {
+		log.Printf("telemetry init failed: %v", err)
+	}
+
 	srv, err := serverpkg.NewServer(cfg)
 	if err != nil {
 		log.Fatalf("init inventory: %v", err)
 	}
 
-	if err := srv.Start(context.Background()); err != nil {
+	if err := srv.Start(ctx); err != nil {
 		log.Fatalf("inventory exit: %v", err)
 	}
 }
diff --git a/deploy/kubernetes/app-deployment.yaml b/deploy/kubernetes/app-deployment.yaml
@@ -15,5 +15,32 @@ spec:
       containers:
         - name: keep-app
           image: ghcr.io/example/app:latest
+          env:
+            - name: APP_ENV
+              valueFrom:
+                configMapKeyRef:
+                  name: keep-config
+                  key: APP_ENV
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: "http://otel-collector:4318"
           ports:
             - containerPort: 5000
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 5000
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 5000
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "64Mi"
+            limits:
+              cpu: "250m"
+              memory: "128Mi"
diff --git a/deploy/kubernetes/authz-deployment.yaml b/deploy/kubernetes/authz-deployment.yaml
@@ -15,5 +15,42 @@ spec:
       containers:
         - name: authz
           image: ghcr.io/example/authz:latest
+          env:
+            - name: OPA_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: keep-config
+                  key: OPA_URL
+            - name: INVENTORY_API
+              valueFrom:
+                configMapKeyRef:
+                  name: keep-config
+                  key: INVENTORY_API
+            - name: GOOGLE_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: keep-secrets
+                  key: google_client_id
           ports:
             - containerPort: 8443
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8443
+            scheme: HTTPS
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8443
+            scheme: HTTPS
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: "150m"
+              memory: "128Mi"
+            limits:
+              cpu: "400m"
+              memory: "256Mi"
diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: keep-config
+data:
+  OPA_URL: http://opa:8181
+  INVENTORY_API: http://inventory:8080
+  APP_ENV: production
diff --git a/deploy/kubernetes/inventory-deployment.yaml b/deploy/kubernetes/inventory-deployment.yaml
@@ -15,5 +15,32 @@ spec:
       containers:
         - name: inventory
           image: ghcr.io/example/inventory:latest
+          env:
+            - name: INVENTORY_ADDR
+              value: ":8080"
+            - name: APP_ENV
+              valueFrom:
+                configMapKeyRef:
+                  name: keep-config
+                  key: APP_ENV
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "128Mi"
+            limits:
+              cpu: "300m"
+              memory: "256Mi"
           ports:
             - containerPort: 8080
diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml
@@ -2,3 +2,5 @@ resources:
   - authz-deployment.yaml
   - inventory-deployment.yaml
   - app-deployment.yaml
+  - configmap.yaml
+  - secret.yaml
diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: keep-secrets
+type: Opaque
+stringData:
+  google_client_id: ""
diff --git a/go.mod b/go.mod
@@ -17,6 +17,8 @@ require (
 	github.com/jackc/pgx/v5 v5.7.6
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rs/zerolog v1.34.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0
+	go.opentelemetry.io/otel v1.38.0
 	tailscale.com v1.68.0
 )
 
diff --git a/pkg/telemetry/http.go b/pkg/telemetry/http.go
@@ -4,13 +4,16 @@ import (
 	"net/http"
 
 	"github.com/go-chi/chi/v5"
-	ogchi "go.opentelemetry.io/contrib/instrumentation/github.com/go-chi/chi/otelchi"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
 )
 
 // InstrumentRouter attaches otel middleware to chi routers.
 func InstrumentRouter(r chi.Router, service string) {
-	r.Use(ogchi.Middleware(service, ogchi.WithChiRoutes(true)))
+	r.Use(func(next http.Handler) http.Handler {
+		return otelhttp.NewHandler(next, service, otelhttp.WithTracerProvider(otel.GetTracerProvider()), otelhttp.WithPropagators(globalPropagator()))
+	})
 }
 
 // WrapClient ensures outgoing requests are traced.
@@ -22,6 +25,10 @@ func WrapClient(c *http.Client) *http.Client {
 	if base == nil {
 		base = http.DefaultTransport
 	}
-	c.Transport = otelhttp.NewTransport(base)
+	c.Transport = otelhttp.NewTransport(base, otelhttp.WithTracerProvider(otel.GetTracerProvider()), otelhttp.WithPropagators(globalPropagator()))
 	return c
 }
+
+func globalPropagator() propagation.TextMapPropagator {
+	return otel.GetTextMapPropagator()
+}
diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go
@@ -8,6 +8,7 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	"go.opentelemetry.io/otel/sdk/trace"
 )
@@ -57,6 +58,7 @@ func Init(ctx context.Context, cfg Config) error {
 		)
 
 		otel.SetTracerProvider(provider)
+		otel.SetTextMapPropagator(propagation.TraceContext{})
 		shutdownFn = provider.Shutdown
 	})
 
diff --git a/services/authz/server/server.go b/services/authz/server/server.go
@@ -81,7 +81,7 @@ func New(cfg Config) (*Server, error) {
 
 	client := telemetry.WrapClient(&http.Client{Timeout: 5 * time.Second})
 	retryCfg := retry.Config{
-		MaxElapsedTime: cfg.RetryMaxElapsed,
+		MaxElapsedTime:  cfg.RetryMaxElapsed,
 		InitialInterval: 200 * time.Millisecond,
 		Multiplier:      1.5,
 		MaxInterval:     2 * time.Second,
@@ -105,7 +105,7 @@ func New(cfg Config) (*Server, error) {
 		invClient:  invClient,
 		tsServer:   tsSrv,
 		tsListener: tailscaleListener,
-		retryCfg:  retryCfg,
+		retryCfg:   retryCfg,
 	}
 
 	// Create chi router with middleware
@@ -395,19 +395,19 @@ func (s *Server) evaluateOPA(ctx context.Context, claims map[string]any, deviceI
 	req.Header.Set("Content-Type", "application/json")
 
 	start := time.Now()
-   var resp *http.Response
-   var err error
-   retryErr := retry.Do(ctx, s.retryCfg, func() error {
-       resp, err = s.client.Do(req)
-       if err != nil {
-           return err
-       }
-       if resp.StatusCode >= 500 {
-           _ = resp.Body.Close()
-           return fmt.Errorf("opa temporary error: %d", resp.StatusCode)
-       }
-       return nil
-   })
+	var resp *http.Response
+	var err error
+	retryErr := retry.Do(ctx, s.retryCfg, func() error {
+		resp, err = s.client.Do(req)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode >= 500 {
+			_ = resp.Body.Close()
+			return fmt.Errorf("opa temporary error: %d", resp.StatusCode)
+		}
+		return nil
+	})
 	if retryErr != nil {
 		telemetry.RecordDependencyRequest(ctx, "authz", "opa", "evaluate", time.Since(start), "error")
 		return "", retryErr
@@ -489,19 +489,19 @@ func (s *Server) lookupDevice(ctx context.Context, deviceID string) map[string]a
 	}
 
 	start := time.Now()
-   var resp *http.Response
-   var err error
-   retryErr := retry.Do(ctx, s.retryCfg, func() error {
-       resp, err = s.invClient.Do(req)
-       if err != nil {
-           return err
-       }
-       if resp.StatusCode >= 500 {
-           _ = resp.Body.Close()
-           return fmt.Errorf("inventory temporary error: %d", resp.StatusCode)
-       }
-       return nil
-   })
+	var resp *http.Response
+	var err error
+	retryErr := retry.Do(ctx, s.retryCfg, func() error {
+		resp, err = s.invClient.Do(req)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode >= 500 {
+			_ = resp.Body.Close()
+			return fmt.Errorf("inventory temporary error: %d", resp.StatusCode)
+		}
+		return nil
+	})
 	if retryErr != nil {
 		telemetry.RecordDependencyRequest(ctx, "authz", "inventory", "lookup", time.Since(start), "error")
 		log.Printf("inventory request failed: %v", retryErr)
@@ -862,19 +862,19 @@ func (s *Server) verifyMFACode(ctx context.Context, sessionID, code string) (boo
 	req.Header.Set("Content-Type", "application/json")
 
 	start := time.Now()
-    var resp *http.Response
-    var err error
-    retryErr := retry.Do(ctx, s.retryCfg, func() error {
-        resp, err = s.client.Do(req)
-        if err != nil {
-            return err
-        }
-        if resp.StatusCode >= 500 {
-            _ = resp.Body.Close()
-            return fmt.Errorf("mfa temporary error: %d", resp.StatusCode)
-        }
-        return nil
-    })
+	var resp *http.Response
+	var err error
+	retryErr := retry.Do(ctx, s.retryCfg, func() error {
+		resp, err = s.client.Do(req)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode >= 500 {
+			_ = resp.Body.Close()
+			return fmt.Errorf("mfa temporary error: %d", resp.StatusCode)
+		}
+		return nil
+	})
 	if retryErr != nil {
 		telemetry.RecordDependencyRequest(ctx, "authz", "mfa", "verify", time.Since(start), "error")
 		return false, retryErr
diff --git a/services/inventory/server/server.go b/services/inventory/server/server.go

Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@ import (`
`7`	`7`	`"time"`
`8`	`8`
`9`	`9`	`"github.com/EvalOps/keep/pkg/secrets"`
	`10`	`+ "github.com/EvalOps/keep/pkg/telemetry"`
`10`	`11`	`serverpkg "github.com/EvalOps/keep/services/inventory/server"`
`11`	`12`	`)`
`12`	`13`
`@@ -33,12 +34,22 @@ func main() {`
`33`	`34`	`RequireMTLS: envOrDefault("INVENTORY_REQUIRE_MTLS", "false") == "true",`
`34`	`35`	`}`
`35`	`36`
	`37`	`+ ctx := context.Background()`
	`38`	`+ if err := telemetry.Init(ctx, telemetry.Config{`
	`39`	`+ Endpoint: os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"),`
	`40`	`+ Insecure: os.Getenv("OTEL_EXPORTER_OTLP_INSECURE") == "true",`
	`41`	`+ ServiceName: "inventory",`
	`42`	`+ Environment: envOrDefault("APP_ENV", "development"),`
	`43`	`+ }); err != nil {`
	`44`	`+ log.Printf("telemetry init failed: %v", err)`
	`45`	`+ }`
	`46`	`+`
`36`	`47`	`srv, err := serverpkg.NewServer(cfg)`
`37`	`48`	`if err != nil {`
`38`	`49`	`log.Fatalf("init inventory: %v", err)`
`39`	`50`	`}`
`40`	`51`
`41`		`- if err := srv.Start(context.Background()); err != nil {`
	`52`	`+ if err := srv.Start(ctx); err != nil {`
`42`	`53`	`log.Fatalf("inventory exit: %v", err)`
`43`	`54`	`}`
`44`	`55`	`}`
Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,8 @@ require (`
`17`	`17`	`github.com/jackc/pgx/v5 v5.7.6`
`18`	`18`	`github.com/prometheus/client_golang v1.23.2`
`19`	`19`	`github.com/rs/zerolog v1.34.0`
	`20`	`+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0`
	`21`	`+ go.opentelemetry.io/otel v1.38.0`
`20`	`22`	`tailscale.com v1.68.0`
`21`	`23`	`)`
`22`	`24`