K8sgpt poc (#439)

typeid · web-flow · commit 91e730d8740c · 2025-05-09T14:24:50.000Z
* Experimental POC: add k8sgpt permissions

* Set customer data access to true

* Feat(experimental): api-ErrorBudgetBurn AI input

* Fix linter

* Revert Makefile change
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,5 @@ dist
 .idea
 .vscode
 cad_testing
-e2e-suite.test
+e2e-suite.test
+payload
diff --git a/Makefile b/Makefile
@@ -19,7 +19,7 @@ help:  # Display this help
 
 ##@ Global:
 .PHONY: all
-all: interceptor cadctl  ## Generate, build, lint, test all subprojects
+all: interceptor cadctl ## Generate, build, lint, test all subprojects
 
 .PHONY: build
 build: build-interceptor build-cadctl ## Build all subprojects in this repository
@@ -49,7 +49,7 @@ lint-cadctl: install-linter ## Lint cadctl subproject
 	@echo
 	@echo "Linting cadctl..."
 	# Explicitly set GOROOT, see https://github.com/golangci/golangci-lint/issues/3107
-	GOROOT=/usr/lib/golang GOLANGCI_LINT_CACHE=$$(mktemp -d) $(GOPATH)/bin/golangci-lint run -c .golangci.yml
+	GOROOT=$$(go env GOROOT) GOLANGCI_LINT_CACHE=$$(mktemp -d) $(GOPATH)/bin/golangci-lint run -c .golangci.yml
 
 .PHONY: test-cadctl
 test-cadctl: check-go121-install ## Run automated tests for cadctl
@@ -72,7 +72,7 @@ lint-interceptor: install-linter ## Lint interceptor subproject
 	@echo
 	@echo "Linting interceptor..."
 	# Explicitly set GOROOT, see https://github.com/golangci/golangci-lint/issues/3107
-	cd interceptor && GOROOT=/usr/lib/golang GOLANGCI_LINT_CACHE=$$(mktemp -d) $(GOPATH)/bin/golangci-lint run -c ../.golangci.yml
+	cd interceptor && GOROOT=$$(go env GOROOT) GOLANGCI_LINT_CACHE=$$(mktemp -d) $(GOPATH)/bin/golangci-lint run -c ../.golangci.yml
 
 .PHONY: test-interceptor
 test-interceptor: check-go121-install check-jq-install build-interceptor ## Run unit tests for interceptor
diff --git a/README.md b/README.md
@@ -182,3 +182,5 @@ Grafana dashboard configmaps are stored in the [Dashboards](./dashboards/) direc
 For Red Hat employees, these environment variables can be found in the SRE-P vault.
 
 - `LOG_LEVEL`: refers to the CAD log level, if not set, the default is `info`. See
+
+- `CAD_HCM_AI_TOKEN`: required for requests to the ai model
diff --git a/go.mod b/go.mod
diff --git a/go.sum b/go.sum
diff --git a/interceptor/go.mod b/interceptor/go.mod
diff --git a/interceptor/go.sum b/interceptor/go.sum
diff --git a/openshift/template.yaml b/openshift/template.yaml
@@ -305,6 +305,11 @@ objects:
         value: ${CAD_EXPERIMENTAL_ENABLED}
       - name: LOG_LEVEL
         value: ${LOG_LEVEL}
+      - name: CAD_HCM_AI_TOKEN
+        valueFrom:
+          secretKeyRef:
+            key: token
+            name: srep-ai-sa-token
       envFrom:
       - secretRef:
           name: cad-ocm-client-secret
diff --git a/pkg/ai/k8sgpt/k8sgpt.go b/pkg/ai/k8sgpt/k8sgpt.go
@@ -0,0 +1,129 @@
+package k8sgpt
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	k8sgpt_ai "github.com/k8sgpt-ai/k8sgpt/pkg/ai"
+	"github.com/k8sgpt-ai/k8sgpt/pkg/analysis"
+	"github.com/k8sgpt-ai/k8sgpt/pkg/cache"
+	gptK8sClient "github.com/k8sgpt-ai/k8sgpt/pkg/kubernetes"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var model = "mistral-small-maas"
+
+func K8sGptAnalysis(k8sRestConfig *rest.Config) (string, error) {
+	ctrlClient, err := client.New(k8sRestConfig, client.Options{Scheme: runtime.NewScheme()})
+	if err != nil {
+		return "", errors.New("unable to init ctrlClient")
+	}
+	clientset := kubernetes.NewForConfigOrDie(k8sRestConfig)
+
+	client := &gptK8sClient.Client{CtrlClient: ctrlClient, Config: k8sRestConfig, Client: clientset}
+
+	aiToken := os.Getenv("CAD_HCM_AI_TOKEN")
+	if aiToken == "" {
+		return "", errors.New("could not find CAD_HCM_AI_TOKEN env")
+	}
+
+	aiClient := k8sgpt_ai.NewClient("openai")
+	aiProvider := &k8sgpt_ai.AIProvider{
+		Name:     "openai",
+		Model:    model,
+		BaseURL:  "https://mistral-small-maas-maas.apps.rosa.hcmaii01ue1.a9ro.p3.openshiftapps.com/v1", // TODO: Let's not hardcode this.
+		Password: aiToken,
+	}
+
+	if err = aiClient.Configure(aiProvider); err != nil {
+		return "", fmt.Errorf("unable to configure ai provider: %w", err)
+	}
+
+	cache, err := cache.GetCacheConfiguration()
+	if err != nil {
+		return "", fmt.Errorf("unable to get k8sgpt cache configuration: %w", err)
+	}
+	cache.DisableCache()
+
+	a := &analysis.Analysis{
+		Context:        context.Background(),
+		Filters:        []string{"Pod", "Deployment", "ReplicaSet", "PersistentVolumeClaim", "Service", "Ingress", "StatefulSet", "CronJob", "Node", "ValidatingWebhookConfiguration", "MutatingWebhookConfiguration"},
+		Client:         client,
+		Language:       "english",
+		Namespace:      "",
+		LabelSelector:  "",
+		Cache:          cache,
+		Explain:        true,
+		MaxConcurrency: 10,
+		WithDoc:        false,
+		WithStats:      false,
+		AIClient:       aiClient,
+	}
+
+	a.RunAnalysis()
+
+	var output string
+	anonymize := false
+	if err := a.GetAIResults(output, anonymize); err != nil {
+		return "", fmt.Errorf("unable to get ai results: %w", err)
+	}
+
+	return formatOutput(a)
+}
+
+func formatOutput(a *analysis.Analysis) (string, error) {
+	var output strings.Builder
+
+	output.WriteString("🤖🔧 AI Analysis Results 🔧🤖\n")
+	output.WriteString(fmt.Sprintf("Model: %s\n", model))
+	if len(a.Errors) != 0 {
+		output.WriteString("⚠️ Analysis failures: \n")
+		for _, aerror := range a.Errors {
+			output.WriteString(fmt.Sprintf("- %s\n", aerror))
+		}
+	}
+	if len(a.Results) == 0 {
+		output.WriteString("✅ No cluster problems detected\n")
+		return output.String(), nil
+	}
+	output.WriteString(fmt.Sprintf("🔍 %d cluster issues detected\n", len(a.Results)))
+	output.WriteString("================\n\n")
+
+	for _, result := range a.Results {
+		if result.Kind != "" {
+			output.WriteString(fmt.Sprintf("Kind: %s\n", result.Kind))
+		}
+
+		if result.Name != "" {
+			output.WriteString(fmt.Sprintf("Name: %s\n", result.Name))
+		}
+
+		if result.ParentObject != "" {
+			output.WriteString(fmt.Sprintf("ParentObject: %s\n", result.ParentObject))
+		}
+
+		if len(result.Error) > 0 {
+			output.WriteString("Issues:\n")
+			for _, err := range result.Error {
+				output.WriteString(fmt.Sprintf("- %s\n", err.Text))
+				if err.KubernetesDoc != "" {
+					output.WriteString(fmt.Sprintf("  Kubernetes Doc: %s\n", err.KubernetesDoc))
+				}
+			}
+		}
+
+		if result.Details != "" {
+			output.WriteString(fmt.Sprintf("Details: %s\n", result.Details))
+		}
+
+		output.WriteString("\n------------------------------------------------------------\n\n")
+	}
+
+	return output.String(), nil
+}
diff --git a/pkg/investigations/apierrorbudgetburn/README.md b/pkg/investigations/apierrorbudgetburn/README.md
@@ -0,0 +1,8 @@
+# apierrorbudgetburn Investigation
+
+POC Api-ErrorBudgetBurn investigation using k8sgpt.
+
+## Testing
+
+Refer to the [testing README](./testing/README.md) for instructions on testing this investigation
+
diff --git a/pkg/investigations/apierrorbudgetburn/apierrorbudgetburn.go b/pkg/investigations/apierrorbudgetburn/apierrorbudgetburn.go
@@ -0,0 +1,59 @@
+// Package apierrorbudgetburn contains the investigation for api-ErrorBudgetBurn alerts
+package apierrorbudgetburn
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/openshift/configuration-anomaly-detection/pkg/ai/k8sgpt"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	k8sclient "github.com/openshift/configuration-anomaly-detection/pkg/k8s"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+)
+
+type Investigation struct{}
+
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	k8sConfig, err := k8sclient.NewCfg(r.Cluster.ID(), r.OcmClient, r.Name)
+	if err != nil {
+		if errors.Is(err, k8sclient.ErrAPIServerUnavailable) {
+			return result, r.PdClient.EscalateIncidentWithNote("CAD was unable to access cluster's kube-api. Please investigate manually.")
+		}
+
+		return result, fmt.Errorf("unable to initialize k8s cli config: %w", err)
+	}
+	defer func() {
+		deferErr := k8sclient.Cleanup(r.Cluster.ID(), r.OcmClient, r.Name)
+		if deferErr != nil {
+			logging.Error(deferErr)
+			err = errors.Join(err, deferErr)
+		}
+	}()
+
+	analysis, err := k8sgpt.K8sGptAnalysis(k8sConfig)
+	if err != nil {
+		return result, fmt.Errorf("failed to run K8sGptAnalysis: %w", err)
+	}
+
+	return result, r.PdClient.EscalateIncidentWithNote(analysis)
+}
+
+func (c *Investigation) Name() string {
+	return "apierrorbudgetburn"
+}
+
+func (c *Investigation) Description() string {
+	return "POC Api-ErrorBudgetBurn investigation using k8sgpt."
+}
+
+func (c *Investigation) ShouldInvestigateAlert(alert string) bool {
+	return strings.Contains(alert, "api-ErrorBudgetBurn")
+}
+
+func (c *Investigation) IsExperimental() bool {
+	// This is an experimental investigation leveraging k8sgpt.
+	return true
+}
diff --git a/pkg/investigations/apierrorbudgetburn/metadata.yaml b/pkg/investigations/apierrorbudgetburn/metadata.yaml
@@ -0,0 +1,8 @@
+name: apierrorbudgetburn
+rbac:
+  roles: []
+  clusterRoleRules:
+    - verbs: ["get", "watch", "list"]
+      apiGroups: ["*"]
+      resources: ["*"]
+customerDataAccess: true
diff --git a/pkg/investigations/apierrorbudgetburn/testing/README.md b/pkg/investigations/apierrorbudgetburn/testing/README.md
@@ -0,0 +1,5 @@
+# Testing apierrorbudgetburn Investigation
+
+TODO:
+- Add a test script or test objects to this  directory for future maintainers to use
+- Edit this README file and add detailed instructions on how to use the script/objects to recreate the conditions for the investigation. Be sure to include any assumptions or prerequisites about the environment (disable hive syncsetting, etc)
diff --git a/pkg/investigations/registry.go b/pkg/investigations/registry.go
@@ -1,6 +1,7 @@
 package investigations
 
 import (
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/apierrorbudgetburn"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/ccam"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/chgm"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/clustermonitoringerrorbudgetburn"
@@ -13,6 +14,7 @@ import (
 
 // availableInvestigations holds all Investigation implementations.
 var availableInvestigations = []investigation.Investigation{
+	&apierrorbudgetburn.Investigation{},
 	&ccam.Investigation{},
 	&chgm.Investiation{},
 	&clustermonitoringerrorbudgetburn.Investigation{},
diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go
@@ -7,11 +7,27 @@ import (
 	"github.com/openshift/backplane-cli/pkg/cli/config"
 	bpremediation "github.com/openshift/backplane-cli/pkg/remediation"
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // New returns a Kubernetes client for the given cluster scoped to a given remediation's permissions.
 func New(clusterID string, ocmClient ocm.Client, remediation string) (client.Client, error) {
+	cfg, err := NewCfg(clusterID, ocmClient, remediation)
+	if err != nil {
+		return nil, err
+	}
+
+	scheme, err := initScheme()
+	if err != nil {
+		return nil, err
+	}
+
+	return client.New(cfg, client.Options{Scheme: scheme})
+}
+
+// New returns a the k8s rest config for the given cluster scoped to a given remediation's permissions.
+func NewCfg(clusterID string, ocmClient ocm.Client, remediation string) (*rest.Config, error) {
 	backplaneURL := os.Getenv("BACKPLANE_URL")
 	if backplaneURL == "" {
 		return nil, fmt.Errorf("could not create new k8sclient: missing environment variable BACKPLANE_URL")
@@ -30,12 +46,7 @@ func New(clusterID string, ocmClient ocm.Client, remediation string) (client.Cli
 		return nil, err
 	}
 
-	scheme, err := initScheme()
-	if err != nil {
-		return nil, err
-	}
-
-	return client.New(cfg, client.Options{Scheme: scheme})
+	return cfg, nil
 }
 
 // Cleanup removes the remediation created for the cluster.
diff --git a/test/generate_incident.sh b/test/generate_incident.sh
@@ -9,6 +9,7 @@ declare -A alert_mapping=(
     ["ClusterMonitoringErrorBudgetBurnSRE"]="ClusterMonitoringErrorBudgetBurnSRE Critical (1)"
     ["InsightsOperatorDown"]="InsightsOperatorDown"
     ["MachineHealthCheckUnterminatedShortCircuitSRE"]="MachineHealthCheckUnterminatedShortCircuitSRE CRITICAL (1)"
+    ["ApiErrorBudgetBurn"]="api-ErrorBudgetBurn k8sgpt test CRITICAL (1)"
 )
 
 # Function to print help message
