Merge branch 'main' into OSD-28718-loglevel-config

Author: MateSaary

Makefile

@@ -83,6 +83,11 @@ test-interceptor-e2e: check-go121-install check-jq-install check-vault-install b
 	@echo "Running e2e tests for interceptor..."
 	cd interceptor && ./test/e2e.sh
 
+##@ Boilerplate:
+.PHONY: boilerplate
+bootstrap-investigation: ## Bootstrap a new boilerplate investigation
+	@cd hack && ./bootstrap-investigation.sh
+
 ##@ Template-updater:
 .PHONY: template-updater
 template-updater: build-template-updater lint-template-updater ## Run all targets for template-updater

README.md

@@ -66,8 +66,26 @@ The required investigation is identified by CAD based on the incident and its pa
 As PagerDuty itself does not provide finer granularity for webhooks than service-based, CAD filters out the alerts it should investigate. For more information, please refer to https://support.pagerduty.com/docs/webhooks.
 
 To add a new alert investigation:
-- create a mapping for the alert to the `GetInvestigation` function in `mapping.go` and write a corresponding CAD investigation (e.g. `Investigate()` in `chgm.go`).
-- if the alert is not yet routed to CAD, add a webhook to the service your alert fires on. For production, the service should also have an escalation policy that escalates to SRE on CAD automation timeout.
+
+- run `make bootstrap-investigation` to generate boilerplate code in `pkg/investigations` (This creates the corresponding folder & .go file, and also appends the investigation to the `availableInvestigations` interface in `registry.go`.).
+- investigation.Resources contain initialized clients for the clusters aws environment, ocm and more. See [Integrations](#integrations)
+
+### Integrations
+
+> **Note:** When writing an investiation, you can use them right away.
+They are initialized for you and passed to the investigation via investigation.Resources.
+
+
+* [AWS](https://github.com/aws/aws-sdk-go) -- Logging into the cluster, retreiving instance info and AWS CloudTrail events.
+    - See `pkg/aws`
+* [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes.
+    - See `pkg/pagerduty`
+* [OCM](https://github.com/openshift-online/ocm-sdk-go) -- Retrieving cluster info, sending service logs, and managing (post, delete) limited support reasons.
+    - See `pkg/ocm`
+    - In case of missing permissions to query an ocm resource, add it to the Configuration-Anomaly-Detection role in uhc-account-manager
+* [osd-network-verifier](https://github.com/openshift/osd-network-verifier) -- Tool to verify the pre-configured networking components for ROSA and OSD CCS clusters.
+* [k8sclient](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/client) -- Interact with clusters kube-api
+    - Requires RBAC definitions for your investigation to be added to `metadata.yaml`
 
 ## Testing locally
 

cadctl/cmd/investigate/investigate.go

@@ -132,7 +132,7 @@ func run(cmd *cobra.Command, _ []string) error {
 	customerAwsClient, err := managedcloud.CreateCustomerAWSClient(cluster, ocmClient)
 	if err != nil {
 		ccamResources := &investigation.Resources{Name: "ccam", Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, AdditionalResources: map[string]interface{}{"error": err}}
-		inv := ccam.CCAM{}
+		inv := ccam.Investigation{}
 		result, err := inv.Run(ccamResources)
 		updateMetrics(alertInvestigation.Name(), &result)
 		return err

hack/bootstrap-investigation.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+set -e
+
+read -p "Enter the new investigation (package) name: " INVESTIGATION_NAME
+if [[ "${INVESTIGATION_NAME}" == "" ]] ; then
+	echo "Investigation name cannot be empty."
+	exit 1
+elif [[ "${INVESTIGATION_NAME}" =~ [^a-zA-Z0-9_] ]] ; then
+	echo "Investigation name must be alphanumeric."
+	exit 1
+fi
+
+read -p "Enter new investigation description: " INVESTIGATION_DESCRIPTION
+if [[ "${INVESTIGATION_DESCRIPTION}" == "" ]] ; then
+	INVESTIGATION_DESCRIPTION="TODO"
+fi
+
+read -p "Should Investigate Alert (y/n): " INVESTIGATE_ALERT_BOOL
+if [[ "${INVESTIGATE_ALERT_BOOL}" == "y" ]] ; then
+	read -p "Investigation alert string: " INVESTIGATION_ALERT_STRING
+	INVESTIGATION_ALERT="strings.Contains(alert, \"${INVESTIGATION_ALERT_STRING}\")"
+elif [[ "${INVESTIGATE_ALERT_BOOL}" == "n" ]] ; then
+	INVESTIGATION_ALERT="false"
+else
+	echo "Invalid input. Please enter 'y' or 'n'."
+	exit 1
+fi
+
+INVESTIGATION_NAME=$(echo "${INVESTIGATION_NAME}" | tr '[:upper:]' '[:lower:]')
+
+INVESTIGATION_DIR="../pkg/investigations/${INVESTIGATION_NAME}"
+
+if [ -d "${INVESTIGATION_DIR}" ]; then
+    echo "Investigation of name ${INVESTIGATION_NAME} already exists."
+    exit 1
+fi
+
+mkdir -p "${INVESTIGATION_DIR}"
+ls "${INVESTIGATION_DIR}"
+
+touch "${INVESTIGATION_DIR}/${INVESTIGATION_NAME}.go"
+touch "${INVESTIGATION_DIR}/metadata.yaml"
+touch "${INVESTIGATION_DIR}/README.md"
+
+# Create README.md file
+cat <<EOF > "${INVESTIGATION_DIR}/README.md"
+# ${INVESTIGATION_NAME} Investigation
+
+${INVESTIGATION_DESCRIPTION}
+
+EOF
+
+# Create metadata.yaml file
+cat <<EOF > "${INVESTIGATION_DIR}/metadata.yaml"
+name: ${INVESTIGATION_NAME}
+rbac:
+  roles: []
+  clusterRoleRules: []
+customerDataAccess: false
+
+EOF
+
+# Create boilerplate investigation file
+cat <<EOF > "${INVESTIGATION_DIR}/${INVESTIGATION_NAME}.go"
+// Package ${INVESTIGATION_NAME} contains...TODO
+package ${INVESTIGATION_NAME}
+
+import (
+	"strings"
+
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
+)
+
+type Investigation struct{}
+
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+	result := investigation.InvestigationResult{}
+
+	// Initialize PagerDuty note writer
+	notes := notewriter.New(r.Name, logging.RawLogger)
+
+	// TODO: Implement investigation logic here
+
+	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+}
+
+func (c *Investigation) Name() string {
+	return "${INVESTIGATION_NAME}"
+}
+
+func (c *Investigation) Description() string {
+	return "${INVESTIGATION_DESCRIPTION}"
+}
+
+func (c *Investigation) ShouldInvestigateAlert(alert string) bool {
+	return ${INVESTIGATION_ALERT}
+}
+
+func (c *Investigation) IsExperimental() bool {
+	// TODO: Update to false when graduating to production.
+	return true
+}
+
+EOF
+
+echo "${INVESTIGATION_NAME} created in ${INVESTIGATION_DIR}"
+echo "metadata.yaml file created in ${INVESTIGATION_DIR}"
+
+# Update registry.go to contain new investigation
+if ! grep -q "${INVESTIGATION_NAME}" ../pkg/investigations/registry.go && ! grep -q "${INVESTIGATION_NAME}" ../pkg/investigations/registry.go; then
+	sed -i "/import (/a \\\t\"github.com/openshift/configuration-anomaly-detection/pkg/investigations/${INVESTIGATION_NAME}\"" ../pkg/investigations/registry.go
+    sed -i "/var availableInvestigations = \[/a \\\t&${INVESTIGATION_NAME}.Investigation{}," ../pkg/investigations/registry.go
+    echo "${INVESTIGATION_NAME} added to registry.go"
+else
+    echo "${INVESTIGATION_NAME} already exists in registry.go"
+fi

interceptor/README.md

@@ -1,15 +1,32 @@
-# CAD Tekton Interceptor 
+# CAD Tekton Interceptor
 
-The tekton interceptor is a component plugged between the event listener and the task runs. The interceptor makes sure we don't start a pipeline for every alert we receive. Instead, alerts are filtered based on whether or not they are handled by CAD. Unhandled alerts are directly escalated and no pipeline is started. 
+The tekton interceptor is a component plugged between the event listener and the task runs. The interceptor makes sure we don't start a pipeline for every alert we receive. Instead, alerts are filtered based on whether or not they are handled by CAD. Unhandled alerts are directly escalated and no pipeline is started.
 
 ## Testing
 
 ### E2E
 
 The interceptor has E2E tests starting the HTTP service and checking the HTTP responses. The tests are based on pre-existing PagerDuty alerts.
-```
+
+``` bash
+
 make e2e-interceptor
 
 # To also print the output of the interceptor service:
 CAD_E2E_VERBOSE=true make test-interceptor
-```
+```
+
+## Development
+
+It is possible to run the interceptor locally in a "minimal" state, where E2E is not used, and only the
+crucial-to-run env variables (seen below) are set as placeholders. This is useful for *local* development/debugging.
+
+``` bash
+$ make build-interceptor
+
+$ CAD_SILENT_POLICY=test
+$ CAD_PD_TOKEN=test
+$ PD_SIGNATURE=test
+
+$ ./bin/interceptor
+```

pkg/investigations/ccam/ccam.go

@@ -12,7 +12,7 @@ import (
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 )
 
-type CCAM struct{}
+type Investigation struct{}
 
 var ccamLimitedSupport = &ocm.LimitedSupportReason{
 	Summary: "Restore missing cloud credentials",
@@ -21,14 +21,14 @@ var ccamLimitedSupport = &ocm.LimitedSupportReason{
 
 // Evaluate estimates if the awsError is a cluster credentials are missing error. If it determines that it is,
 // the cluster is placed into limited support (if the cluster state allows it), otherwise an error is returned.
-func (c *CCAM) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
 	cluster := r.Cluster
 	ocmClient := r.OcmClient
 	pdClient := r.PdClient
 	bpError, ok := r.AdditionalResources["error"].(error)
 	if !ok {
-		return result, fmt.Errorf("Missing required CCAM field 'error'")
+		return result, fmt.Errorf("Missing required Investigation field 'error'")
 	}
 	logging.Info("Investigating possible missing cloud credentials...")
 
@@ -64,19 +64,19 @@ func (c *CCAM) Run(r *investigation.Resources) (investigation.InvestigationResul
 	}
 }
 
-func (c *CCAM) Name() string {
+func (c *Investigation) Name() string {
 	return "Cluster Credentials Are Missing (CCAM)"
 }
 
-func (c *CCAM) Description() string {
+func (c *Investigation) Description() string {
 	return "Detects missing cluster credentials"
 }
 
-func (c *CCAM) ShouldInvestigateAlert(alert string) bool {
+func (c *Investigation) ShouldInvestigateAlert(alert string) bool {
 	return false
 }
 
-func (c *CCAM) IsExperimental() bool {
+func (c *Investigation) IsExperimental() bool {
 	return false
 }
 

pkg/investigations/ccam/ccam_test.go

@@ -20,7 +20,7 @@ func TestEvaluateRandomError(t *testing.T) {
 		},
 	}
 
-	inv := CCAM{}
+	inv := Investigation{}
 
 	_, err := inv.Run(&input)
 	if err.Error() != timeoutError.Error() {

pkg/investigations/chgm/chgm.go

@@ -36,10 +36,10 @@ var (
 	}
 )
 
-type CHGM struct{}
+type Investiation struct{}
 
 // Run runs the investigation for a triggered chgm pagerduty event
-func (c *CHGM) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investiation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
 	notes := notewriter.New("CHGM", logging.RawLogger)
 
@@ -118,19 +118,19 @@ func (c *CHGM) Run(r *investigation.Resources) (investigation.InvestigationResul
 	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
 }
 
-func (c *CHGM) Name() string {
+func (c *Investiation) Name() string {
 	return "Cluster Has Gone Missing (CHGM)"
 }
 
-func (c *CHGM) Description() string {
+func (c *Investiation) Description() string {
 	return "Detects reason for clusters that have gone missing"
 }
 
-func (c *CHGM) ShouldInvestigateAlert(alert string) bool {
+func (c *Investiation) ShouldInvestigateAlert(alert string) bool {
 	return strings.Contains(alert, "has gone missing")
 }
 
-func (c *CHGM) IsExperimental() bool {
+func (c *Investiation) IsExperimental() bool {
 	return false
 }
 

pkg/investigations/chgm/chgm_test.go

@@ -92,7 +92,7 @@ var _ = Describe("chgm", func() {
 		mockCtrl.Finish()
 	})
 
-	inv := CHGM{}
+	inv := Investiation{}
 
 	Describe("Triggered", func() {
 		When("Triggered finds instances stopped by the customer", func() {

pkg/investigations/clustermonitoringerrorbudgetburn/clustermonitoringerrorbudgetburn.go

@@ -25,9 +25,9 @@ var uwmMisconfiguredSL = ocm.ServiceLog{
 	InternalOnly: false,
 }
 
-type CMEBB struct{}
+type Investigation struct{}
 
-func (c *CMEBB) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	// Initialize k8s client
 	// This would be better suited to be passend in with the investigation resources
 	// In turn we would need to split out ccam and k8sclient, as those are tied to a cluster
@@ -84,20 +84,20 @@ func (c *CMEBB) Run(r *investigation.Resources) (investigation.InvestigationResu
 	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
 }
 
-func (c *CMEBB) Name() string {
+func (c *Investigation) Name() string {
 	return "clustermonitoringerrorbudgetburn"
 }
 
-func (c *CMEBB) Description() string {
+func (c *Investigation) Description() string {
 	return "Investigate the cluster monitoring error budget burn alert"
 }
 
-func (c *CMEBB) ShouldInvestigateAlert(alert string) bool {
+func (c *Investigation) ShouldInvestigateAlert(alert string) bool {
 	return strings.Contains(alert, "ClusterMonitoringErrorBudgetBurnSRE")
 }
 
-func (c *CMEBB) IsExperimental() bool {
-	return true
+func (c *Investigation) IsExperimental() bool {
+	return false
 }
 
 // Check if the `Available` status condition reports a broken UWM config