Merge branch 'main' into main

Author: MateSaary

Makefile

@@ -26,6 +26,9 @@ build: build-interceptor build-cadctl build-template-updater ## Build all subpro
 .PHONY: lint
 lint: lint-cadctl lint-interceptor lint-template-updater ## Lint all subprojects
 
+.PHONY: test
+test: test-cadctl test-interceptor
+
 ##@ cadctl:
 .PHONY: cadctl
 cadctl: generate-cadctl build-cadctl test-cadctl lint-cadctl generate-template-file ## Run all targets for cadctl (generate, build, test, lint, generation)
@@ -54,7 +57,7 @@ test-cadctl: check-go121-install ## Run automated tests for cadctl
 
 ##@ Interceptor:
 .PHONY: interceptor
-interceptor: build-interceptor test-interceptor lint-interceptor ## Run all targets for interceptor (build, test, lint)
+interceptor: build-interceptor test-interceptor test-interceptor-e2e lint-interceptor ## Run all targets for interceptor (build, test, lint)
 
 .PHONY: build-interceptor
 build-interceptor: check-go121-install ## Build the interceptor binary
@@ -69,10 +72,13 @@ lint-interceptor: install-linter ## Lint interceptor subproject
 	cd interceptor && GOLANGCI_LINT_CACHE=$$(mktemp -d) $(GOPATH)/bin/golangci-lint run -c ../.golangci.yml
 
 .PHONY: test-interceptor
-test-interceptor: check-go121-install check-jq-install check-vault-install build-interceptor ## Run automated tests for interceptor
+test-interceptor: check-go121-install check-jq-install build-interceptor ## Run unit tests for interceptor
 	@echo
 	@echo "Running unit tests for interceptor..."
 	cd interceptor && go test -race -mod=readonly ./...
+
+.PHONY: test-interceptor-e2e
+test-interceptor-e2e: check-go121-install check-jq-install check-vault-install build-interceptor ## Run e2e tests for interceptor
 	@echo
 	@echo "Running e2e tests for interceptor..."
 	cd interceptor && ./test/e2e.sh

README.md

@@ -66,9 +66,27 @@ The required investigation is identified by CAD based on the incident and its pa
 As PagerDuty itself does not provide finer granularity for webhooks than service-based, CAD filters out the alerts it should investigate. For more information, please refer to https://support.pagerduty.com/docs/webhooks.
 
 To add a new alert investigation:
+
 - run `make bootstrap-investigation` to generate boilerplate code in `pkg/investigations` (This creates the corresponding folder & .go file, and also appends the investigation to the `availableInvestigations` interface in `registry.go`.).
 - if the alert is not yet routed to CAD, add a webhook to the service your alert fires on. For production, the service should also have an escalation policy that escalates to SRE on CAD automation timeout.
 
+### Integrations
+
+> **Note:** When writing an investiation, you can use them right away.
+They are initialized for you and passed to the investigation via investigation.Resources.
+
+
+* [AWS](https://github.com/aws/aws-sdk-go) -- Logging into the cluster, retreiving instance info and AWS CloudTrail events.
+    - See `pkg/aws`
+* [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes.
+    - See `pkg/pagerduty`
+* [OCM](https://github.com/openshift-online/ocm-sdk-go) -- Retrieving cluster info, sending service logs, and managing (post, delete) limited support reasons.
+    - See `pkg/ocm`
+    - In case of missing permissions to query an ocm resource, add it to the Configuration-Anomaly-Detection role in uhc-account-manager
+* [osd-network-verifier](https://github.com/openshift/osd-network-verifier) -- Tool to verify the pre-configured networking components for ROSA and OSD CCS clusters.
+* [k8sclient](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/client) -- Interact with clusters kube-api
+    - Requires RBAC definitions for your investigation to be added to `metadata.yaml`
+
 ## Testing locally
 
 ### Pre-requirements
@@ -98,13 +116,6 @@ Every alert managed by CAD corresponds to an investigation, representing the exe
 
 Investigation specific documentation can be found in the according investigation folder,  e.g. for [ClusterHasGoneMissing](./pkg/investigations/chgm/README.md).
 
-### Integrations
-
-* [AWS](https://github.com/aws/aws-sdk-go) -- Logging into the cluster, retreiving instance info and AWS CloudTrail events.
-* [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes. 
-* [OCM](https://github.com/openshift-online/ocm-sdk-go) -- Retrieving cluster info, sending service logs, and managing (post, delete) limited support reasons.
-* [osd-network-verifier](https://github.com/openshift/osd-network-verifier) -- Tool to verify the pre-configured networking components for ROSA and OSD CCS clusters.
-
 ### Templates
 
 * [Update-Template](./hack/update-template/README.md) -- Updating configuration-anomaly-detection-template.Template.yaml.

cadctl/cmd/investigate/investigate.go

@@ -122,7 +122,7 @@ func run(_ *cobra.Command, _ []string) error {
 	customerAwsClient, err := managedcloud.CreateCustomerAWSClient(cluster, ocmClient)
 	if err != nil {
 		ccamResources := &investigation.Resources{Name: "ccam", Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, AdditionalResources: map[string]interface{}{"error": err}}
-		inv := ccam.CCAM{}
+		inv := ccam.Investigation{}
 		result, err := inv.Run(ccamResources)
 		updateMetrics(alertInvestigation.Name(), &result)
 		return err

interceptor/README.md

@@ -1,15 +1,32 @@
-# CAD Tekton Interceptor 
+# CAD Tekton Interceptor
 
-The tekton interceptor is a component plugged between the event listener and the task runs. The interceptor makes sure we don't start a pipeline for every alert we receive. Instead, alerts are filtered based on whether or not they are handled by CAD. Unhandled alerts are directly escalated and no pipeline is started. 
+The tekton interceptor is a component plugged between the event listener and the task runs. The interceptor makes sure we don't start a pipeline for every alert we receive. Instead, alerts are filtered based on whether or not they are handled by CAD. Unhandled alerts are directly escalated and no pipeline is started.
 
 ## Testing
 
 ### E2E
 
 The interceptor has E2E tests starting the HTTP service and checking the HTTP responses. The tests are based on pre-existing PagerDuty alerts.
-```
+
+``` bash
+
 make e2e-interceptor
 
 # To also print the output of the interceptor service:
 CAD_E2E_VERBOSE=true make test-interceptor
-```
+```
+
+## Development
+
+It is possible to run the interceptor locally in a "minimal" state, where E2E is not used, and only the
+crucial-to-run env variables (seen below) are set as placeholders. This is useful for *local* development/debugging.
+
+``` bash
+$ make build-interceptor
+
+$ CAD_SILENT_POLICY=test
+$ CAD_PD_TOKEN=test
+$ PD_SIGNATURE=test
+
+$ ./bin/interceptor
+```

interceptor/pkg/interceptor/pdinterceptor.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/PagerDuty/go-pagerduty/webhookv3"
 	investigations "github.com/openshift/configuration-anomaly-detection/pkg/investigations"
 	"github.com/openshift/configuration-anomaly-detection/pkg/pagerduty"
 	triggersv1 "github.com/tektoncd/triggers/pkg/apis/triggers/v1beta1"
@@ -39,6 +40,7 @@ func (pdi PagerDutyInterceptor) ServeHTTP(w http.ResponseWriter, r *http.Request
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 		}
 	}
+
 	w.Header().Add("Content-Type", "application/json")
 	if _, err := w.Write(b); err != nil {
 		pdi.Logger.Errorf("failed to write response: %s", err)
@@ -86,7 +88,44 @@ func (pdi *PagerDutyInterceptor) executeInterceptor(r *http.Request) ([]byte, er
 	if _, err := io.Copy(&body, r.Body); err != nil {
 		return nil, internal(fmt.Errorf("failed to read body: %w", err))
 	}
+	r.Body = io.NopCloser(bytes.NewReader(body.Bytes()))
+
+	// originalReq is the original request that was sent to the interceptor,
+	// due to be unwrapped into a new header and body for signature verification.
+	var originalReq struct {
+		Body   string              `json:"body"`
+		Header map[string][]string `json:"header"`
+	}
+	if err := json.Unmarshal(body.Bytes(), &originalReq); err != nil {
+		return nil, badRequest(fmt.Errorf("failed to parse request body: %w", err))
+	}
+
+	extractedRequest, err := http.NewRequestWithContext(ctx, r.Method, r.URL.String(), bytes.NewReader([]byte(originalReq.Body)))
+	if err != nil {
+		return nil, internal(fmt.Errorf("malformed body/header in unwrapped request: %w", err))
+	}
+
+	for k, v := range originalReq.Header {
+		for _, v := range v {
+			extractedRequest.Header.Add(k, v)
+		}
+	}
+
 	var ireq triggersv1.InterceptorRequest
+
+	// logging request
+	pdi.Logger.Info("Wrapped Request header: %v", r.Header)
+	pdi.Logger.Info("Wrapped Request body: ", body.String())
+	pdi.Logger.Info("Unwrapped Request header: %v", extractedRequest.Header)
+	pdi.Logger.Info("Unwrapped Request body: ", originalReq.Body)
+
+	token, _ := os.LookupEnv("PD_SIGNATURE")
+
+	err = webhookv3.VerifySignature(extractedRequest, token)
+	if err != nil {
+		return nil, badRequest(fmt.Errorf("failed to verify signature: %w", err))
+	}
+
 	if err := json.Unmarshal(body.Bytes(), &ireq); err != nil {
 		return nil, badRequest(fmt.Errorf("failed to parse body as InterceptorRequest: %w", err))
 	}

interceptor/test/e2e.sh

@@ -19,7 +19,11 @@ temp_log_file=$(mktemp)
 function test_interceptor {
     # Run the interceptor and print logs to temporary log file
     CAD_PD_TOKEN=$(echo $pd_test_token) CAD_SILENT_POLICY=$(echo $pd_test_silence_policy) ./../bin/interceptor > $temp_log_file  2>&1 &
-    
+    PD_SIGNATURE="test"
+    PAYLOAD="{\"body\":\"{\\\"__pd_metadata\\\":{\\\"incident\\\":{\\\"id\\\":\\\"$incident_id\\\"}}}\",\"header\":{\"Content-Type\":[\"application/json\"]},\"extensions\":{},\"interceptor_params\":{},\"context\":null}"
+    SIGN=$(echo -n "$PAYLOAD" | sha256hmac -K $PD_SIGNATURE | tr -d "[:space:]-")
+    echo "Sign: $SIGN"
+
     # Store the PID of the interceptor process
     INTERCEPTOR_PID=$!
 
@@ -32,8 +36,8 @@ function test_interceptor {
     # Send an interceptor request to localhost:8080
     # See https://pkg.go.dev/github.com/tektoncd/triggers/pkg/apis/triggers/v1alpha1#InterceptorRequest
     CURL_EXITCODE=0
-    CURL_OUTPUT=$(curl -s -X POST -H "Content-Type: application/json" \
-        -d "{\"body\":\"{\\\"__pd_metadata\\\":{\\\"incident\\\":{\\\"id\\\":\\\"$incident_id\\\"}}}\",\"header\":{\"Content-Type\":[\"application/json\"]},\"extensions\":{},\"interceptor_params\":{},\"context\":null}" \
+    CURL_OUTPUT=$(curl -s -X POST -H "X-PagerDuty-Signature:v1=${SIGN}" -H "Content-Type: application/json" \
+        -d "$PAYLOAD" \
         http://localhost:8080) || CURL_EXITCODE=$?
 
     # Check if the curl output matches the expected response
@@ -69,5 +73,9 @@ echo "Test 1: alert with existing handling returns a 'continue: true' response"
 test_interceptor "Q12WO44XJLR3H3" "$EXPECTED_RESPONSE_CONTINUE"
 
 # Test for an alert we don't handle (alert called unhandled)
-echo "Test 1: unhandled alerts returns a 'continue: false' response"
+echo "Test 2: unhandled alerts returns a 'continue: false' response"
 test_interceptor "Q3722KGCG12ZWD" "$EXPECTED_RESPONSE_STOP"
+
+echo "Test 3: expected failure due to invalid signature"
+PD_SIGNATURE="invalid-signature"
+test_interceptor "Q12WO44XJLR3H3" "$EXPECTED_RESPONSE_STOP"

pkg/investigations/ccam/ccam.go

@@ -12,7 +12,7 @@ import (
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 )
 
-type CCAM struct{}
+type Investigation struct{}
 
 var ccamLimitedSupport = &ocm.LimitedSupportReason{
 	Summary: "Restore missing cloud credentials",
@@ -21,14 +21,14 @@ var ccamLimitedSupport = &ocm.LimitedSupportReason{
 
 // Evaluate estimates if the awsError is a cluster credentials are missing error. If it determines that it is,
 // the cluster is placed into limited support (if the cluster state allows it), otherwise an error is returned.
-func (c *CCAM) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
 	cluster := r.Cluster
 	ocmClient := r.OcmClient
 	pdClient := r.PdClient
 	bpError, ok := r.AdditionalResources["error"].(error)
 	if !ok {
-		return result, fmt.Errorf("Missing required CCAM field 'error'")
+		return result, fmt.Errorf("Missing required Investigation field 'error'")
 	}
 	logging.Info("Investigating possible missing cloud credentials...")
 
@@ -64,19 +64,19 @@ func (c *CCAM) Run(r *investigation.Resources) (investigation.InvestigationResul
 	}
 }
 
-func (c *CCAM) Name() string {
+func (c *Investigation) Name() string {
 	return "Cluster Credentials Are Missing (CCAM)"
 }
 
-func (c *CCAM) Description() string {
+func (c *Investigation) Description() string {
 	return "Detects missing cluster credentials"
 }
 
-func (c *CCAM) ShouldInvestigateAlert(alert string) bool {
+func (c *Investigation) ShouldInvestigateAlert(alert string) bool {
 	return false
 }
 
-func (c *CCAM) IsExperimental() bool {
+func (c *Investigation) IsExperimental() bool {
 	return false
 }
 

pkg/investigations/ccam/ccam_test.go

@@ -20,7 +20,7 @@ func TestEvaluateRandomError(t *testing.T) {
 		},
 	}
 
-	inv := CCAM{}
+	inv := Investigation{}
 
 	_, err := inv.Run(&input)
 	if err.Error() != timeoutError.Error() {

pkg/investigations/chgm/chgm.go

@@ -36,10 +36,10 @@ var (
 	}
 )
 
-type CHGM struct{}
+type Investiation struct{}
 
 // Run runs the investigation for a triggered chgm pagerduty event
-func (c *CHGM) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investiation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
 	notes := notewriter.New("CHGM", logging.RawLogger)
 
@@ -118,19 +118,19 @@ func (c *CHGM) Run(r *investigation.Resources) (investigation.InvestigationResul
 	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
 }
 
-func (c *CHGM) Name() string {
+func (c *Investiation) Name() string {
 	return "Cluster Has Gone Missing (CHGM)"
 }
 
-func (c *CHGM) Description() string {
+func (c *Investiation) Description() string {
 	return "Detects reason for clusters that have gone missing"
 }
 
-func (c *CHGM) ShouldInvestigateAlert(alert string) bool {
+func (c *Investiation) ShouldInvestigateAlert(alert string) bool {
 	return strings.Contains(alert, "has gone missing")
 }
 
-func (c *CHGM) IsExperimental() bool {
+func (c *Investiation) IsExperimental() bool {
 	return false
 }
 

pkg/investigations/chgm/chgm_test.go

@@ -92,7 +92,7 @@ var _ = Describe("chgm", func() {
 		mockCtrl.Finish()
 	})
 
-	inv := CHGM{}
+	inv := Investiation{}
 
 	Describe("Triggered", func() {
 		When("Triggered finds instances stopped by the customer", func() {