Merge branch 'main' of https://github.com/openshift/configuration-anomaly-detection into deprecate-x-secret-token

MateSaary · MateSaary · commit 91214fca1d0d · 2025-03-26T19:46:49.000Z
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,5 +1,13 @@
 version: 2
 updates:
+  - package-ecosystem: "gomod"
+    directories:
+      - "/"
+      - "interceptor/"
+    allow:
+      - dependency-type: all
+    schedule:
+      interval: "daily"
   - package-ecosystem: "docker"
     directory: "/build"
     labels:
diff --git a/OWNERS b/OWNERS
@@ -1,25 +1,23 @@
 reviewers:
 - Makdaam
 - Nikokolas3270
-- Tessg22
-- ninabauer
 - rafael-azevedo
 - RaphaelBut
 - bng0y
 - typeid
 - tnierman
-- sam-nguyen7
 - zmird-r
+- joshbranham
 approvers:
 - Makdaam
 - Nikokolas3270
-- Tessg22
-- ninabauer
 - rafael-azevedo
 - RaphaelBut
 - bng0y
 - typeid
-- sam-nguyen7
+- tnierman
 - zmird-r
+- joshbranham
+- srep-team-leads
 maintainers:
 - rafael-azevedo
diff --git a/README.md b/README.md
@@ -69,13 +69,13 @@ To add a new alert investigation:
 
 - run `make bootstrap-investigation` to generate boilerplate code in `pkg/investigations` (This creates the corresponding folder & .go file, and also appends the investigation to the `availableInvestigations` interface in `registry.go`.).
 - investigation.Resources contain initialized clients for the clusters aws environment, ocm and more. See [Integrations](#integrations)
+- Add test objects or scripts used to recreate the alert symptoms to the `pkg/investigations/$INVESTIGATION_NAME/testing/` directory for future use. Be sure to clearly document the testing procedure under the `Testing` section of the investigation-specific README.md file
 
 ### Integrations
 
 > **Note:** When writing an investiation, you can use them right away.
 They are initialized for you and passed to the investigation via investigation.Resources.
 
-
 * [AWS](https://github.com/aws/aws-sdk-go) -- Logging into the cluster, retreiving instance info and AWS CloudTrail events.
     - See `pkg/aws`
 * [PagerDuty](https://github.com/PagerDuty/go-pagerduty) -- Retrieving alert info, esclating or silencing incidents, and adding notes.
@@ -89,24 +89,28 @@ They are initialized for you and passed to the investigation via investigation.R
 
 ## Testing locally
 
-### Pre-requirements
-- an existing cluster
-- an existing PagerDuty incident for the cluster and alert type that is being tested
-
-To quickly create an incident for a cluster_id, you can run `./test/generate_incident.sh <alertname> <clusterid>`.
-Example usage:`./test/generate_incident.sh ClusterHasGoneMissing 2b94brrrrrrrrrrrrrrrrrrhkaj`.
-
-### Running cadctl for an incident ID
-1) Export the required ENV variables, see [required ENV variables](#required-env-variables).
-2) Create a payload file containing the incident ID
-  ```bash
-  export INCIDENT_ID=
-  echo '{"__pd_metadata":{"incident":{"id":"'${INCIDENT_ID}'"}}}' > ./payload
-  ```
-3) Run `cadctl` using the payload file
-  ```bash
-  ./bin/cadctl investigate --payload-path payload
-  ```
+Requires an existing cluster.
+
+1. Create a test incident and payload file for your cluster
+
+   ```bash
+   ./test/generate_incident.sh <alertname> <clusterid>
+   ```
+
+2. Export the required env variables from vault
+
+   > **Note:** For information on the envs see [required env variables](#required-env-variables).
+
+   ```
+   source test/set_stage_env.sh
+   ```
+
+3. `make build`
+4. Run `cadctl` with the payload file created by `test/generate_incident.sh`
+
+   ```bash
+   ./bin/cadctl investigate --payload-path payload
+   ```
 
 ### Logging levels
 
diff --git a/cadctl/cmd/investigate/investigate.go b/cadctl/cmd/investigate/investigate.go
@@ -140,7 +140,7 @@ func run(cmd *cobra.Command, _ []string) error {
 
 	investigationResources := &investigation.Resources{Name: alertInvestigation.Name(), Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient}
 
-	logging.Infof("Starting investigation for %s", alertInvestigation.Name)
+	logging.Infof("Starting investigation for %s", alertInvestigation.Name())
 	result, err := alertInvestigation.Run(investigationResources)
 	updateMetrics(alertInvestigation.Name(), &result)
 	return err
diff --git a/deploy/README.md b/deploy/README.md
@@ -27,7 +27,7 @@ See [../pkg/pagerduty/](../pkg/pagerduty/) for more details.
 ##### OCM
 [task-cad-checks-secrets-ocm-client.yaml](./task-cad-checks-secrets-ocm-client.yaml) This will hold the ocm creds.
 
-CAD_OCM_CLIENT_* env vars are in internal kv store. 
+CAD_OCM_CLIENT_* env vars are in internal kv store.
 
 See [../pkg/ocm/](../pkg/ocm/) for more details.
 
@@ -49,11 +49,11 @@ Install CAD by running the following commands:
     ```
 
 2. Configure secrets
-   
+
     See section at the bottom of `Tasks Secrets` to configure.
 
 3. Deploy container image
-   
+
     The repo builds the binary to a container using [../Dockerfile](a container file). build it using:
 
     ```console
@@ -92,7 +92,7 @@ Install CAD by running the following commands:
 Pipeline runs can be started via the following post command:
 
 ```console
-oc exec -it deploy/el-cad-event-listener -- curl -X POST -H 'X-Secret-Token: samplesecret' --connect-timeout 1 -v --data '{"event": {"data": {"id":"12312"}}}' http://el-cad-event-listener.configuration-anomaly-detection.svc.cluster.local:8080
+oc exec -it deploy/el-cad-event-listener -- curl -X POST -H 'X-PagerDuty-Signature: v1=samplesecret' --connect-timeout 1 -v --data '{"event": {"data": {"id":"12312"}}}' http://el-cad-event-listener.configuration-anomaly-detection.svc.cluster.local:8080
 ```
 
 For more details, see the [Tekton Documentation](https://github.com/tektoncd/triggers/tree/main/examples#invoking-the-triggers-locally).
@@ -110,7 +110,7 @@ The `tkn` tool is pulled from https://github.com/tektoncd/cli.
 The result of the last runs can be seen with:
 
 ```console
-tkn pipelinerun list -n configuration-anomaly-detection 
+tkn pipelinerun list -n configuration-anomaly-detection
 ```
 
 See the [Tekton documentation](https://docs.openshift.com/container-platform/4.4/cli_reference/tkn_cli/op-tkn-reference.html) for further commands.
diff --git a/deploy/task-cad-checks-secrets-pd.yaml b/deploy/task-cad-checks-secrets-pd.yaml
@@ -10,4 +10,4 @@ stringData:
   CAD_PD_TOKEN: CHANGEME # refers to the generated private access token for token-based authentication
   CAD_PD_USERNAME: CHANGEME # refers to the username in case username/pw credentials should be used
   CAD_SILENT_POLICY: CHANGEME # refers to the silent policy CAD should use if the incident shall be silent
-  PD_SIGNATURE: CHANGEME # refers to our custom Secret Token for authenticating against our pipeline
+  PD_SIGNATURE: CHANGEME # refers to the PagerDuty webhook signature (HMAC+SHA256)
diff --git a/hack/bootstrap-investigation.sh b/hack/bootstrap-investigation.sh
@@ -42,15 +42,30 @@ ls "${INVESTIGATION_DIR}"
 touch "${INVESTIGATION_DIR}/${INVESTIGATION_NAME}.go"
 touch "${INVESTIGATION_DIR}/metadata.yaml"
 touch "${INVESTIGATION_DIR}/README.md"
+mkdir "${INVESTIGATION_DIR}/testing/"
 
 # Create README.md file
 cat <<EOF > "${INVESTIGATION_DIR}/README.md"
 # ${INVESTIGATION_NAME} Investigation
 
 ${INVESTIGATION_DESCRIPTION}
 
+## Testing
+
+Refer to the [testing README](./testing/README.md) for instructions on testing this investigation
+
+EOF
+
+# Create testing/README.md file
+cat <<EOF > "${INVESTIGATION_DIR}/testing/README.md"
+# Testing ${INVESTIGATION_NAME} Investigation
+
+TODO:
+- Add a test script or test objects to this `testing/` directory for future maintainers to use
+- Edit this README file and add detailed instructions on how to use the script/objects to recreate the conditions for the investigation. Be sure to include any assumptions or prerequisites about the environment (disable hive syncsetting, etc)
 EOF
 
+
 # Create metadata.yaml file
 cat <<EOF > "${INVESTIGATION_DIR}/metadata.yaml"
 name: ${INVESTIGATION_NAME}
diff --git a/interceptor/pkg/interceptor/pdinterceptor.go b/interceptor/pkg/interceptor/pdinterceptor.go
@@ -110,8 +110,6 @@ func (pdi *PagerDutyInterceptor) executeInterceptor(r *http.Request) ([]byte, er
 
 	var ireq triggersv1.InterceptorRequest
 
-	// logging request
-	logging.Debug("Unwrapped Request header: %v", extractedRequest.Header)
 	logging.Debug("Unwrapped Request body: ", originalReq.Body)
 
 	token, _ := os.LookupEnv("PD_SIGNATURE")
@@ -159,7 +157,7 @@ func (pdi *PagerDutyInterceptor) Process(ctx context.Context, r *triggersv1.Inte
 		}
 	}
 
-	logging.Infof("Incident %s is mapped to investigation '%s', returning InterceptorResponse `Continue: true`.", pdClient.GetIncidentID(), investigation.Name)
+	logging.Infof("Incident %s is mapped to investigation '%s', returning InterceptorResponse `Continue: true`.", pdClient.GetIncidentID(), investigation.Name())
 	return &triggersv1.InterceptorResponse{
 		Continue: true,
 	}
diff --git a/interceptor/test/e2e.sh b/interceptor/test/e2e.sh
@@ -20,12 +20,19 @@ function test_interceptor {
 
     local incident_id=$1
     local expected_response=$2
+    local override_signature=$3
 
     # Run the interceptor and print logs to temporary log file
+    export PD_SIGNATURE="test"
     CAD_PD_TOKEN=$(echo $pd_test_token) CAD_SILENT_POLICY=$(echo $pd_test_silence_policy) ./../bin/interceptor > $temp_log_file  2>&1 &
-    PD_SIGNATURE="test"
     PAYLOAD_BODY="{\\\"__pd_metadata\\\":{\\\"incident\\\":{\\\"id\\\":\\\"$incident_id\\\"}}}"
     PAYLOAD_BODY_FORMATTED='{"__pd_metadata":{"incident":{"id":"'$incident_id'"}}}'
+
+    # Allow for test 3; override the signature after correct one has already been added to env
+    if [[ "$override_signature" != "" ]]; then
+      export PD_SIGNATURE=$override_signature
+    fi
+
     SIGN=$(echo -n "$PAYLOAD_BODY_FORMATTED" | sha256hmac -K $PD_SIGNATURE | tr -d "[:space:]-")
 
     # Store the PID of the interceptor process
@@ -85,4 +92,4 @@ test_interceptor "Q3722KGCG12ZWD" "$EXPECTED_RESPONSE_STOP"
 # Test for an alert with invalid signature
 echo "Test 3: expected failure due to invalid signature"
 PD_SIGNATURE="invalid-signature"
-test_interceptor "Q12WO44XJLR3H3" "$EXPECTED_RESPONSE_SIGNATURE_ERROR"
+test_interceptor "Q12WO44XJLR3H3" "$EXPECTED_RESPONSE_SIGNATURE_ERROR" "invalid-signature"
diff --git a/pkg/investigations/insightsoperatordown/metadata.yaml b/pkg/investigations/insightsoperatordown/metadata.yaml
@@ -0,0 +1,12 @@
+name: insightsoperatordown
+rbac:
+  roles: []
+  clusterRoleRules:
+    - verbs:
+        - "get"
+        - "list"
+      apiGroups:
+        - "config.openshift.io"
+      resources:
+        - clusteroperators
+customerDataAccess: false
diff --git a/pkg/investigations/machineHealthCheckUnterminatedShortCircuitSRE/metadata.yaml b/pkg/investigations/machineHealthCheckUnterminatedShortCircuitSRE/metadata.yaml
@@ -1,12 +1,21 @@
 name: MachineHealthCheckUnterminatedShortCircuitSRE
 rbac:
-  roles: []
+  roles:
+    - namespace: "openshift-machine-api"
+      rules:
+        - verbs:
+            - "get"
+            - "list"
+          apiGroups:
+            - "machine.openshift.io"
+          resources:
+            - "machines"
   clusterRoleRules:
     - verbs:
         - "get"
         - "list"
       apiGroups:
-        - "machine.openshift.io"
+        - ""
       resources:
-        - machines
+        - "nodes"
 customerDataAccess: true
diff --git a/pkg/investigations/pruningcronjoberror/metadata.yaml b/pkg/investigations/pruningcronjoberror/metadata.yaml
@@ -0,0 +1,20 @@
+name: pruningcronjoberror
+rbac:
+  roles: []
+  clusterRoleRules:
+    - verbs:
+        - "get"
+        - "list"
+      apiGroups:
+        - "config.openshift.io"
+      resources:
+        - clusteroperators
+    - apiGroups:
+        - ""
+      resources:
+        - pods
+        - namespaces
+      verbs:
+        - get
+        - list
+customerDataAccess: false
diff --git a/pkg/investigations/upgradeconfigsyncfailureover4hr/metadata.yaml b/pkg/investigations/upgradeconfigsyncfailureover4hr/metadata.yaml
@@ -0,0 +1,14 @@
+name: upgradeconfigsyncfailureover4hr
+rbac:
+  roles:
+    - namespace: "openshift-config"
+      rules:
+        - verbs:
+            - "get"
+          apiGroups:
+            - ""
+          resources:
+            - "secrets"
+          resourceNames:
+            - "pull-secret"
+customerDataAccess: false
diff --git a/test/generate_incident.sh b/test/generate_incident.sh
@@ -1,15 +1,6 @@
 #!/bin/bash
 set -e
 
-# Check if pd_token variable is set
-if [ -z "$pd_token" ]; then
-  echo "Warning: pd_token is not set. For more information, see https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTUx-authentication."
-  echo "The pd_token is needed to automatically retrieve the incident after creating it."
-  echo "You can still search in the service directory for the created incident, but setting the pd_token ENV in the future will facilitate getting the created incident id."
-  echo
-  echo
-fi
-
 # Define the mapping of alert names to titles
 # Add more mappings as needed: for the standard service, we should not need to go by title but by the `alertname` field instead.
 declare -A alert_mapping=(
@@ -25,7 +16,7 @@ print_help() {
     for alert_name in "${!alert_mapping[@]}"; do
         echo -n "$alert_name, "
     done
-    echo 
+    echo
 }
 # Check if the correct number of arguments is provided
 if [ "$#" -ne 2 ]; then
@@ -49,9 +40,9 @@ alert_title="${alert_mapping[$alert_name]}"
 # Load testing routing key and test service url from vault
 export VAULT_ADDR="https://vault.devshift.net"
 export VAULT_TOKEN="$(vault login -method=oidc -token-only)"
-for v in $(vault kv get  -format=json osd-sre/configuration-anomaly-detection/cad-testing | jq -r ".data.data|to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]"); do export $v; done	
+for v in $(vault kv get  -format=json osd-sre/configuration-anomaly-detection/cad-testing | jq -r ".data.data|to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]"); do export $v; done
 unset VAULT_ADDR VAULT_TOKEN
-echo 
+echo
 
 dedup_key=$(uuidgen)
 
@@ -82,20 +73,14 @@ if [[ $response != *"Event processed"* ]]; then
 fi
 echo
 
-# Point to the service directory if we can't auto retrieve the incident via the pd token
-if [ -z "$pd_token" ]; then
-  echo "Incident successfully created. See ${pd_test_service_url} to retrieve the incident ID."
-  exit 0
-fi
-
 # Pagerduty seems to need a short while to create the incident
 # Added this as we intermittently fail to get the incident id otherwise
 sleep 2
 
 INCIDENT_ID=$(curl --silent --request GET \
   --url "https://api.pagerduty.com/incidents?incident_key=${dedup_key}" \
   --header 'Accept: application/json' \
-  --header "Authorization: Token token=${pd_token}" \
+  --header "Authorization: Token token=${pd_test_token}" \
   --header 'Content-Type: application/json' | jq -r '.incidents[0].id')
 echo $INCIDENT_ID
 echo '{"__pd_metadata":{"incident":{"id":"'$INCIDENT_ID'"}}}' > ./payload

Original file line number	Diff line number	Diff line change
`@@ -110,8 +110,6 @@ func (pdi PagerDutyInterceptor) executeInterceptor(r http.Request) ([]byte, er`
`110`	`110`
`111`	`111`	`var ireq triggersv1.InterceptorRequest`
`112`	`112`
`113`		`- // logging request`
`114`		`- logging.Debug("Unwrapped Request header: %v", extractedRequest.Header)`
`115`	`113`	`logging.Debug("Unwrapped Request body: ", originalReq.Body)`
`116`	`114`
`117`	`115`	`token, _ := os.LookupEnv("PD_SIGNATURE")`
`@@ -159,7 +157,7 @@ func (pdi PagerDutyInterceptor) Process(ctx context.Context, r triggersv1.Inte`
`159`	`157`	`}`
`160`	`158`	`}`
`161`	`159`
`162`		- logging.Infof("Incident %s is mapped to investigation '%s', returning InterceptorResponse `Continue: true`.", pdClient.GetIncidentID(), investigation.Name)
	`160`	+ logging.Infof("Incident %s is mapped to investigation '%s', returning InterceptorResponse `Continue: true`.", pdClient.GetIncidentID(), investigation.Name())
`163`	`161`	`return &triggersv1.InterceptorResponse{`
`164`	`162`	`Continue: true,`
`165`	`163`	`}`