Feat: remove escalation policy and replace by escalating to the next level

Author: typeid

README.md

@@ -137,7 +137,6 @@ Grafana dashboard configmaps are stored in the [Dashboards](./dashboards/) direc
 * `AWS_SECRET_ACCESS_KEY`: refers to the secret access key of the base AWS account used by CAD
 * `CAD_AWS_CSS_JUMPROLE`:  refers to the arn of the RH-SRE-CCS-Access jumprole
 * `CAD_AWS_SUPPORT_JUMPROLE`: refers to the arn of the RH-Technical-Support-Access jumprole
-* `CAD_ESCALATION_POLICY`:  refers to the escalation policy CAD should use to escalate the incident to
 * `CAD_PD_EMAIL`: refers  to the email for a login via mail/pw credentials
 * `CAD_PD_PW`: refers to the password for a login via mail/pw credentials
 * `CAD_PD_TOKEN`: refers to the generated private access token for token-based authentication

cadctl/cmd/investigate/investigate.go

@@ -75,7 +75,7 @@ func run(_ *cobra.Command, _ []string) error {
 
 	// Escalate all unsupported alerts
 	if alertInvestigation == nil {
-		err = pdClient.EscalateAlert()
+		err = pdClient.EscalateIncident()
 		if err != nil {
 			return fmt.Errorf("could not escalate unsupported alert: %w", err)
 		}
@@ -152,12 +152,12 @@ func GetOCMClient() (*ocm.SdkClient, error) {
 func clusterRequiresInvestigation(cluster *cmv1.Cluster, pdClient *pagerduty.SdkClient, ocmClient *ocm.SdkClient) (bool, error) {
 	if cluster.State() == cmv1.ClusterStateUninstalling {
 		logging.Info("Cluster is uninstalling and requires no investigation. Silencing alert.")
-		return false, pdClient.SilenceAlertWithNote("CAD: Cluster is already uninstalling, silencing alert.")
+		return false, pdClient.SilenceIncidentWithNote("CAD: Cluster is already uninstalling, silencing alert.")
 	}
 
 	if cluster.AWS() == nil {
 		logging.Info("Cloud provider unsupported, forwarding to primary.")
-		return false, pdClient.EscalateAlertWithNote("CAD could not run an automated investigation on this cluster: unsupported cloud provider.")
+		return false, pdClient.EscalateIncidentWithNote("CAD could not run an automated investigation on this cluster: unsupported cloud provider.")
 	}
 
 	isAccessProtected, err := ocmClient.IsAccessProtected(cluster)
@@ -166,7 +166,7 @@ func clusterRequiresInvestigation(cluster *cmv1.Cluster, pdClient *pagerduty.Sdk
 	}
 	if isAccessProtected {
 		logging.Info("Cluster is access protected. Escalating alert.")
-		return false, pdClient.EscalateAlertWithNote("CAD is unable to run against access protected clusters. Please investigate.")
+		return false, pdClient.EscalateIncidentWithNote("CAD is unable to run against access protected clusters. Please investigate.")
 	}
 	return true, nil
 }

deploy/task-cad-checks-secrets-pd.yaml

@@ -5,7 +5,6 @@ metadata:
   name: cad-pd-token
 type: Opaque
 stringData:
-  CAD_ESCALATION_POLICY: CHANGEME # refers to the escalation policy CAD should use to escalate the incident to
   CAD_PD_EMAIL: CHANGEME # refers  to the email for a login via mail/pw credentials
   CAD_PD_PW: CHANGEME # refers to the password for a login via mail/pw credentials
   CAD_PD_TOKEN: CHANGEME # refers to the generated private access token for token-based authentication

interceptor/pkg/interceptor/pdinterceptor.go

@@ -111,7 +111,7 @@ func (pdi *PagerDutyInterceptor) Process(ctx context.Context, r *triggersv1.Inte
 	// and escalate the alert to SRE
 	if investigation == nil {
 		pdi.Logger.Infof("Incident %s is not mapped to an investigation, escalating incident and returning InterceptorResponse `Continue: false`.", pdClient.GetIncidentID())
-		err = pdClient.EscalateAlert()
+		err = pdClient.EscalateIncident()
 		if err != nil {
 			pdi.Logger.Errorf("failed to escalate incident '%s': %w", pdClient.GetIncidentID(), err)
 		}

interceptor/test/e2e.sh

@@ -18,7 +18,7 @@ temp_log_file=$(mktemp)
 # Function to send an interceptor request and check the response
 function test_interceptor {
     # Run the interceptor and print logs to temporary log file
-    CAD_PD_TOKEN=$(echo $pd_test_token) CAD_ESCALATION_POLICY=$(echo $pd_test_escalation_policy) CAD_SILENT_POLICY=$(echo $pd_test_silence_policy) ./../bin/interceptor > $temp_log_file  2>&1 &
+    CAD_PD_TOKEN=$(echo $pd_test_token) CAD_SILENT_POLICY=$(echo $pd_test_silence_policy) ./../bin/interceptor > $temp_log_file  2>&1 &
 
     # Store the PID of the interceptor process
     INTERCEPTOR_PID=$!

openshift/template.yaml

@@ -279,7 +279,6 @@ objects:
   metadata:
     name: cad-pd-token
   stringData:
-    CAD_ESCALATION_POLICY: CHANGEME
     CAD_PD_EMAIL: CHANGEME
     CAD_PD_PW: CHANGEME
     CAD_PD_TOKEN: CHANGEME

pkg/investigations/ccam/ccam.go

@@ -43,14 +43,14 @@ func Evaluate(cluster *cmv1.Cluster, bpError error, ocmClient ocm.Client, pdClie
 			return fmt.Errorf("could not post limited support reason for %s: %w", cluster.Name(), err)
 		}
 
-		return pdClient.SilenceAlertWithNote(fmt.Sprintf("Added the following Limited Support reason to cluster: %#v. Silencing alert.\n", ccamLimitedSupport))
+		return pdClient.SilenceIncidentWithNote(fmt.Sprintf("Added the following Limited Support reason to cluster: %#v. Silencing alert.\n", ccamLimitedSupport))
 	case cmv1.ClusterStateUninstalling:
 		// A cluster in uninstalling state should not alert primary - we just skip this
-		return pdClient.SilenceAlertWithNote(fmt.Sprintf("Skipped adding limited support reason '%s': cluster is already uninstalling.", ccamLimitedSupport.Summary))
+		return pdClient.SilenceIncidentWithNote(fmt.Sprintf("Skipped adding limited support reason '%s': cluster is already uninstalling.", ccamLimitedSupport.Summary))
 	default:
 		// Anything else is an unknown state to us and/or requires investigation.
 		// E.g. we land here if we run into a CPD alert where credentials were removed (installing state) and don't want to put it in LS yet.
-		return pdClient.EscalateAlertWithNote(fmt.Sprintf("Cluster has invalid cloud credentials (support role/policy is missing) and the cluster is in state '%s'. Please investigate.", cluster.State()))
+		return pdClient.EscalateIncidentWithNote(fmt.Sprintf("Cluster has invalid cloud credentials (support role/policy is missing) and the cluster is in state '%s'. Please investigate.", cluster.State()))
 	}
 }
 

pkg/investigations/chgm/chgm.go

@@ -46,7 +46,7 @@ func Investigate(r *investigation.Resources) error {
 	// 1. Check if the user stopped instances
 	res, err := investigateStoppedInstances(r.Cluster, r.ClusterDeployment, r.AwsClient, r.OcmClient)
 	if err != nil {
-		return r.PdClient.EscalateAlertWithNote(fmt.Sprintf("InvestigateInstances failed: %s\n", err.Error()))
+		return r.PdClient.EscalateIncidentWithNote(fmt.Sprintf("InvestigateInstances failed: %s\n", err.Error()))
 	}
 	logging.Debugf("the investigation returned: [infras running: %d] - [masters running: %d]", res.RunningInstances.Infra, res.RunningInstances.Master)
 
@@ -95,7 +95,7 @@ func Investigate(r *investigation.Resources) error {
 			metrics.Inc(metrics.LimitedSupportSet, investigationName, "EgressBlocked")
 
 			notes.AppendAutomation("Egress `nosnch.in` blocked, sent limited support.")
-			return r.PdClient.SilenceAlertWithNote(notes.String())
+			return r.PdClient.SilenceIncidentWithNote(notes.String())
 		}
 
 		err := r.OcmClient.PostServiceLog(r.Cluster.ID(), createEgressSL(failureReason))
@@ -112,7 +112,7 @@ func Investigate(r *investigation.Resources) error {
 	}
 
 	// Found no issues that CAD can handle by itself - forward notes to SRE.
-	return r.PdClient.EscalateAlertWithNote(notes.String())
+	return r.PdClient.EscalateIncidentWithNote(notes.String())
 }
 
 // investigateHibernation checks if the cluster was recently woken up from
@@ -439,5 +439,5 @@ func postChgmSLAndSilence(clusterID string, ocmCli ocm.Client, pdCli pagerduty.C
 		return fmt.Errorf("failed sending service log: %w", err)
 	}
 
-	return pdCli.SilenceAlertWithNote("Customer stopped instances. Sent SL and silencing alert.")
+	return pdCli.SilenceIncidentWithNote("Customer stopped instances. Sent SL and silencing alert.")
 }