Add handling to post failure notes to PagerDuty

Author: MateSaary

cadctl/cmd/investigate/investigate.go

@@ -76,6 +76,14 @@ func run(cmd *cobra.Command, _ []string) error {
 
 	logging.Infof("Incident link: %s", pdClient.GetIncidentRef())
 
+	var investigationResources *investigation.Resources
+
+	defer func() {
+		if err != nil {
+			handleCADFailure(err, investigationResources)
+		}
+	}()
+
 	_, cadExperimentalEnabled := os.LookupEnv("CAD_EXPERIMENTAL_ENABLED")
 	alertInvestigation := investigations.GetInvestigation(pdClient.GetTitle(), cadExperimentalEnabled)
 
@@ -131,21 +139,36 @@ func run(cmd *cobra.Command, _ []string) error {
 
 	customerAwsClient, err := managedcloud.CreateCustomerAWSClient(cluster, ocmClient)
 	if err != nil {
-		ccamResources := &investigation.Resources{Name: "ccam", Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, AdditionalResources: map[string]interface{}{"error": err}}
+		ccamResources := &investigation.Resources{Name: "ccam", Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, Notes: nil, AdditionalResources: map[string]interface{}{"error": err}}
 		inv := ccam.Investigation{}
 		result, err := inv.Run(ccamResources)
 		updateMetrics(alertInvestigation.Name(), &result)
 		return err
 	}
 
-	investigationResources := &investigation.Resources{Name: alertInvestigation.Name(), Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient}
+	investigationResources = &investigation.Resources{Name: alertInvestigation.Name(), Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, Notes: nil}
 
 	logging.Infof("Starting investigation for %s", alertInvestigation.Name())
 	result, err := alertInvestigation.Run(investigationResources)
 	updateMetrics(alertInvestigation.Name(), &result)
+
 	return err
 }
 
+func handleCADFailure(err error, resources *investigation.Resources) {
+	logging.Errorf("CAD investigation failed: %v", err)
+
+	notes := resources.Notes
+
+	notes.AppendWarning("🚨 CAD investigation failed, CAD team has been notified. Please investigate manually. 🚨")
+	pdErr := resources.PdClient.EscalateIncidentWithNote(notes.String())
+	if pdErr != nil {
+		logging.Errorf("Failed to escalate notes to PagerDuty: %v", pdErr)
+	} else {
+		logging.Info("CAD failure & incident notes added to PagerDuty")
+	}
+}
+
 // GetOCMClient will retrieve the OcmClient from the 'ocm' package
 func GetOCMClient() (*ocm.SdkClient, error) {
 	cadOcmFilePath := os.Getenv("CAD_OCM_FILE_PATH")

hack/bootstrap-investigation.sh

@@ -96,6 +96,7 @@ func (c *Investigation) Run(r *investigation.Resources) (investigation.Investiga
 
 	// Initialize PagerDuty note writer
 	notes := notewriter.New(r.Name, logging.RawLogger)
+	defer func() { r.Notes = notes }()
 
 	// TODO: Implement investigation logic here
 

pkg/investigations/chgm/chgm.go

@@ -42,6 +42,7 @@ type Investiation struct{}
 func (c *Investiation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
 	notes := notewriter.New("CHGM", logging.RawLogger)
+	defer func() { r.Notes = notes }()
 
 	// 1. Check if the user stopped instances
 	res, err := investigateStoppedInstances(r.Cluster, r.ClusterDeployment, r.AwsClient, r.OcmClient)

pkg/investigations/clustermonitoringerrorbudgetburn/clustermonitoringerrorbudgetburn.go

@@ -27,14 +27,13 @@ var uwmMisconfiguredSL = ocm.ServiceLog{
 
 type Investigation struct{}
 
-func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investigation) Run(r *investigation.Resources) (result investigation.InvestigationResult, err error) {
 	// Initialize k8s client
 	// This would be better suited to be passend in with the investigation resources
 	// In turn we would need to split out ccam and k8sclient, as those are tied to a cluster
 	// Failing the cleanup call is not critical as there is garbage collection for the RBAC within MCC https://issues.redhat.com/browse/OSD-27692
 	// We can revisit backplane-apis remediation implementation to improve this behavior, by e.g.
 	// patching the existing RBAC etc...
-	result := investigation.InvestigationResult{}
 	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, r.Name)
 	if err != nil {
 		return result, fmt.Errorf("unable to initialize k8s cli: %w", err)
@@ -49,6 +48,7 @@ func (c *Investigation) Run(r *investigation.Resources) (investigation.Investiga
 
 	// Initialize PagerDuty note writer
 	notes := notewriter.New(r.Name, logging.RawLogger)
+	defer func() { r.Notes = notes }()
 
 	// List the monitoring cluster operator
 	coList := &configv1.ClusterOperatorList{}

pkg/investigations/cpd/cpd.go

@@ -30,6 +30,7 @@ var byovpcRoutingSL = &ocm.ServiceLog{Severity: "Major", Summary: "Installation
 func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
 	notes := notewriter.New("CPD", logging.RawLogger)
+	defer func() { r.Notes = notes }()
 
 	if r.Cluster.Status().State() == "ready" {
 		// We are unsure when this happens, in theory, if the cluster is ready, the alert shouldn't fire or should autoresolve.

pkg/investigations/investigation/investigation.go

@@ -4,6 +4,7 @@ package investigation
 import (
 	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
 	"github.com/openshift/configuration-anomaly-detection/pkg/aws"
+	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 	"github.com/openshift/configuration-anomaly-detection/pkg/pagerduty"
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
@@ -38,5 +39,6 @@ type Resources struct {
 	AwsClient           aws.Client
 	OcmClient           ocm.Client
 	PdClient            pagerduty.Client
+	Notes               *notewriter.NoteWriter
 	AdditionalResources map[string]interface{}
 }