Add caching to the builder.

Use the builder to report the error on failures.

Author: bergmannf

cadctl/cmd/investigate/investigate.go

@@ -78,14 +78,6 @@ func run(cmd *cobra.Command, _ []string) error {
 
 	logging.Infof("Incident link: %s", pdClient.GetIncidentRef())
 
-	var investigationResources *investigation.Resources
-
-	defer func() {
-		if err != nil {
-			handleCADFailure(err, investigationResources, pdClient)
-		}
-	}()
-
 	experimentalEnabledVar := os.Getenv("CAD_EXPERIMENTAL_ENABLED")
 	cadExperimentalEnabled, _ := strconv.ParseBool(experimentalEnabledVar)
 	alertInvestigation := investigations.GetInvestigation(pdClient.GetTitle(), cadExperimentalEnabled)
@@ -115,36 +107,54 @@ func run(cmd *cobra.Command, _ []string) error {
 	}
 
 	builder := &investigation.ResourceBuilderT{}
+	defer func() {
+		if err != nil {
+			handleCADFailure(err, builder, pdClient)
+		}
+	}()
+
 	// Prime the builder with information required for all investigations.
 	builder.WithName(alertInvestigation.Name()).WithCluster(clusterID).WithPagerDutyClient(pdClient).WithOcmClient(ocmClient).WithLogger(logLevelFlag, pipelineNameEnv)
 
+	// handleClusterNotFound centralizes the logic for this specific error case.
+	handleClusterNotFound := func(investigationErr error) error {
+		if investigationErr != nil && strings.Contains(investigationErr.Error(), "no cluster found") {
+			logging.Warnf("No cluster found with ID '%s'. Escalating and exiting.", clusterID)
+			return pdClient.EscalateIncidentWithNote("CAD was unable to find the incident cluster in OCM. An alert for a non-existing cluster is unexpected. Please investigate manually.")
+		}
+		return investigationErr
+	}
+
 	precheck := precheck.Investigation{}
 	result, err := precheck.Run(builder)
-	if err != nil || result.StopInvestigations {
+	if err = handleClusterNotFound(err); err != nil || result.StopInvestigations {
 		return err
 	}
 
 	inv := ccam.Investigation{}
 	result, err = inv.Run(builder)
-	updateMetrics(alertInvestigation.Name(), &result)
-	if err != nil {
+	if err = handleClusterNotFound(err); err != nil {
 		return err
 	}
+	updateMetrics(alertInvestigation.Name(), &result)
 
 	logging.Infof("Starting investigation for %s", alertInvestigation.Name())
 	result, err = alertInvestigation.Run(builder)
-	updateMetrics(alertInvestigation.Name(), &result)
-	if err != nil {
+	if err = handleClusterNotFound(err); err != nil {
 		return err
 	}
+	updateMetrics(alertInvestigation.Name(), &result)
 
 	return updateIncidentTitle(pdClient)
 }
 
-func handleCADFailure(err error, resources *investigation.Resources, pdClient *pagerduty.SdkClient) {
+func handleCADFailure(err error, builder *investigation.ResourceBuilderT, pdClient *pagerduty.SdkClient) {
 	logging.Errorf("CAD investigation failed: %v", err)
 
 	var notes string
+	// The builder caches resources, so we can access them here even if a later step failed.
+	// We ignore the error here because we just want to get any notes that were created.
+	resources, _ := builder.Build()
 	if resources != nil && resources.Notes != nil {
 		resources.Notes.AppendWarning("🚨 CAD investigation failed, CAD team has been notified. Please investigate manually. 🚨")
 		notes = resources.Notes.String()

pkg/investigations/investigation/investigation.go

@@ -77,6 +77,10 @@ type ResourceBuilderT struct {
 	name         string
 	logLevel     string
 	pipelineName string
+
+	// cache
+	builtResources *Resources
+	buildErr       error
 }
 
 func (r *ResourceBuilderT) WithCluster(clusterId string) ResourceBuilder {
@@ -123,66 +127,71 @@ func (r *ResourceBuilderT) WithLogger(logLevel string, pipelineName string) Reso
 }
 
 func (r *ResourceBuilderT) Build() (*Resources, error) {
-	var awsClient aws.Client
-	var cluster *cmv1.Cluster
-	var clusterDeployment *hivev1.ClusterDeployment
-	var notes *notewriter.NoteWriter
-	var internalClusterId string
+	if r.buildErr != nil {
+		return nil, r.buildErr
+	}
+
+	if r.builtResources == nil {
+		r.builtResources = &Resources{
+			Name:      r.name,
+			OcmClient: r.ocmClient,
+			PdClient:  r.pdClient,
+		}
+	}
+
 	var err error
 
 	if r.buildClusterDeployment && !r.buildCluster {
-		return nil, errors.New("can not build ClusterDeployment without Cluster")
+		r.buildErr = errors.New("cannot build ClusterDeployment without Cluster")
+		return nil, r.buildErr
+	}
+	if r.buildAwsClient && !r.buildCluster {
+		r.buildErr = errors.New("cannot build AwsClient without Cluster")
+		return nil, r.buildErr
 	}
 
-	if r.buildAwsClient {
-		awsClient, err = managedcloud.CreateCustomerAWSClient(cluster, r.ocmClient)
+	if r.buildCluster && r.builtResources.Cluster == nil {
+		r.builtResources.Cluster, err = r.ocmClient.GetClusterInfo(r.clusterId)
 		if err != nil {
+			// Let the caller handle how to respond to this error.
+			err = fmt.Errorf("could not retrieve cluster info for %s: %w", r.clusterId, err)
+			r.buildErr = err
 			return nil, err
 		}
 	}
 
-	if r.buildCluster {
-		cluster, err = r.ocmClient.GetClusterInfo(r.clusterId)
-		if err != nil {
-			if strings.Contains(err.Error(), "no cluster found") {
-				logging.Warnf("No cluster found with ID '%s'. Exiting.", r.clusterId)
-				err = r.pdClient.EscalateIncidentWithNote("CAD was unable to find the incident cluster in OCM. An alert for a non-existing cluster is unexpected. Please investigate manually.")
-				logging.Errorf("Could not escalate via PagerDuty: ", err)
-				return nil, errors.New("unable to find incident cluster in OCM")
+	// Dependent resources can only be built if a cluster object exists.
+	if r.builtResources.Cluster != nil {
+		internalClusterId := r.builtResources.Cluster.ID()
+
+		if r.buildAwsClient && r.builtResources.AwsClient == nil {
+			r.builtResources.AwsClient, err = managedcloud.CreateCustomerAWSClient(r.builtResources.Cluster, r.ocmClient)
+			if err != nil {
+				r.buildErr = err
+				return nil, err
 			}
-			return nil, fmt.Errorf("could not retrieve cluster info for %s: %w", r.clusterId, err)
 		}
 
-		// From this point on, we normalize to internal ID, as this ID always exists.
-		// For installing clusters, externalID can be empty.
-		internalClusterId = cluster.ID()
-	}
-
-	if r.buildClusterDeployment {
-		clusterDeployment, err = r.ocmClient.GetClusterDeployment(internalClusterId)
-		if err != nil {
-			return nil, fmt.Errorf("could not retrieve Cluster Deployment for %s: %w", internalClusterId, err)
+		if r.buildClusterDeployment && r.builtResources.ClusterDeployment == nil {
+			r.builtResources.ClusterDeployment, err = r.ocmClient.GetClusterDeployment(internalClusterId)
+			if err != nil {
+				err = fmt.Errorf("could not retrieve Cluster Deployment for %s: %w", internalClusterId, err)
+				r.buildErr = err
+				return nil, err
+			}
 		}
-	}
 
-	if r.buildLogger {
-		logging.RawLogger = logging.InitLogger(r.logLevel, "", internalClusterId)
+		if r.buildLogger {
+			// Re-initialize the logger with the cluster ID.
+			logging.RawLogger = logging.InitLogger(r.logLevel, "", internalClusterId)
+		}
 	}
 
-	if r.buildNotes {
-		// Initialize NoteWriter with sane defaults
-		notes = notewriter.New(r.name, logging.RawLogger)
+	if r.buildNotes && r.builtResources.Notes == nil {
+		r.builtResources.Notes = notewriter.New(r.name, logging.RawLogger)
 	}
 
-	return &Resources{
-		Name:              r.name,
-		Cluster:           cluster,
-		ClusterDeployment: clusterDeployment,
-		AwsClient:         awsClient,
-		OcmClient:         r.ocmClient,
-		PdClient:          r.pdClient,
-		Notes:             notes,
-	}, nil
+	return r.builtResources, nil
 }
 
 // This is an implementation to be used in tests, but putting it into a _test.go file will make it not resolvable.