Merge pull request #505 from bergmannf/srep-600-resource-init

[SREP-600] Use a builder pattern to initialize resources used by investigations.

Author: openshift-merge-bot[bot]

cadctl/cmd/investigate/investigate.go

@@ -22,12 +22,11 @@ import (
 	"strconv"
 	"strings"
 
-	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
 	investigations "github.com/openshift/configuration-anomaly-detection/pkg/investigations"
 	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/ccam"
 	investigation "github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/precheck"
 	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
-	"github.com/openshift/configuration-anomaly-detection/pkg/managedcloud"
 	"github.com/openshift/configuration-anomaly-detection/pkg/metrics"
 	ocm "github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 	"github.com/openshift/configuration-anomaly-detection/pkg/pagerduty"
@@ -79,14 +78,6 @@ func run(cmd *cobra.Command, _ []string) error {
 
 	logging.Infof("Incident link: %s", pdClient.GetIncidentRef())
 
-	var investigationResources *investigation.Resources
-
-	defer func() {
-		if err != nil {
-			handleCADFailure(err, investigationResources, pdClient)
-		}
-	}()
-
 	experimentalEnabledVar := os.Getenv("CAD_EXPERIMENTAL_ENABLED")
 	cadExperimentalEnabled, _ := strconv.ParseBool(experimentalEnabledVar)
 	alertInvestigation := investigations.GetInvestigation(pdClient.GetTitle(), cadExperimentalEnabled)
@@ -115,57 +106,47 @@ func run(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("could not initialize ocm client: %w", err)
 	}
 
-	cluster, err := ocmClient.GetClusterInfo(clusterID)
-	if err != nil {
-		if strings.Contains(err.Error(), "no cluster found") {
-			logging.Warnf("No cluster found with ID '%s'. Exiting.", clusterID)
-			return pdClient.EscalateIncidentWithNote("CAD was unable to find the incident cluster in OCM. An alert for a non-existing cluster is unexpected. Please investigate manually.")
+	builder := &investigation.ResourceBuilderT{}
+	defer func() {
+		if err != nil {
+			handleCADFailure(err, builder, pdClient)
 		}
-		return fmt.Errorf("could not retrieve cluster info for %s: %w", clusterID, err)
-	}
-
-	// From this point on, we normalize to internal ID, as this ID always exists.
-	// For installing clusters, externalID can be empty.
-	internalClusterID := cluster.ID()
+	}()
 
-	// re-initialize logger for the internal-cluster-id context
-	logging.RawLogger = logging.InitLogger(logLevelFlag, pipelineNameEnv, internalClusterID)
+	// Prime the builder with information required for all investigations.
+	builder.WithName(alertInvestigation.Name()).WithCluster(clusterID).WithPagerDutyClient(pdClient).WithOcmClient(ocmClient).WithLogger(logLevelFlag, pipelineNameEnv)
 
-	requiresInvestigation, err := clusterRequiresInvestigation(cluster, pdClient, ocmClient)
-	if err != nil || !requiresInvestigation {
-		return err
+	precheck := precheck.ClusterStatePrecheck{}
+	result, err := precheck.Run(builder)
+	if err != nil && strings.Contains(err.Error(), "no cluster found") {
+		logging.Warnf("No cluster found with ID '%s'. Escalating and exiting.", clusterID)
+		return pdClient.EscalateIncidentWithNote("CAD was unable to find the incident cluster in OCM. An alert for a non-existing cluster is unexpected. Please investigate manually.")
 	}
 
-	clusterDeployment, err := ocmClient.GetClusterDeployment(internalClusterID)
+	ccamInvestigation := ccam.Investigation{}
+	result, err = ccamInvestigation.Run(builder)
 	if err != nil {
-		return fmt.Errorf("could not retrieve Cluster Deployment for %s: %w", internalClusterID, err)
-	}
-
-	customerAwsClient, err := managedcloud.CreateCustomerAWSClient(cluster, ocmClient)
-	if err != nil {
-		ccamResources := &investigation.Resources{Name: "ccam", Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, Notes: nil, AdditionalResources: map[string]interface{}{"error": err}}
-		inv := ccam.Investigation{}
-		result, err := inv.Run(ccamResources)
-		updateMetrics(alertInvestigation.Name(), &result)
 		return err
 	}
-
-	investigationResources = &investigation.Resources{Name: alertInvestigation.Name(), Cluster: cluster, ClusterDeployment: clusterDeployment, AwsClient: customerAwsClient, OcmClient: ocmClient, PdClient: pdClient, Notes: nil}
+	updateMetrics(alertInvestigation.Name(), &result)
 
 	logging.Infof("Starting investigation for %s", alertInvestigation.Name())
-	result, err := alertInvestigation.Run(investigationResources)
-	updateMetrics(alertInvestigation.Name(), &result)
+	result, err = alertInvestigation.Run(builder)
 	if err != nil {
 		return err
 	}
+	updateMetrics(alertInvestigation.Name(), &result)
 
 	return updateIncidentTitle(pdClient)
 }
 
-func handleCADFailure(err error, resources *investigation.Resources, pdClient *pagerduty.SdkClient) {
+func handleCADFailure(err error, rb *investigation.ResourceBuilderT, pdClient *pagerduty.SdkClient) {
 	logging.Errorf("CAD investigation failed: %v", err)
 
 	var notes string
+	// The builder caches resources, so we can access them here even if a later step failed.
+	// We ignore the error here because we just want to get any notes that were created.
+	resources, _ := rb.Build()
 	if resources != nil && resources.Notes != nil {
 		resources.Notes.AppendWarning("🚨 CAD investigation failed, CAD team has been notified. Please investigate manually. 🚨")
 		notes = resources.Notes.String()
@@ -185,32 +166,6 @@ func handleCADFailure(err error, resources *investigation.Resources, pdClient *p
 	}
 }
 
-// Checks pre-requisites for a cluster investigation:
-// - the cluster's state is supported by CAD for an investigation (= not uninstalling)
-// - the cloud provider is supported by CAD (cluster is AWS)
-// Performs according pagerduty actions and returns whether CAD needs to investigate the cluster
-func clusterRequiresInvestigation(cluster *cmv1.Cluster, pdClient *pagerduty.SdkClient, ocmClient *ocm.SdkClient) (bool, error) {
-	if cluster.State() == cmv1.ClusterStateUninstalling {
-		logging.Info("Cluster is uninstalling and requires no investigation. Silencing alert.")
-		return false, pdClient.SilenceIncidentWithNote("CAD: Cluster is already uninstalling, silencing alert.")
-	}
-
-	if cluster.AWS() == nil {
-		logging.Info("Cloud provider unsupported, forwarding to primary.")
-		return false, pdClient.EscalateIncidentWithNote("CAD could not run an automated investigation on this cluster: unsupported cloud provider.")
-	}
-
-	isAccessProtected, err := ocmClient.IsAccessProtected(cluster)
-	if err != nil {
-		logging.Warnf("failed to get access protection status for cluster. %w. Continuing...")
-	}
-	if isAccessProtected {
-		logging.Info("Cluster is access protected. Escalating alert.")
-		return false, pdClient.EscalateIncidentWithNote("CAD is unable to run against access protected clusters. Please investigate.")
-	}
-	return true, nil
-}
-
 func updateMetrics(investigationName string, result *investigation.InvestigationResult) {
 	if result.ServiceLogSent.Performed {
 		metrics.Inc(metrics.ServicelogSent, append([]string{investigationName}, result.ServiceLogSent.Labels...)...)

pkg/investigations/cannotretrieveupdatessre/cannotretrieveupdatessre.go

@@ -18,8 +18,12 @@ import (
 type Investigation struct{}
 
 // Run executes the investigation for the CannotRetrieveUpdatesSRE alert
-func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investigation) Run(rb investigation.ResourceBuilder) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
+	r, err := rb.WithAwsClient().Build()
+	if err != nil {
+		return result, err
+	}
 	notes := notewriter.New("CannotRetrieveUpdatesSRE", logging.RawLogger)
 	k8scli, err := k8sclient.New(r.Cluster.ID(), r.OcmClient, r.Name)
 	if err != nil {

pkg/investigations/ccam/ccam.go

@@ -21,47 +21,47 @@ var ccamLimitedSupport = &ocm.LimitedSupportReason{
 
 // Evaluate estimates if the awsError is a cluster credentials are missing error. If it determines that it is,
 // the cluster is placed into limited support (if the cluster state allows it), otherwise an error is returned.
-func (c *Investigation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investigation) Run(r investigation.ResourceBuilder) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
-	cluster := r.Cluster
-	ocmClient := r.OcmClient
-	pdClient := r.PdClient
-	bpError, ok := r.AdditionalResources["error"].(error)
-	if !ok {
-		return result, fmt.Errorf("missing required Investigation field 'error'")
-	}
+	// Apart from the defaults this investigation requires an AWS client which can fail to build
+	resources, err := r.WithAwsClient().Build()
 	logging.Info("Investigating possible missing cloud credentials...")
-
-	if customerRemovedPermissions := customerRemovedPermissions(bpError.Error()); !customerRemovedPermissions {
-		// We aren't able to jumpRole because of an error that is different than
-		// a removed support role/policy or removed installer role/policy
-		// This would normally be a backplane failure.
-		return result, fmt.Errorf("credentials are there, error is different: %w", bpError)
-	}
-
-	// The jumprole failed because of a missing support role/policy:
-	// we need to figure out if we cluster state allows us to set limited support
-	// (the cluster is in a ready state, not uninstalling, installing, etc.)
-
-	switch cluster.State() {
-	case cmv1.ClusterStateReady:
-		// Cluster is in functional sate but we can't jumprole to it: post limited support
-		result.LimitedSupportSet.Performed = true
-		result.LimitedSupportSet.Labels = []string{ccamLimitedSupport.Summary}
-		err := ocmClient.PostLimitedSupportReason(ccamLimitedSupport, cluster.ID())
-		if err != nil {
-			return result, fmt.Errorf("could not post limited support reason for %s: %w", cluster.Name(), err)
+	if err != nil {
+		if customerRemovedPermissions := customerRemovedPermissions(err.Error()); !customerRemovedPermissions {
+			// We aren't able to jumpRole because of an error that is different than
+			// a removed support role/policy or removed installer role/policy
+			// This would normally be a backplane failure.
+			return result, err
+		}
+		cluster := resources.Cluster
+		ocmClient := resources.OcmClient
+		pdClient := resources.PdClient
+
+		// The jumprole failed because of a missing support role/policy:
+		// we need to figure out if we cluster state allows us to set limited support
+		// (the cluster is in a ready state, not uninstalling, installing, etc.)
+
+		switch cluster.State() {
+		case cmv1.ClusterStateReady:
+			// Cluster is in functional sate but we can't jumprole to it: post limited support
+			result.LimitedSupportSet.Performed = true
+			result.LimitedSupportSet.Labels = []string{ccamLimitedSupport.Summary}
+			err := ocmClient.PostLimitedSupportReason(ccamLimitedSupport, cluster.ID())
+			if err != nil {
+				return result, fmt.Errorf("could not post limited support reason for %s: %w", cluster.Name(), err)
+			}
+
+			return result, pdClient.SilenceIncidentWithNote(fmt.Sprintf("Added the following Limited Support reason to cluster: %#v. Silencing alert.\n", ccamLimitedSupport))
+		case cmv1.ClusterStateUninstalling:
+			// A cluster in uninstalling state should not alert primary - we just skip this
+			return result, pdClient.SilenceIncidentWithNote(fmt.Sprintf("Skipped adding limited support reason '%s': cluster is already uninstalling.", ccamLimitedSupport.Summary))
+		default:
+			// Anything else is an unknown state to us and/or requires investigation.
+			// E.g. we land here if we run into a CPD alert where credentials were removed (installing state) and don't want to put it in LS yet.
+			return result, pdClient.EscalateIncidentWithNote(fmt.Sprintf("Cluster has invalid cloud credentials (support role/policy is missing) and the cluster is in state '%s'. Please investigate.", cluster.State()))
 		}
-
-		return result, pdClient.SilenceIncidentWithNote(fmt.Sprintf("Added the following Limited Support reason to cluster: %#v. Silencing alert.\n", ccamLimitedSupport))
-	case cmv1.ClusterStateUninstalling:
-		// A cluster in uninstalling state should not alert primary - we just skip this
-		return result, pdClient.SilenceIncidentWithNote(fmt.Sprintf("Skipped adding limited support reason '%s': cluster is already uninstalling.", ccamLimitedSupport.Summary))
-	default:
-		// Anything else is an unknown state to us and/or requires investigation.
-		// E.g. we land here if we run into a CPD alert where credentials were removed (installing state) and don't want to put it in LS yet.
-		return result, pdClient.EscalateIncidentWithNote(fmt.Sprintf("Cluster has invalid cloud credentials (support role/policy is missing) and the cluster is in state '%s'. Please investigate.", cluster.State()))
 	}
+	return result, nil
 }
 
 func (c *Investigation) Name() string {

pkg/investigations/ccam/ccam_test.go

@@ -9,15 +9,15 @@ import (
 
 func TestEvaluateRandomError(t *testing.T) {
 	timeoutError := errors.New("credentials are there, error is different: timeout")
-	input := investigation.Resources{
-		Cluster:           nil,
-		ClusterDeployment: nil,
-		AwsClient:         nil,
-		OcmClient:         nil,
-		PdClient:          nil,
-		AdditionalResources: map[string]interface{}{
-			"error": errors.New("timeout"),
+	input := investigation.ResourceBuilderMock{
+		Resources: &investigation.Resources{
+			Cluster:           nil,
+			ClusterDeployment: nil,
+			AwsClient:         nil,
+			OcmClient:         nil,
+			PdClient:          nil,
 		},
+		BuildError: timeoutError,
 	}
 
 	inv := Investigation{}

pkg/investigations/chgm/chgm.go

@@ -36,8 +36,12 @@ var (
 type Investiation struct{}
 
 // Run runs the investigation for a triggered chgm pagerduty event
-func (c *Investiation) Run(r *investigation.Resources) (investigation.InvestigationResult, error) {
+func (c *Investiation) Run(rb investigation.ResourceBuilder) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
+	r, err := rb.WithClusterDeployment().Build()
+	if err != nil {
+		return result, err
+	}
 	notes := notewriter.New("CHGM", logging.RawLogger)
 	defer func() { r.Notes = notes }()