Add retry mechanism for scan enqueue failures(AST-0000) (#1374)

Author: cx-sumit-morchhale

internal/commands/scan.go

@@ -124,11 +124,12 @@ const (
 		"--scs-repo-url your_repo_url --scs-repo-token your_repo_token"
 	ScsScorecardUnsupportedHostWarningMsg = "SCS scan warning: Unable to run Scorecard scanner due to unsupported repo host. Currently, Scorecard can only run on GitHub Cloud repos."
 
-	jsonExt                  = ".json"
-	xmlExt                   = ".xml"
-	sbomScanTypeErrMsg       = "The --sbom-only flag can only be used when the scan type is sca"
-	BranchPrimaryPrefix      = "--branch-primary="
-	OverridePolicyManagement = "override-policy-management"
+	jsonExt                      = ".json"
+	xmlExt                       = ".xml"
+	sbomScanTypeErrMsg           = "The --sbom-only flag can only be used when the scan type is sca"
+	BranchPrimaryPrefix          = "--branch-primary="
+	OverridePolicyManagement     = "override-policy-management"
+	defaultScanEnqueueRetryDelay = 5
 )
 
 var (
@@ -708,6 +709,16 @@ func scanCreateSubCommand(
 		0,
 		"Cancel the scan and fail after the timeout in minutes",
 	)
+	createScanCmd.PersistentFlags().Int(
+		commonParams.ScanEnqueueRetriesFlag,
+		0,
+		"Number of retry attempts for scan enqueue failures due to queue capacity (default: 0, no retries)",
+	)
+	createScanCmd.PersistentFlags().Int(
+		commonParams.ScanEnqueueRetryDelayFlag,
+		defaultScanEnqueueRetryDelay,
+		"Base delay in seconds between scan enqueue retry attempts with exponential backoff (default: 5)",
+	)
 	createScanCmd.PersistentFlags().StringP(
 		commonParams.SourcesFlag,
 		commonParams.SourcesFlagSh,
@@ -859,6 +870,14 @@ func scanCreateSubCommand(
 	if err != nil {
 		log.Fatal(err)
 	}
+	err = viper.BindPFlag(commonParams.ScanEnqueueRetriesKey, createScanCmd.PersistentFlags().Lookup(commonParams.ScanEnqueueRetriesFlag))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = viper.BindPFlag(commonParams.ScanEnqueueRetryDelayKey, createScanCmd.PersistentFlags().Lookup(commonParams.ScanEnqueueRetryDelayFlag))
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	createScanCmd.PersistentFlags().String(commonParams.SSHKeyFlag, "", "Path to ssh private key")
 

internal/params/envs.go

@@ -4,6 +4,8 @@ const (
 	CustomStatesAPIPathEnv              = "CX_CUSTOM_STATES_PATH"
 	TenantEnv                           = "CX_TENANT"
 	BranchEnv                           = "CX_BRANCH"
+	ScanEnqueueRetriesEnv               = "CX_SCAN_ENQUEUE_RETRIES"
+	ScanEnqueueRetryDelayEnv            = "CX_SCAN_ENQUEUE_RETRY_DELAY"
 	BaseURIEnv                          = "CX_BASE_URI"
 	ClientTimeoutEnv                    = "CX_TIMEOUT"
 	ProxyEnv                            = "HTTP_PROXY"

internal/params/flags.go

@@ -33,6 +33,8 @@ const (
 	AsyncFlag                      = "async"
 	WaitDelayFlag                  = "wait-delay"
 	ScanTimeoutFlag                = "scan-timeout"
+	ScanEnqueueRetriesFlag         = "scan-enqueue-retries"
+	ScanEnqueueRetryDelayFlag      = "scan-enqueue-retry-delay"
 	PolicyTimeoutFlag              = "policy-timeout"
 	IgnorePolicyFlag               = "ignore-policy"
 	SourceDirFilterFlag            = "file-filter"

internal/params/keys.go

@@ -6,6 +6,8 @@ var (
 	CustomStatesAPIPathKey              = strings.ToLower(CustomStatesAPIPathEnv)
 	TenantKey                           = strings.ToLower(TenantEnv)
 	BranchKey                           = strings.ToLower(BranchEnv)
+	ScanEnqueueRetriesKey               = strings.ToLower(ScanEnqueueRetriesEnv)
+	ScanEnqueueRetryDelayKey            = strings.ToLower(ScanEnqueueRetryDelayEnv)
 	BaseURIKey                          = strings.ToLower(BaseURIEnv)
 	ProxyKey                            = strings.ToLower(ProxyEnv)
 	ProxyTypeKey                        = strings.ToLower(ProxyTypeEnv)

internal/wrappers/mock/scans-mock.go

@@ -9,6 +9,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+const ErrCodeMaxQueuedReached = 142
+
 var HasScs bool
 
 type ScansMockWrapper struct {
@@ -21,6 +23,13 @@ func (m *ScansMockWrapper) GetWorkflowByID(_ string) ([]*wrappers.ScanTaskRespon
 
 func (m *ScansMockWrapper) Create(scanModel *wrappers.Scan) (*wrappers.ScanResponseModel, *wrappers.ErrorModel, error) {
 	fmt.Println("Called Create in ScansMockWrapper")
+	if scanModel.Project.ID == "fake-queue-capacity-error-id" {
+		return nil, &wrappers.ErrorModel{
+			Code:    ErrCodeMaxQueuedReached,
+			Message: "Failed to enqueue scan. Max Queued reached",
+			Type:    "ERROR",
+		}, nil
+	}
 	if scanModel.Project.ID == "fake-kics-scanner-fail-id" {
 		return &wrappers.ScanResponseModel{
 			ID:     "fake-scan-id-kics-scanner-fail",

internal/wrappers/scans-http.go

@@ -5,16 +5,19 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"time"
 
+	"github.com/checkmarx/ast-cli/internal/logger"
 	commonParams "github.com/checkmarx/ast-cli/internal/params"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 )
 
 const (
-	failedToParseGetAll   = "Failed to parse list response"
-	failedToParseTags     = "Failed to parse tags response"
-	failedToParseBranches = "Failed to parse branches response"
+	failedToParseGetAll    = "Failed to parse list response"
+	failedToParseTags      = "Failed to parse tags response"
+	failedToParseBranches  = "Failed to parse branches response"
+	queueCapacityErrorCode = 142
 )
 
 type ScansHTTPWrapper struct {
@@ -29,26 +32,63 @@ func NewHTTPScansWrapper(path string) ScansWrapper {
 	}
 }
 
+// isQueueCapacityError checks if the error is due to queue capacity limits (error code 142)
+func isQueueCapacityError(errorModel *ErrorModel) bool {
+	return errorModel != nil && errorModel.Code == queueCapacityErrorCode
+}
+
 func (s *ScansHTTPWrapper) Create(model *Scan) (*ScanResponseModel, *ErrorModel, error) {
 	clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
 	jsonBytes, err := json.Marshal(model)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	fn := func() (*http.Response, error) {
-		return SendHTTPRequest(http.MethodPost, s.path, bytes.NewBuffer(jsonBytes), true, clientTimeout)
-	}
-	resp, err := retryHTTPRequest(fn, retryAttempts, retryDelay)
-	if err != nil {
-		return nil, nil, err
-	}
-	defer func() {
-		if err == nil {
-			_ = resp.Body.Close()
+	// Get scan enqueue retry configuration
+	scanEnqueueRetries := viper.GetInt(commonParams.ScanEnqueueRetriesKey)
+	scanEnqueueRetryDelay := viper.GetInt(commonParams.ScanEnqueueRetryDelayKey)
+
+	var scanResp *ScanResponseModel
+	var errorModel *ErrorModel
+
+	// Retry loop for scan creation (queue capacity errors)
+	for attempt := 0; attempt <= scanEnqueueRetries; attempt++ {
+		// Standard HTTP retry (for 502, 401)
+		fn := func() (*http.Response, error) {
+			return SendHTTPRequest(http.MethodPost, s.path, bytes.NewBuffer(jsonBytes), true, clientTimeout)
 		}
-	}()
-	return handleScanResponseWithBody(resp, err, http.StatusCreated)
+		resp, err := retryHTTPRequest(fn, retryAttempts, retryDelay)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// Parse response
+		scanResp, errorModel, err = handleScanResponseWithBody(resp, err, http.StatusCreated)
+		// Close response body explicitly after parsing
+		_ = resp.Body.Close()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// Check if it's a queue capacity error and we have retries left
+		if isQueueCapacityError(errorModel) && attempt < scanEnqueueRetries {
+			// Calculate exponential backoff delay
+			waitDuration := time.Duration(scanEnqueueRetryDelay) * time.Second * (1 << attempt)
+			logger.PrintIfVerbose(fmt.Sprintf(
+				"Scan creation failed due to queue capacity (attempt %d/%d). Waiting %v before retry...",
+				attempt+1,
+				scanEnqueueRetries,
+				waitDuration,
+			))
+			time.Sleep(waitDuration)
+			continue
+		}
+
+		// Success or non-retryable error - break out of loop
+		break
+	}
+
+	return scanResp, errorModel, err
 }
 
 func (s *ScansHTTPWrapper) Get(params map[string]string) (*ScansCollectionResponseModel, *ErrorModel, error) {