Skip to content

Commit 83e63b4

Browse files
Adds option to retry enqueuing a scan when the queue is full (fixes #1349)
1 parent 71f2784 commit 83e63b4

File tree

7 files changed

+119
-15
lines changed

7 files changed

+119
-15
lines changed

ast-cli.sln

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
Microsoft Visual Studio Solution File, Format Version 12.00
2+
# Visual Studio Version 17
3+
VisualStudioVersion = 17.5.2.0
4+
MinimumVisualStudioVersion = 10.0.40219.1
5+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "internal", "internal", "{5A380C17-196E-6321-DE74-1F06A4E0A5CA}"
6+
EndProject
7+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "commands", "commands", "{E5A21A7A-5BCD-2715-58A7-4DD23F70B1FE}"
8+
EndProject
9+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "data", "data", "{D12D9C30-2E74-3E87-8F12-FB4BFDB3464B}"
10+
EndProject
11+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test", "internal\commands\data\manifests\test.csproj", "{443B93EE-5877-B8D2-3E73-6AA63E426C0F}"
12+
EndProject
13+
Global
14+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
15+
Debug|Any CPU = Debug|Any CPU
16+
Release|Any CPU = Release|Any CPU
17+
EndGlobalSection
18+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
19+
{443B93EE-5877-B8D2-3E73-6AA63E426C0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
20+
{443B93EE-5877-B8D2-3E73-6AA63E426C0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
21+
{443B93EE-5877-B8D2-3E73-6AA63E426C0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
22+
{443B93EE-5877-B8D2-3E73-6AA63E426C0F}.Release|Any CPU.Build.0 = Release|Any CPU
23+
EndGlobalSection
24+
GlobalSection(SolutionProperties) = preSolution
25+
HideSolutionNode = FALSE
26+
EndGlobalSection
27+
GlobalSection(NestedProjects) = preSolution
28+
{E5A21A7A-5BCD-2715-58A7-4DD23F70B1FE} = {5A380C17-196E-6321-DE74-1F06A4E0A5CA}
29+
{D12D9C30-2E74-3E87-8F12-FB4BFDB3464B} = {E5A21A7A-5BCD-2715-58A7-4DD23F70B1FE}
30+
{443B93EE-5877-B8D2-3E73-6AA63E426C0F} = {D12D9C30-2E74-3E87-8F12-FB4BFDB3464B}
31+
EndGlobalSection
32+
GlobalSection(ExtensibilityGlobals) = postSolution
33+
SolutionGuid = {4F24302C-87F4-4E43-828F-7B9B0589000F}
34+
EndGlobalSection
35+
EndGlobal

internal/commands/scan.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,16 @@ func scanCreateSubCommand(
707707
0,
708708
"Cancel the scan and fail after the timeout in minutes",
709709
)
710+
createScanCmd.PersistentFlags().Int(
711+
commonParams.ScanEnqueueRetriesFlag,
712+
0,
713+
"Number of retry attempts for scan enqueue failures due to queue capacity (default: 0, no retries)",
714+
)
715+
createScanCmd.PersistentFlags().Int(
716+
commonParams.ScanEnqueueRetryDelayFlag,
717+
5,
718+
"Base delay in seconds between scan enqueue retry attempts with exponential backoff (default: 5)",
719+
)
710720
createScanCmd.PersistentFlags().StringP(
711721
commonParams.SourcesFlag,
712722
commonParams.SourcesFlagSh,
@@ -858,6 +868,14 @@ func scanCreateSubCommand(
858868
if err != nil {
859869
log.Fatal(err)
860870
}
871+
err = viper.BindPFlag(commonParams.ScanEnqueueRetriesKey, createScanCmd.PersistentFlags().Lookup(commonParams.ScanEnqueueRetriesFlag))
872+
if err != nil {
873+
log.Fatal(err)
874+
}
875+
err = viper.BindPFlag(commonParams.ScanEnqueueRetryDelayKey, createScanCmd.PersistentFlags().Lookup(commonParams.ScanEnqueueRetryDelayFlag))
876+
if err != nil {
877+
log.Fatal(err)
878+
}
861879

862880
createScanCmd.PersistentFlags().String(commonParams.SSHKeyFlag, "", "Path to ssh private key")
863881

internal/params/envs.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ const (
44
CustomStatesAPIPathEnv = "CX_CUSTOM_STATES_PATH"
55
TenantEnv = "CX_TENANT"
66
BranchEnv = "CX_BRANCH"
7+
ScanEnqueueRetriesEnv = "CX_SCAN_ENQUEUE_RETRIES"
8+
ScanEnqueueRetryDelayEnv = "CX_SCAN_ENQUEUE_RETRY_DELAY"
79
BaseURIEnv = "CX_BASE_URI"
810
ClientTimeoutEnv = "CX_TIMEOUT"
911
ProxyEnv = "HTTP_PROXY"

internal/params/flags.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ const (
3333
AsyncFlag = "async"
3434
WaitDelayFlag = "wait-delay"
3535
ScanTimeoutFlag = "scan-timeout"
36+
ScanEnqueueRetriesFlag = "scan-enqueue-retries"
37+
ScanEnqueueRetryDelayFlag = "scan-enqueue-retry-delay"
3638
PolicyTimeoutFlag = "policy-timeout"
3739
IgnorePolicyFlag = "ignore-policy"
3840
SourceDirFilterFlag = "file-filter"

internal/params/keys.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ var (
66
CustomStatesAPIPathKey = strings.ToLower(CustomStatesAPIPathEnv)
77
TenantKey = strings.ToLower(TenantEnv)
88
BranchKey = strings.ToLower(BranchEnv)
9+
ScanEnqueueRetriesKey = strings.ToLower(ScanEnqueueRetriesEnv)
10+
ScanEnqueueRetryDelayKey = strings.ToLower(ScanEnqueueRetryDelayEnv)
911
BaseURIKey = strings.ToLower(BaseURIEnv)
1012
ProxyKey = strings.ToLower(ProxyEnv)
1113
ProxyTypeKey = strings.ToLower(ProxyTypeEnv)

internal/wrappers/mock/scans-mock.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ func (m *ScansMockWrapper) GetWorkflowByID(_ string) ([]*wrappers.ScanTaskRespon
2121

2222
func (m *ScansMockWrapper) Create(scanModel *wrappers.Scan) (*wrappers.ScanResponseModel, *wrappers.ErrorModel, error) {
2323
fmt.Println("Called Create in ScansMockWrapper")
24+
if scanModel.Project.ID == "fake-queue-capacity-error-id" {
25+
return nil, &wrappers.ErrorModel{
26+
Code: 142,
27+
Message: "Failed to enqueue scan. Max Queued reached",
28+
Type: "ERROR",
29+
}, nil
30+
}
2431
if scanModel.Project.ID == "fake-kics-scanner-fail-id" {
2532
return &wrappers.ScanResponseModel{
2633
ID: "fake-scan-id-kics-scanner-fail",

internal/wrappers/scans-http.go

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,19 @@ import (
55
"encoding/json"
66
"fmt"
77
"net/http"
8+
"time"
89

10+
"github.com/checkmarx/ast-cli/internal/logger"
911
commonParams "github.com/checkmarx/ast-cli/internal/params"
1012
"github.com/pkg/errors"
1113
"github.com/spf13/viper"
1214
)
1315

1416
const (
15-
failedToParseGetAll = "Failed to parse list response"
16-
failedToParseTags = "Failed to parse tags response"
17-
failedToParseBranches = "Failed to parse branches response"
17+
failedToParseGetAll = "Failed to parse list response"
18+
failedToParseTags = "Failed to parse tags response"
19+
failedToParseBranches = "Failed to parse branches response"
20+
queueCapacityErrorCode = 142
1821
)
1922

2023
type ScansHTTPWrapper struct {
@@ -29,26 +32,61 @@ func NewHTTPScansWrapper(path string) ScansWrapper {
2932
}
3033
}
3134

35+
// isQueueCapacityError checks if the error is due to queue capacity limits (error code 142)
36+
func isQueueCapacityError(errorModel *ErrorModel) bool {
37+
return errorModel != nil && errorModel.Code == queueCapacityErrorCode
38+
}
39+
3240
func (s *ScansHTTPWrapper) Create(model *Scan) (*ScanResponseModel, *ErrorModel, error) {
3341
clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
3442
jsonBytes, err := json.Marshal(model)
3543
if err != nil {
3644
return nil, nil, err
3745
}
3846

39-
fn := func() (*http.Response, error) {
40-
return SendHTTPRequest(http.MethodPost, s.path, bytes.NewBuffer(jsonBytes), true, clientTimeout)
41-
}
42-
resp, err := retryHTTPRequest(fn, retryAttempts, retryDelay)
43-
if err != nil {
44-
return nil, nil, err
45-
}
46-
defer func() {
47-
if err == nil {
48-
_ = resp.Body.Close()
47+
// Get scan enqueue retry configuration
48+
scanEnqueueRetries := viper.GetInt(commonParams.ScanEnqueueRetriesKey)
49+
scanEnqueueRetryDelay := viper.GetInt(commonParams.ScanEnqueueRetryDelayKey)
50+
51+
var scanResp *ScanResponseModel
52+
var errorModel *ErrorModel
53+
54+
// Retry loop for scan creation (queue capacity errors)
55+
for attempt := 0; attempt <= scanEnqueueRetries; attempt++ {
56+
// Standard HTTP retry (for 502, 401)
57+
fn := func() (*http.Response, error) {
58+
return SendHTTPRequest(http.MethodPost, s.path, bytes.NewBuffer(jsonBytes), true, clientTimeout)
4959
}
50-
}()
51-
return handleScanResponseWithBody(resp, err, http.StatusCreated)
60+
resp, err := retryHTTPRequest(fn, retryAttempts, retryDelay)
61+
if err != nil {
62+
return nil, nil, err
63+
}
64+
65+
// Parse response
66+
scanResp, errorModel, err = handleScanResponseWithBody(resp, err, http.StatusCreated)
67+
68+
// Close response body
69+
_ = resp.Body.Close()
70+
71+
// Check if it's a queue capacity error and we have retries left
72+
if isQueueCapacityError(errorModel) && attempt < scanEnqueueRetries {
73+
// Calculate exponential backoff delay
74+
waitDuration := time.Duration(scanEnqueueRetryDelay) * time.Second * (1 << attempt)
75+
logger.PrintIfVerbose(fmt.Sprintf(
76+
"Scan creation failed due to queue capacity (attempt %d/%d). Waiting %v before retry...",
77+
attempt+1,
78+
scanEnqueueRetries,
79+
waitDuration,
80+
))
81+
time.Sleep(waitDuration)
82+
continue
83+
}
84+
85+
// Success or non-retryable error - break out of loop
86+
break
87+
}
88+
89+
return scanResp, errorModel, err
5290
}
5391

5492
func (s *ScansHTTPWrapper) Get(params map[string]string) (*ScansCollectionResponseModel, *ErrorModel, error) {

0 commit comments

Comments
 (0)