fix: hotfix for v3.17.0 set maximum of goroutines

.github/workflows/security.yml

@@ -33,37 +33,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0
 
-  trivy-scanning:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Source
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-      - name: Build and load (not push)
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
-        with:
-          load: true
-          context: .
-          file: ./Dockerfile
-          platforms: linux/amd64
-          push: false
-          tags: checkmarx/2ms:scanme
-
-      - name: Run Trivy Scan
-        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0
-        with:
-          image-ref: checkmarx/2ms:scanme
-          vuln-type: os,library
-          format: table
-          ignore-unfixed: true
-          severity: CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN
-          trivy-config: trivy.yaml
-          exit-code: '1'
-        env:
-          TRIVY_SKIP_DB_UPDATE: true
-          TRIVY_SKIP_JAVA_DB_UPDATE: true
-
-
   secret-scanning:
     runs-on: ubuntu-latest
     steps:

.github/workflows/update-trivy-cache.yml → .github/workflows/trivy-cache.yaml

@@ -36,4 +36,4 @@ jobs:
         uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2.0
         with:
           path: ${{ github.workspace }}/.cache/trivy
-          key: cache-trivy-${{ steps.date.outputs.date }}
+          key: cache-trivy-${{ steps.date.outputs.date }}

.github/workflows/trivy-vulnerability-scan.yaml

@@ -0,0 +1,40 @@
+name: Trivy-scan
+on:
+  push:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: '5 6 * * *'  # Runs every day at 06:05 UTC
+
+jobs:
+  trivy-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Build and load (not push)
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          load: true
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64
+          push: false
+          tags: checkmarx/2ms:scanme
+
+      - name: Run Trivy Scan
+        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0
+        with:
+          image-ref: checkmarx/2ms:scanme
+          vuln-type: os,library
+          format: table
+          ignore-unfixed: true
+          severity: CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN
+          trivy-config: trivy.yaml
+          exit-code: '1'
+        env:
+          TRIVY_SKIP_DB_UPDATE: true
+          TRIVY_SKIP_JAVA_DB_UPDATE: true

Dockerfile

@@ -3,7 +3,7 @@
 # and "Missing User Instruction" since 2ms container is stopped after scan
 
 # Builder image
-FROM cgr.dev/chainguard/go@sha256:2453e92671fb693999e65fde99bbd5744b120b7dd70f3f7c7b220e185ec35050 AS builder
+FROM cgr.dev/chainguard/go@sha256:7f9e74e1af376a6d238077d8df037a25001997581630bc121c8aecfa5c8da8b3 AS builder
 
 WORKDIR /app
 
@@ -20,7 +20,7 @@ COPY . .
 RUN GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o /app/2ms .
 
 # Runtime image
-FROM cgr.dev/chainguard/git@sha256:9e3ec4c4f1465ac810a7e4335d458582c43ad4e8dbaf8ab3a74f8f2a7fdffec2
+FROM cgr.dev/chainguard/git@sha256:fb9f28194b4dda3ea74c68d731238d1f32023849bca04c5652638e8e199fb956
 
 WORKDIR /app
 

cmd/workers.go

@@ -1,23 +1,24 @@
 package cmd
 
 import (
-	"github.com/checkmarx/2ms/lib/secrets"
-	"sync"
-
 	"github.com/checkmarx/2ms/engine"
 	"github.com/checkmarx/2ms/engine/extra"
+	"golang.org/x/sync/errgroup"
 )
 
 func processItems(engine *engine.Engine, pluginName string) {
 	defer channels.WaitGroup.Done()
 
-	wgItems := &sync.WaitGroup{}
+	g := errgroup.Group{}
+	g.SetLimit(1000)
 	for item := range channels.Items {
 		report.TotalItemsScanned++
-		wgItems.Add(1)
-		go engine.Detect(item, secretsChan, wgItems, pluginName, channels.Errors)
+		g.Go(func() error {
+			engine.Detect(item, secretsChan, pluginName, channels.Errors)
+			return nil
+		})
 	}
-	wgItems.Wait()
+	g.Wait()
 	close(secretsChan)
 }
 
@@ -42,37 +43,43 @@ func processSecrets() {
 func processSecretsExtras() {
 	defer channels.WaitGroup.Done()
 
-	wgExtras := &sync.WaitGroup{}
+	g := errgroup.Group{}
+	g.SetLimit(10)
 	for secret := range secretsExtrasChan {
-		wgExtras.Add(1)
-		go extra.AddExtraToSecret(secret, wgExtras)
+		g.Go(func() error {
+			extra.AddExtraToSecret(secret)
+			return nil
+		})
 	}
-	wgExtras.Wait()
+	g.Wait()
 }
 
 func processValidationAndScoreWithValidation(engine *engine.Engine) {
 	defer channels.WaitGroup.Done()
 
-	wgValidation := &sync.WaitGroup{}
+	g := errgroup.Group{}
+	g.SetLimit(10)
 	for secret := range validationChan {
-		wgValidation.Add(2)
-		go func(secret *secrets.Secret, wg *sync.WaitGroup) {
-			engine.RegisterForValidation(secret, wg)
-			engine.Score(secret, true, wg)
-		}(secret, wgValidation)
+		g.Go(func() error {
+			engine.RegisterForValidation(secret)
+			engine.Score(secret, true)
+			return nil
+		})
 	}
-	wgValidation.Wait()
-
+	g.Wait()
 	engine.Validate()
 }
 
 func processScoreWithoutValidation(engine *engine.Engine) {
 	defer channels.WaitGroup.Done()
 
-	wgScore := &sync.WaitGroup{}
+	g := errgroup.Group{}
+	g.SetLimit(10)
 	for secret := range cvssScoreWithoutValidationChan {
-		wgScore.Add(1)
-		go engine.Score(secret, false, wgScore)
+		g.Go(func() error {
+			engine.Score(secret, false)
+			return nil
+		})
 	}
-	wgScore.Wait()
+	g.Wait()
 }

engine/engine.go

@@ -3,14 +3,14 @@ package engine
 import (
 	"crypto/sha1"
 	"fmt"
-	"github.com/checkmarx/2ms/engine/linecontent"
-	"github.com/checkmarx/2ms/engine/score"
 	"os"
 	"regexp"
 	"strings"
-	"sync"
 	"text/tabwriter"
 
+	"github.com/checkmarx/2ms/engine/linecontent"
+	"github.com/checkmarx/2ms/engine/score"
+
 	"github.com/checkmarx/2ms/engine/rules"
 	"github.com/checkmarx/2ms/engine/validation"
 	"github.com/checkmarx/2ms/lib/secrets"
@@ -78,9 +78,7 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 	}, nil
 }
 
-func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) {
-	defer wg.Done()
-
+func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, pluginName string, errors chan error) {
 	fragment := detect.Fragment{
 		Raw:      *item.GetContent(),
 		FilePath: item.GetSource(),
@@ -137,13 +135,11 @@ func (e *Engine) AddRegexRules(patterns []string) error {
 	return nil
 }
 
-func (s *Engine) RegisterForValidation(secret *secrets.Secret, wg *sync.WaitGroup) {
-	defer wg.Done()
+func (s *Engine) RegisterForValidation(secret *secrets.Secret) {
 	s.validator.RegisterForValidation(secret)
 }
 
-func (s *Engine) Score(secret *secrets.Secret, validateFlag bool, wg *sync.WaitGroup) {
-	defer wg.Done()
+func (s *Engine) Score(secret *secrets.Secret, validateFlag bool) {
 	validationStatus := secrets.UnknownResult // default validity
 	if validateFlag {
 		validationStatus = secret.ValidationStatus

engine/engine_test.go

@@ -2,10 +2,10 @@ package engine
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/assert"
-	"sync"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/checkmarx/2ms/engine/rules"
 	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/checkmarx/2ms/plugins"
@@ -79,9 +79,7 @@ func TestDetector(t *testing.T) {
 
 		secretsChan := make(chan *secrets.Secret, 1)
 		errorsChan := make(chan error, 1)
-		wg := &sync.WaitGroup{}
-		wg.Add(1)
-		detector.Detect(i, secretsChan, wg, fsPlugin.GetName(), errorsChan)
+		detector.Detect(i, secretsChan, fsPlugin.GetName(), errorsChan)
 		close(secretsChan)
 
 		s := <-secretsChan
@@ -155,9 +153,7 @@ func TestSecrets(t *testing.T) {
 			fmt.Printf("Start test %s", name)
 			secretsChan := make(chan *secrets.Secret, 1)
 			errorsChan := make(chan error, 1)
-			wg := &sync.WaitGroup{}
-			wg.Add(1)
-			detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName(), errorsChan)
+			detector.Detect(item{content: &secret.Content}, secretsChan, fsPlugin.GetName(), errorsChan)
 			close(secretsChan)
 			close(errorsChan)
 

engine/extra/extra.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
-	"sync"
 
 	"github.com/checkmarx/2ms/lib/secrets"
 )
@@ -16,8 +15,7 @@ var ruleIDToFunction = map[string]addExtraFunc{
 	"jwt": addExtraJWT,
 }
 
-func AddExtraToSecret(secret *secrets.Secret, wg *sync.WaitGroup) {
-	defer wg.Done()
+func AddExtraToSecret(secret *secrets.Secret) {
 	if addExtra, ok := ruleIDToFunction[secret.RuleID]; ok {
 		extraData := addExtra(secret)
 		if extraData != nil && extraData != "" {

engine/extra/extra_test.go

@@ -3,10 +3,10 @@ package extra
 import (
 	"encoding/base64"
 	"fmt"
+	"testing"
+
 	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/stretchr/testify/assert"
-	"sync"
-	"testing"
 )
 
 func TestAddExtraToSecret(t *testing.T) {
@@ -50,10 +50,7 @@ func TestAddExtraToSecret(t *testing.T) {
 				ExtraDetails: make(map[string]interface{}),
 			}
 
-			var wg sync.WaitGroup
-			wg.Add(1)
-			AddExtraToSecret(secret, &wg)
-			wg.Wait()
+			AddExtraToSecret(secret)
 
 			assert.Equal(t, tt.expectedOutput, secret.ExtraDetails["secretDetails"])
 		})

engine/score/score_test.go

@@ -1,14 +1,15 @@
 package score_test
 
 import (
+	"sync"
+	"testing"
+
 	. "github.com/checkmarx/2ms/engine"
 	"github.com/checkmarx/2ms/engine/rules"
 	"github.com/checkmarx/2ms/engine/score"
 	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/stretchr/testify/assert"
 	ruleConfig "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules"
-	"sync"
-	"testing"
 )
 
 func TestScore(t *testing.T) {
@@ -216,9 +217,9 @@ func TestScore(t *testing.T) {
 		expectedRuleScores := expectedCvssScores[secret.RuleID]
 		validityIndex := getValidityIndex(secret.ValidationStatus)
 		unknownIndex := getValidityIndex(secrets.UnknownResult)
-		engine.Score(secret, true, &wg)
+		engine.Score(secret, true)
 		assert.Equal(t, expectedRuleScores[validityIndex], secret.CvssScore, "rule: %s", secret.RuleID)
-		engine.Score(secret, false, &wg)
+		engine.Score(secret, false)
 		assert.Equal(t, expectedRuleScores[unknownIndex], secret.CvssScore, "rule: %s", secret.RuleID)
 	}
 }

engine/validation/pairs.go

@@ -1,8 +1,6 @@
 package validation
 
 import (
-	"sync"
-
 	"github.com/checkmarx/2ms/lib/secrets"
 )
 
@@ -38,8 +36,7 @@ func (p *pairsCollector) addIfNeeded(secret *secrets.Secret) bool {
 	return true
 }
 
-func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) {
-	defer wg.Done()
+func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId) {
 	generalKeyToValidation[generalKey](rulesById)
 }
 

engine/validation/validator.go

@@ -1,8 +1,6 @@
 package validation
 
 import (
-	"sync"
-
 	"github.com/checkmarx/2ms/engine/extra"
 	"github.com/checkmarx/2ms/lib/secrets"
 )
@@ -35,14 +33,11 @@ func (v *Validator) RegisterForValidation(secret *secrets.Secret) {
 }
 
 func (v *Validator) Validate() {
-	wg := &sync.WaitGroup{}
 	for generalKey, bySource := range v.pairsCollector.pairs {
 		for _, byRule := range bySource {
-			wg.Add(1)
-			v.pairsCollector.validate(generalKey, byRule, wg)
+			v.pairsCollector.validate(generalKey, byRule)
 		}
 	}
-	wg.Wait()
 }
 
 func IsCanValidateRule(ruleID string) bool {