From a95cd8521d91194d18ccb902a18ecf4a2f1fa1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Dalot?= <206845486+cx-rogerio-dalot-x@users.noreply.github.com> Date: Wed, 4 Jun 2025 13:20:19 +0100 Subject: [PATCH 1/4] chore: set trivy flags to skip DB (#274) --- .github/workflows/security.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 1298731c..4178ad23 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -60,8 +60,8 @@ jobs: trivy-config: trivy.yaml exit-code: '1' env: - TRIVY_SKIP_DB_UPDATE: false - TRIVY_SKIP_JAVA_DB_UPDATE: false + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true secret-scanning: From 544c49400ae67c0c259f8c4b16db1f2ea2be9e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Dalot?= <206845486+cx-rogerio-dalot-x@users.noreply.github.com> Date: Wed, 4 Jun 2025 13:24:01 +0100 Subject: [PATCH 2/4] fix: set maximum of goroutines --- cmd/workers.go | 53 +++++++++++++++++++--------------- engine/engine.go | 16 ++++------ engine/engine_test.go | 12 +++----- engine/extra/extra.go | 4 +-- engine/extra/extra_test.go | 9 ++---- engine/score/score_test.go | 9 +++--- engine/validation/pairs.go | 5 +--- engine/validation/validator.go | 7 +---- go.mod | 2 +- plugins/filesystem.go | 24 +++++++-------- plugins/filesystem_test.go | 8 ++--- 11 files changed, 67 insertions(+), 82 deletions(-) diff --git a/cmd/workers.go b/cmd/workers.go index 497e8c5d..1c5922ca 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -1,23 +1,24 @@ package cmd import ( - "github.com/checkmarx/2ms/lib/secrets" - "sync" - "github.com/checkmarx/2ms/engine" "github.com/checkmarx/2ms/engine/extra" + "golang.org/x/sync/errgroup" ) func processItems(engine *engine.Engine, pluginName string) { defer channels.WaitGroup.Done() - wgItems := &sync.WaitGroup{} + g := errgroup.Group{} + g.SetLimit(1000) for item := range channels.Items { report.TotalItemsScanned++ - wgItems.Add(1) - go engine.Detect(item, secretsChan, wgItems, pluginName, channels.Errors) + g.Go(func() error { + engine.Detect(item, secretsChan, pluginName, channels.Errors) + return nil + }) } - wgItems.Wait() + g.Wait() close(secretsChan) } @@ -42,37 +43,43 @@ func processSecrets() { func processSecretsExtras() { defer channels.WaitGroup.Done() - wgExtras := &sync.WaitGroup{} + g := errgroup.Group{} + g.SetLimit(10) for secret := range secretsExtrasChan { - wgExtras.Add(1) - go extra.AddExtraToSecret(secret, wgExtras) + g.Go(func() error { + extra.AddExtraToSecret(secret) + return nil + }) } - wgExtras.Wait() + g.Wait() } func processValidationAndScoreWithValidation(engine *engine.Engine) { defer channels.WaitGroup.Done() - wgValidation := &sync.WaitGroup{} + g := errgroup.Group{} + g.SetLimit(10) for secret := range validationChan { - wgValidation.Add(2) - go func(secret *secrets.Secret, wg *sync.WaitGroup) { - engine.RegisterForValidation(secret, wg) - engine.Score(secret, true, wg) - }(secret, wgValidation) + g.Go(func() error { + engine.RegisterForValidation(secret) + engine.Score(secret, true) + return nil + }) } - wgValidation.Wait() - + g.Wait() engine.Validate() } func processScoreWithoutValidation(engine *engine.Engine) { defer channels.WaitGroup.Done() - wgScore := &sync.WaitGroup{} + g := errgroup.Group{} + g.SetLimit(10) for secret := range cvssScoreWithoutValidationChan { - wgScore.Add(1) - go engine.Score(secret, false, wgScore) + g.Go(func() error { + engine.Score(secret, false) + return nil + }) } - wgScore.Wait() + g.Wait() } diff --git a/engine/engine.go b/engine/engine.go index 03a69353..46d13b69 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -3,14 +3,14 @@ package engine import ( "crypto/sha1" "fmt" - "github.com/checkmarx/2ms/engine/linecontent" - "github.com/checkmarx/2ms/engine/score" "os" "regexp" "strings" - "sync" "text/tabwriter" + "github.com/checkmarx/2ms/engine/linecontent" + "github.com/checkmarx/2ms/engine/score" + "github.com/checkmarx/2ms/engine/rules" "github.com/checkmarx/2ms/engine/validation" "github.com/checkmarx/2ms/lib/secrets" @@ -78,9 +78,7 @@ func Init(engineConfig EngineConfig) (*Engine, error) { }, nil } -func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) { - defer wg.Done() - +func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, pluginName string, errors chan error) { fragment := detect.Fragment{ Raw: *item.GetContent(), FilePath: item.GetSource(), @@ -137,13 +135,11 @@ func (e *Engine) AddRegexRules(patterns []string) error { return nil } -func (s *Engine) RegisterForValidation(secret *secrets.Secret, wg *sync.WaitGroup) { - defer wg.Done() +func (s *Engine) RegisterForValidation(secret *secrets.Secret) { s.validator.RegisterForValidation(secret) } -func (s *Engine) Score(secret *secrets.Secret, validateFlag bool, wg *sync.WaitGroup) { - defer wg.Done() +func (s *Engine) Score(secret *secrets.Secret, validateFlag bool) { validationStatus := secrets.UnknownResult // default validity if validateFlag { validationStatus = secret.ValidationStatus diff --git a/engine/engine_test.go b/engine/engine_test.go index d31fa39e..3db7490b 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -2,10 +2,10 @@ package engine import ( "fmt" - "github.com/stretchr/testify/assert" - "sync" "testing" + "github.com/stretchr/testify/assert" + "github.com/checkmarx/2ms/engine/rules" "github.com/checkmarx/2ms/lib/secrets" "github.com/checkmarx/2ms/plugins" @@ -79,9 +79,7 @@ func TestDetector(t *testing.T) { secretsChan := make(chan *secrets.Secret, 1) errorsChan := make(chan error, 1) - wg := &sync.WaitGroup{} - wg.Add(1) - detector.Detect(i, secretsChan, wg, fsPlugin.GetName(), errorsChan) + detector.Detect(i, secretsChan, fsPlugin.GetName(), errorsChan) close(secretsChan) s := <-secretsChan @@ -155,9 +153,7 @@ func TestSecrets(t *testing.T) { fmt.Printf("Start test %s", name) secretsChan := make(chan *secrets.Secret, 1) errorsChan := make(chan error, 1) - wg := &sync.WaitGroup{} - wg.Add(1) - detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName(), errorsChan) + detector.Detect(item{content: &secret.Content}, secretsChan, fsPlugin.GetName(), errorsChan) close(secretsChan) close(errorsChan) diff --git a/engine/extra/extra.go b/engine/extra/extra.go index 638e4855..351d5a13 100644 --- a/engine/extra/extra.go +++ b/engine/extra/extra.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "strings" - "sync" "github.com/checkmarx/2ms/lib/secrets" ) @@ -16,8 +15,7 @@ var ruleIDToFunction = map[string]addExtraFunc{ "jwt": addExtraJWT, } -func AddExtraToSecret(secret *secrets.Secret, wg *sync.WaitGroup) { - defer wg.Done() +func AddExtraToSecret(secret *secrets.Secret) { if addExtra, ok := ruleIDToFunction[secret.RuleID]; ok { extraData := addExtra(secret) if extraData != nil && extraData != "" { diff --git a/engine/extra/extra_test.go b/engine/extra/extra_test.go index c4fce11e..521ee727 100644 --- a/engine/extra/extra_test.go +++ b/engine/extra/extra_test.go @@ -3,10 +3,10 @@ package extra import ( "encoding/base64" "fmt" + "testing" + "github.com/checkmarx/2ms/lib/secrets" "github.com/stretchr/testify/assert" - "sync" - "testing" ) func TestAddExtraToSecret(t *testing.T) { @@ -50,10 +50,7 @@ func TestAddExtraToSecret(t *testing.T) { ExtraDetails: make(map[string]interface{}), } - var wg sync.WaitGroup - wg.Add(1) - AddExtraToSecret(secret, &wg) - wg.Wait() + AddExtraToSecret(secret) assert.Equal(t, tt.expectedOutput, secret.ExtraDetails["secretDetails"]) }) diff --git a/engine/score/score_test.go b/engine/score/score_test.go index 4515a4a0..64186e25 100644 --- a/engine/score/score_test.go +++ b/engine/score/score_test.go @@ -1,14 +1,15 @@ package score_test import ( + "sync" + "testing" + . "github.com/checkmarx/2ms/engine" "github.com/checkmarx/2ms/engine/rules" "github.com/checkmarx/2ms/engine/score" "github.com/checkmarx/2ms/lib/secrets" "github.com/stretchr/testify/assert" ruleConfig "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" - "sync" - "testing" ) func TestScore(t *testing.T) { @@ -216,9 +217,9 @@ func TestScore(t *testing.T) { expectedRuleScores := expectedCvssScores[secret.RuleID] validityIndex := getValidityIndex(secret.ValidationStatus) unknownIndex := getValidityIndex(secrets.UnknownResult) - engine.Score(secret, true, &wg) + engine.Score(secret, true) assert.Equal(t, expectedRuleScores[validityIndex], secret.CvssScore, "rule: %s", secret.RuleID) - engine.Score(secret, false, &wg) + engine.Score(secret, false) assert.Equal(t, expectedRuleScores[unknownIndex], secret.CvssScore, "rule: %s", secret.RuleID) } } diff --git a/engine/validation/pairs.go b/engine/validation/pairs.go index 68ed1e0d..0f6e13a2 100644 --- a/engine/validation/pairs.go +++ b/engine/validation/pairs.go @@ -1,8 +1,6 @@ package validation import ( - "sync" - "github.com/checkmarx/2ms/lib/secrets" ) @@ -38,8 +36,7 @@ func (p *pairsCollector) addIfNeeded(secret *secrets.Secret) bool { return true } -func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) { - defer wg.Done() +func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId) { generalKeyToValidation[generalKey](rulesById) } diff --git a/engine/validation/validator.go b/engine/validation/validator.go index ae8e8a18..f1969a6b 100644 --- a/engine/validation/validator.go +++ b/engine/validation/validator.go @@ -1,8 +1,6 @@ package validation import ( - "sync" - "github.com/checkmarx/2ms/engine/extra" "github.com/checkmarx/2ms/lib/secrets" ) @@ -35,14 +33,11 @@ func (v *Validator) RegisterForValidation(secret *secrets.Secret) { } func (v *Validator) Validate() { - wg := &sync.WaitGroup{} for generalKey, bySource := range v.pairsCollector.pairs { for _, byRule := range bySource { - wg.Add(1) - v.pairsCollector.validate(generalKey, byRule, wg) + v.pairsCollector.validate(generalKey, byRule) } } - wg.Wait() } func IsCanValidateRule(ruleID string) bool { diff --git a/go.mod b/go.mod index 003a49b1..13cbb14d 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e github.com/stretchr/testify v1.9.0 github.com/zricethezav/gitleaks/v8 v8.18.2 + golang.org/x/sync v0.10.0 golang.org/x/time v0.5.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -48,7 +49,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/plugins/filesystem.go b/plugins/filesystem.go index fc4138ec..f349f503 100644 --- a/plugins/filesystem.go +++ b/plugins/filesystem.go @@ -4,11 +4,11 @@ import ( "fmt" "os" "path/filepath" - "sync" "time" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" ) const ( @@ -38,9 +38,7 @@ func (p *FileSystemPlugin) DefineCommand(items chan ISourceItem, errors chan err Run: func(cmd *cobra.Command, args []string) { log.Info().Msg("Folder plugin started") - wg := &sync.WaitGroup{} - p.getFiles(items, errors, wg) - wg.Wait() + p.getFiles(items, errors) close(items) }, } @@ -60,7 +58,7 @@ func (p *FileSystemPlugin) DefineCommand(items chan ISourceItem, errors chan err return cmd, nil } -func (p *FileSystemPlugin) getFiles(items chan ISourceItem, errs chan error, wg *sync.WaitGroup) { +func (p *FileSystemPlugin) getFiles(items chan ISourceItem, errs chan error) { fileList := make([]string, 0) err := filepath.Walk(p.Path, func(path string, fInfo os.FileInfo, err error) error { if err != nil { @@ -98,23 +96,25 @@ func (p *FileSystemPlugin) getFiles(items chan ISourceItem, errs chan error, wg return } - p.getItems(items, errs, wg, fileList) + p.getItems(items, errs, fileList) } -func (p *FileSystemPlugin) getItems(items chan ISourceItem, errs chan error, wg *sync.WaitGroup, fileList []string) { +func (p *FileSystemPlugin) getItems(items chan ISourceItem, errs chan error, fileList []string) { + g := errgroup.Group{} + g.SetLimit(1000) for _, filePath := range fileList { - wg.Add(1) - go func(filePath string) { - defer wg.Done() + g.Go(func() error { actualFile, err := p.getItem(filePath) if err != nil { errs <- err time.Sleep(time.Second) // Temporary fix for incorrect non-error exits; needs a better solution. - return + return nil } items <- *actualFile - }(filePath) + return nil + }) } + g.Wait() } func (p *FileSystemPlugin) getItem(filePath string) (*item, error) { diff --git a/plugins/filesystem_test.go b/plugins/filesystem_test.go index 2f631113..cdd063ce 100644 --- a/plugins/filesystem_test.go +++ b/plugins/filesystem_test.go @@ -2,11 +2,12 @@ package plugins import ( "fmt" - "github.com/stretchr/testify/assert" "os" "path/filepath" "sync" "testing" + + "github.com/stretchr/testify/assert" ) func TestGetItem(t *testing.T) { @@ -58,15 +59,12 @@ func TestGetItems(t *testing.T) { itemsChan := make(chan ISourceItem, len(fileList)) errsChan := make(chan error, len(fileList)) - var wg sync.WaitGroup plugin := &FileSystemPlugin{ ProjectName: "TestProject", } - plugin.getItems(itemsChan, errsChan, &wg, fileList) - - wg.Wait() + plugin.getItems(itemsChan, errsChan, fileList) close(itemsChan) close(errsChan) From ab7013b8f09cf6c209afc8732023e3535a897aae Mon Sep 17 00:00:00 2001 From: Lior Poterman <191881919+cx-lior-poterman@users.noreply.github.com> Date: Fri, 30 May 2025 18:17:01 +0300 Subject: [PATCH 3/4] chore: cx standardization (#276) Closes # **Proposed Changes** **Checklist** - [ ] I covered my changes with tests. - [ ] I Updated the documentation that is affected by my changes: - [ ] Change in the CLI arguments - [ ] Change in the configuration file I submit this contribution under the Apache-2.0 license. --------- Co-authored-by: cx-leonardo-fontes <204389152+cx-leonardo-fontes@users.noreply.github.com> --- .github/workflows/security.yml | 31 -------------- ...pdate-trivy-cache.yml => trivy-cache.yaml} | 2 +- .../workflows/trivy-vulnerability-scan.yaml | 40 +++++++++++++++++++ Dockerfile | 2 +- trivy-whitelist.openvex | 9 +++++ 5 files changed, 51 insertions(+), 33 deletions(-) rename .github/workflows/{update-trivy-cache.yml => trivy-cache.yaml} (95%) create mode 100644 .github/workflows/trivy-vulnerability-scan.yaml create mode 100644 trivy-whitelist.openvex diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 4178ad23..81ede610 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -33,37 +33,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0 - trivy-scanning: - runs-on: ubuntu-latest - steps: - - name: Checkout Source - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Build and load (not push) - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 - with: - load: true - context: . - file: ./Dockerfile - platforms: linux/amd64 - push: false - tags: checkmarx/2ms:scanme - - - name: Run Trivy Scan - uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0 - with: - image-ref: checkmarx/2ms:scanme - vuln-type: os,library - format: table - ignore-unfixed: true - severity: CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN - trivy-config: trivy.yaml - exit-code: '1' - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - - secret-scanning: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/update-trivy-cache.yml b/.github/workflows/trivy-cache.yaml similarity index 95% rename from .github/workflows/update-trivy-cache.yml rename to .github/workflows/trivy-cache.yaml index 394ceb53..bf3887e8 100644 --- a/.github/workflows/update-trivy-cache.yml +++ b/.github/workflows/trivy-cache.yaml @@ -36,4 +36,4 @@ jobs: uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2.0 with: path: ${{ github.workspace }}/.cache/trivy - key: cache-trivy-${{ steps.date.outputs.date }} \ No newline at end of file + key: cache-trivy-${{ steps.date.outputs.date }} diff --git a/.github/workflows/trivy-vulnerability-scan.yaml b/.github/workflows/trivy-vulnerability-scan.yaml new file mode 100644 index 00000000..8b23fb50 --- /dev/null +++ b/.github/workflows/trivy-vulnerability-scan.yaml @@ -0,0 +1,40 @@ +name: Trivy-scan +on: + push: + workflow_dispatch: + pull_request: + branches: + - master + schedule: + - cron: '5 6 * * *' # Runs every day at 06:05 UTC + +jobs: + trivy-scan: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Build and load (not push) + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + load: true + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: false + tags: checkmarx/2ms:scanme + + - name: Run Trivy Scan + uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0 + with: + image-ref: checkmarx/2ms:scanme + vuln-type: os,library + format: table + ignore-unfixed: true + severity: CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN + trivy-config: trivy.yaml + exit-code: '1' + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/Dockerfile b/Dockerfile index 9fc998a3..f731a654 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ COPY . . RUN GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o /app/2ms . # Runtime image -FROM cgr.dev/chainguard/git@sha256:9e3ec4c4f1465ac810a7e4335d458582c43ad4e8dbaf8ab3a74f8f2a7fdffec2 +FROM cgr.dev/chainguard/git@sha256:fb9f28194b4dda3ea74c68d731238d1f32023849bca04c5652638e8e199fb956 WORKDIR /app diff --git a/trivy-whitelist.openvex b/trivy-whitelist.openvex new file mode 100644 index 00000000..de08cdaf --- /dev/null +++ b/trivy-whitelist.openvex @@ -0,0 +1,9 @@ +{ + "@context": "https://openvex.dev/ns", + "@id": "https://openvex.dev/docs/public/vex-2e67563e128250cbcb3e98930df948dd053e43271d70dc50cfa22d57e03fe96f", + "timestamp": "2024-05-08T16:00:16.853479631-06:00", + "version": 1, + "author":"Omer fainshtein", + "statements": [ + ] +} From 10ffadf154d78be1b535c585803548f60170b94e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Dalot?= <206845486+cx-rogerio-dalot-x@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:31:09 +0100 Subject: [PATCH 4/4] fix:fix test --- plugins/filesystem_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/filesystem_test.go b/plugins/filesystem_test.go index cdd063ce..03167199 100644 --- a/plugins/filesystem_test.go +++ b/plugins/filesystem_test.go @@ -186,7 +186,7 @@ func TestGetFiles(t *testing.T) { errsChan := make(chan error, 10) var wg sync.WaitGroup - plugin.getFiles(itemsChan, errsChan, &wg) + plugin.getFiles(itemsChan, errsChan) wg.Wait() close(itemsChan) close(errsChan)