diff --git a/.2ms.yml b/.2ms.yml index 46335dae..36fa6e70 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -29,3 +29,23 @@ ignore-result: - 33a14f1d1e4a1201a3e0062ebf09079fe8c84714 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go - da610673906f695e3e85bda6fc0a916762f01a70 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go - f8da5c56428cf708773be38269932c46aaf44cd4 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go + - 0d49f4953e8c5b2e04cca54d40bd2a91c079926b # value used for testing + - 5fb857fa72e8d568e6cfd96119d6b2db87c1e9b2 # value used for testing + - 6b92e79146584c6263671b7bcaac79a9c0852465 # value used for testing + - 22a792422372ef239494839d11c188258d18abc8 # value used for testing + - 29ce1990ca4555a207e77a66ffc26d46575a7911 # value used for testing + - 98a2f843609061bba58b69d4d31b70624de299ee # value used for testing + - 0188f28d26c2ae3f87df20092ab39c4465d6bbba # value used for testing + - 468bdfec08e1660b6ec73d78d15f03c320c68078 # value used for testing + - 5586d6fb77d9fa54224604ab158c2ceda4ab0995 # value used for testing + - 6403ca0ffb2abf3f1c9f70202474fb8f6564c4d7 # value used for testing + - a6fe66dfd9531c5415c1d1fed28b71f13a855a46 # value used for testing + - aaf4ba87a3bdbaf9346c0229f404eb86c0e6aabe # value used for testing + - b09e3219bca2cbcc4d7bc34f46e394e1f80d6574 # value used for testing + - c00a0d0af6bac8b20572bbb3b0b2cbea70476a0d # value used for testing + - c94ccae65acb14fdd2b9db7c9119e58875346a3b # value used for testing + - d4ac7947e0a7a4b387bf46279daa74e9dbe7f66f # value used for testing + - dd2e802e4c3205e57e291a89dfd469946531292b # value used for testing + - e475d6cf0a94469ea1717db008936a4e8749fe6a # value used for testing + - eed7c634d36422d7276cd8623c149e4c8d874f95 # value used for testing + - ff933778f18c92254c15369564b7d359f44018b5 # value used for testing diff --git a/Dockerfile b/Dockerfile index 9fc998a3..3f6adbb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # and "Missing User Instruction" since 2ms container is stopped after scan # Builder image -FROM cgr.dev/chainguard/go@sha256:2453e92671fb693999e65fde99bbd5744b120b7dd70f3f7c7b220e185ec35050 AS builder +FROM cgr.dev/chainguard/go@sha256:411f37ae52643cf040cfaca740aa78951009f3e7e399eef2ec797c153fe4c892 AS builder WORKDIR /app @@ -20,7 +20,7 @@ COPY . . RUN GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o /app/2ms . # Runtime image -FROM cgr.dev/chainguard/git@sha256:9e3ec4c4f1465ac810a7e4335d458582c43ad4e8dbaf8ab3a74f8f2a7fdffec2 +FROM cgr.dev/chainguard/git@sha256:c893f65bcc5d3de1c327af6db17566139af7663ef89001d536e8370226dcf881 WORKDIR /app diff --git a/cmd/main.go b/cmd/main.go index 628e66b6..b3ccf9a0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -66,17 +66,17 @@ var allPlugins = []plugins.IPlugin{ &plugins.GitPlugin{}, } -var channels = plugins.Channels{ +var Channels = plugins.Channels{ Items: make(chan plugins.ISourceItem), Errors: make(chan error), WaitGroup: &sync.WaitGroup{}, } -var report = reporting.Init() -var secretsChan = make(chan *secrets.Secret) -var secretsExtrasChan = make(chan *secrets.Secret) -var validationChan = make(chan *secrets.Secret) -var cvssScoreWithoutValidationChan = make(chan *secrets.Secret) +var Report = reporting.Init() +var SecretsChan = make(chan *secrets.Secret) +var SecretsExtrasChan = make(chan *secrets.Secret) +var ValidationChan = make(chan *secrets.Secret) +var CvssScoreWithoutValidationChan = make(chan *secrets.Secret) func Execute() (int, error) { vConfig.SetEnvPrefix(envPrefix) @@ -104,7 +104,7 @@ func Execute() (int, error) { rootCmd.AddGroup(&cobra.Group{Title: group, ID: group}) for _, plugin := range allPlugins { - subCommand, err := plugin.DefineCommand(channels.Items, channels.Errors) + subCommand, err := plugin.DefineCommand(Channels.Items, Channels.Errors) if err != nil { return 0, fmt.Errorf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error()) } @@ -116,13 +116,13 @@ func Execute() (int, error) { rootCmd.AddCommand(subCommand) } - listenForErrors(channels.Errors) + listenForErrors(Channels.Errors) if err := rootCmd.Execute(); err != nil { return 0, err } - return report.TotalSecretsFound, nil + return Report.TotalSecretsFound, nil } func preRun(pluginName string, cmd *cobra.Command, args []string) error { @@ -139,38 +139,38 @@ func preRun(pluginName string, cmd *cobra.Command, args []string) error { return err } - channels.WaitGroup.Add(1) - go processItems(engine, pluginName) + Channels.WaitGroup.Add(1) + go ProcessItems(engine, pluginName) - channels.WaitGroup.Add(1) - go processSecrets() + Channels.WaitGroup.Add(1) + go ProcessSecrets() - channels.WaitGroup.Add(1) - go processSecretsExtras() + Channels.WaitGroup.Add(1) + go ProcessSecretsExtras() if validateVar { - channels.WaitGroup.Add(1) - go processValidationAndScoreWithValidation(engine) + Channels.WaitGroup.Add(1) + go ProcessValidationAndScoreWithValidation(engine) } else { - channels.WaitGroup.Add(1) - go processScoreWithoutValidation(engine) + Channels.WaitGroup.Add(1) + go ProcessScoreWithoutValidation(engine) } return nil } func postRun(cmd *cobra.Command, args []string) error { - channels.WaitGroup.Wait() + Channels.WaitGroup.Wait() cfg := config.LoadConfig("2ms", Version) - if report.TotalItemsScanned > 0 { - if err := report.ShowReport(stdoutFormatVar, cfg); err != nil { + if Report.TotalItemsScanned > 0 { + if err := Report.ShowReport(stdoutFormatVar, cfg); err != nil { return err } if len(reportPathVar) > 0 { - err := report.WriteFile(reportPathVar, cfg) + err := Report.WriteFile(reportPathVar, cfg) if err != nil { return fmt.Errorf("failed to create report file with error: %s", err) } diff --git a/cmd/main_test.go b/cmd/main_test.go index 25037c5f..2dfe0d82 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -76,23 +76,23 @@ func TestPreRun(t *testing.T) { engineConfigVar = tt.engineConfigVar customRegexRuleVar = tt.customRegexRuleVar validateVar = tt.validateVar - channels.Items = make(chan plugins.ISourceItem) - channels.Errors = make(chan error) - channels.WaitGroup = &sync.WaitGroup{} - secretsChan = make(chan *secrets.Secret) - secretsExtrasChan = make(chan *secrets.Secret) - validationChan = make(chan *secrets.Secret) - cvssScoreWithoutValidationChan = make(chan *secrets.Secret) + Channels.Items = make(chan plugins.ISourceItem) + Channels.Errors = make(chan error) + Channels.WaitGroup = &sync.WaitGroup{} + SecretsChan = make(chan *secrets.Secret) + SecretsExtrasChan = make(chan *secrets.Secret) + ValidationChan = make(chan *secrets.Secret) + CvssScoreWithoutValidationChan = make(chan *secrets.Secret) err := preRun("mock", nil, nil) - close(channels.Items) - close(channels.Errors) - channels.WaitGroup.Wait() + close(Channels.Items) + close(Channels.Errors) + Channels.WaitGroup.Wait() if tt.expectedErr != nil { assert.Error(t, err) assert.EqualError(t, err, tt.expectedErr.Error()) } else { assert.NoError(t, err) - assert.Empty(t, channels.Errors) + assert.Empty(t, Channels.Errors) } }) } diff --git a/cmd/workers.go b/cmd/workers.go index 497e8c5d..958f8327 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -1,60 +1,58 @@ package cmd import ( - "github.com/checkmarx/2ms/lib/secrets" - "sync" - "github.com/checkmarx/2ms/engine" "github.com/checkmarx/2ms/engine/extra" + "github.com/checkmarx/2ms/lib/secrets" + "sync" ) -func processItems(engine *engine.Engine, pluginName string) { - defer channels.WaitGroup.Done() - +func ProcessItems(engine *engine.Engine, pluginName string) { + defer Channels.WaitGroup.Done() wgItems := &sync.WaitGroup{} - for item := range channels.Items { - report.TotalItemsScanned++ + for item := range Channels.Items { + Report.TotalItemsScanned++ wgItems.Add(1) - go engine.Detect(item, secretsChan, wgItems, pluginName, channels.Errors) + go engine.Detect(item, SecretsChan, wgItems, pluginName, Channels.Errors) } wgItems.Wait() - close(secretsChan) + close(SecretsChan) } -func processSecrets() { - defer channels.WaitGroup.Done() +func ProcessSecrets() { + defer Channels.WaitGroup.Done() - for secret := range secretsChan { - report.TotalSecretsFound++ - secretsExtrasChan <- secret + for secret := range SecretsChan { + Report.TotalSecretsFound++ + SecretsExtrasChan <- secret if validateVar { - validationChan <- secret + ValidationChan <- secret } else { - cvssScoreWithoutValidationChan <- secret + CvssScoreWithoutValidationChan <- secret } - report.Results[secret.ID] = append(report.Results[secret.ID], secret) + Report.Results[secret.ID] = append(Report.Results[secret.ID], secret) } - close(secretsExtrasChan) - close(validationChan) - close(cvssScoreWithoutValidationChan) + close(SecretsExtrasChan) + close(ValidationChan) + close(CvssScoreWithoutValidationChan) } -func processSecretsExtras() { - defer channels.WaitGroup.Done() +func ProcessSecretsExtras() { + defer Channels.WaitGroup.Done() wgExtras := &sync.WaitGroup{} - for secret := range secretsExtrasChan { + for secret := range SecretsExtrasChan { wgExtras.Add(1) go extra.AddExtraToSecret(secret, wgExtras) } wgExtras.Wait() } -func processValidationAndScoreWithValidation(engine *engine.Engine) { - defer channels.WaitGroup.Done() +func ProcessValidationAndScoreWithValidation(engine *engine.Engine) { + defer Channels.WaitGroup.Done() wgValidation := &sync.WaitGroup{} - for secret := range validationChan { + for secret := range ValidationChan { wgValidation.Add(2) go func(secret *secrets.Secret, wg *sync.WaitGroup) { engine.RegisterForValidation(secret, wg) @@ -66,11 +64,11 @@ func processValidationAndScoreWithValidation(engine *engine.Engine) { engine.Validate() } -func processScoreWithoutValidation(engine *engine.Engine) { - defer channels.WaitGroup.Done() +func ProcessScoreWithoutValidation(engine *engine.Engine) { + defer Channels.WaitGroup.Done() wgScore := &sync.WaitGroup{} - for secret := range cvssScoreWithoutValidationChan { + for secret := range CvssScoreWithoutValidationChan { wgScore.Add(1) go engine.Score(secret, false, wgScore) } diff --git a/cmd/workers_test.go b/cmd/workers_test.go index 4d571542..8152ae20 100644 --- a/cmd/workers_test.go +++ b/cmd/workers_test.go @@ -35,22 +35,22 @@ func TestProcessItems(t *testing.T) { engineConfig := engine.EngineConfig{} engineTest, err := engine.Init(engineConfig) assert.NoError(t, err) - report = reporting.Init() - channels.Items = make(chan plugins.ISourceItem) - secretsChan = make(chan *secrets.Secret) - channels.WaitGroup = &sync.WaitGroup{} - channels.WaitGroup.Add(1) - go processItems(engineTest, "mockPlugin") + Report = reporting.Init() + Channels.Items = make(chan plugins.ISourceItem) + SecretsChan = make(chan *secrets.Secret) + Channels.WaitGroup = &sync.WaitGroup{} + Channels.WaitGroup.Add(1) + go ProcessItems(engineTest, "mockPlugin") for i := 0; i < totalItemsToProcess; i++ { mockData := strconv.Itoa(i) - channels.Items <- &mockItem{ + Channels.Items <- &mockItem{ content: &mockData, id: mockData, } } - close(channels.Items) - channels.WaitGroup.Wait() - assert.Equal(t, totalItemsToProcess, report.TotalItemsScanned) + close(Channels.Items) + Channels.WaitGroup.Wait() + assert.Equal(t, totalItemsToProcess, Report.TotalItemsScanned) } func TestProcessSecrets(t *testing.T) { @@ -70,22 +70,22 @@ func TestProcessSecrets(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - report = reporting.Init() - secretsChan = make(chan *secrets.Secret, 3) - secretsExtrasChan = make(chan *secrets.Secret, 3) - validationChan = make(chan *secrets.Secret, 3) - cvssScoreWithoutValidationChan = make(chan *secrets.Secret, 3) - channels.WaitGroup = &sync.WaitGroup{} + Report = reporting.Init() + SecretsChan = make(chan *secrets.Secret, 3) + SecretsExtrasChan = make(chan *secrets.Secret, 3) + ValidationChan = make(chan *secrets.Secret, 3) + CvssScoreWithoutValidationChan = make(chan *secrets.Secret, 3) + Channels.WaitGroup = &sync.WaitGroup{} validateVar = tt.validateVar - secretsChan <- &secrets.Secret{ID: "mockId", StartLine: 1} - secretsChan <- &secrets.Secret{ID: "mockId2"} - secretsChan <- &secrets.Secret{ID: "mockId", StartLine: 2} - close(secretsChan) + SecretsChan <- &secrets.Secret{ID: "mockId", StartLine: 1} + SecretsChan <- &secrets.Secret{ID: "mockId2"} + SecretsChan <- &secrets.Secret{ID: "mockId", StartLine: 2} + close(SecretsChan) - channels.WaitGroup.Add(1) - go processSecrets() + Channels.WaitGroup.Add(1) + go ProcessSecrets() - channels.WaitGroup.Wait() + Channels.WaitGroup.Wait() expectedSecrets := []*secrets.Secret{ {ID: "mockId", StartLine: 1}, @@ -93,7 +93,7 @@ func TestProcessSecrets(t *testing.T) { {ID: "mockId2"}, } var actualSecrets []*secrets.Secret - for val := range secretsExtrasChan { + for val := range SecretsExtrasChan { actualSecrets = append(actualSecrets, val) } sort.Slice(actualSecrets, func(i, j int) bool { @@ -105,9 +105,9 @@ func TestProcessSecrets(t *testing.T) { assert.Equal(t, expectedSecrets, actualSecrets) if validateVar { - assert.Empty(t, cvssScoreWithoutValidationChan) + assert.Empty(t, CvssScoreWithoutValidationChan) var actualSecretsWithValidation []*secrets.Secret - for val := range validationChan { + for val := range ValidationChan { actualSecretsWithValidation = append(actualSecretsWithValidation, val) } sort.Slice(actualSecretsWithValidation, func(i, j int) bool { @@ -118,9 +118,9 @@ func TestProcessSecrets(t *testing.T) { }) assert.Equal(t, expectedSecrets, actualSecretsWithValidation) } else { - assert.Empty(t, validationChan) + assert.Empty(t, ValidationChan) var actualSecretsWithoutValidation []*secrets.Secret - for val := range cvssScoreWithoutValidationChan { + for val := range CvssScoreWithoutValidationChan { actualSecretsWithoutValidation = append(actualSecretsWithoutValidation, val) } sort.Slice(actualSecretsWithoutValidation, func(i, j int) bool { @@ -132,12 +132,12 @@ func TestProcessSecrets(t *testing.T) { assert.Equal(t, expectedSecrets, actualSecretsWithoutValidation) } - assert.Equal(t, 3, report.TotalSecretsFound) - assert.Equal(t, 2, len(report.Results["mockId"])) - assert.Equal(t, 1, len(report.Results["mockId2"])) - assert.Equal(t, &secrets.Secret{ID: "mockId", StartLine: 1}, report.Results["mockId"][0]) - assert.Equal(t, &secrets.Secret{ID: "mockId", StartLine: 2}, report.Results["mockId"][1]) - assert.Equal(t, &secrets.Secret{ID: "mockId2"}, report.Results["mockId2"][0]) + assert.Equal(t, 3, Report.TotalSecretsFound) + assert.Equal(t, 2, len(Report.Results["mockId"])) + assert.Equal(t, 1, len(Report.Results["mockId2"])) + assert.Equal(t, &secrets.Secret{ID: "mockId", StartLine: 1}, Report.Results["mockId"][0]) + assert.Equal(t, &secrets.Secret{ID: "mockId", StartLine: 2}, Report.Results["mockId"][1]) + assert.Equal(t, &secrets.Secret{ID: "mockId2"}, Report.Results["mockId2"][0]) }) } } @@ -191,15 +191,15 @@ func TestProcessSecretsExtras(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - secretsExtrasChan = make(chan *secrets.Secret, len(tt.inputSecrets)) + SecretsExtrasChan = make(chan *secrets.Secret, len(tt.inputSecrets)) for _, secret := range tt.inputSecrets { - secretsExtrasChan <- secret + SecretsExtrasChan <- secret } - close(secretsExtrasChan) + close(SecretsExtrasChan) - channels.WaitGroup.Add(1) - go processSecretsExtras() - channels.WaitGroup.Wait() + Channels.WaitGroup.Add(1) + go ProcessSecretsExtras() + Channels.WaitGroup.Wait() for i, expected := range tt.expectedSecrets { assert.Equal(t, expected, tt.inputSecrets[i]) @@ -252,15 +252,15 @@ func TestProcessValidationAndScoreWithValidation(t *testing.T) { engineConfig := engine.EngineConfig{} engineTest, err := engine.Init(engineConfig) assert.NoError(t, err) - validationChan = make(chan *secrets.Secret, len(tt.inputSecrets)) + ValidationChan = make(chan *secrets.Secret, len(tt.inputSecrets)) for _, secret := range tt.inputSecrets { - validationChan <- secret + ValidationChan <- secret } - close(validationChan) + close(ValidationChan) - channels.WaitGroup.Add(1) - go processValidationAndScoreWithValidation(engineTest) - channels.WaitGroup.Wait() + Channels.WaitGroup.Add(1) + go ProcessValidationAndScoreWithValidation(engineTest) + Channels.WaitGroup.Wait() for i, expected := range tt.expectedSecrets { assert.Equal(t, expected, tt.inputSecrets[i]) @@ -313,15 +313,15 @@ func TestProcessScoreWithoutValidation(t *testing.T) { engineConfig := engine.EngineConfig{} engineTest, err := engine.Init(engineConfig) assert.NoError(t, err) - cvssScoreWithoutValidationChan = make(chan *secrets.Secret, len(tt.inputSecrets)) + CvssScoreWithoutValidationChan = make(chan *secrets.Secret, len(tt.inputSecrets)) for _, secret := range tt.inputSecrets { - cvssScoreWithoutValidationChan <- secret + CvssScoreWithoutValidationChan <- secret } - close(cvssScoreWithoutValidationChan) + close(CvssScoreWithoutValidationChan) - channels.WaitGroup.Add(1) - go processScoreWithoutValidation(engineTest) - channels.WaitGroup.Wait() + Channels.WaitGroup.Add(1) + go ProcessScoreWithoutValidation(engineTest) + Channels.WaitGroup.Wait() for i, expected := range tt.expectedSecrets { assert.Equal(t, expected, tt.inputSecrets[i]) diff --git a/engine/engine.go b/engine/engine.go index 03a69353..a0581b57 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -94,11 +94,11 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S } else { startLine = value.StartLine endLine = value.EndLine - } lineContent, err := linecontent.GetLineContent(value.Line, value.Secret) if err != nil { - errors <- err + errors <- fmt.Errorf("failed to get line content for source %s: %w", item.GetSource(), err) + return } secret := &secrets.Secret{ ID: itemId, diff --git a/engine/linecontent/linecontent.go b/engine/linecontent/linecontent.go index 65620212..a88119d0 100644 --- a/engine/linecontent/linecontent.go +++ b/engine/linecontent/linecontent.go @@ -14,11 +14,11 @@ const ( func GetLineContent(line, secret string) (string, error) { lineSize := len(line) if lineSize == 0 { - return "", fmt.Errorf("failed to get line content: line empty") + return "", fmt.Errorf("line empty") } if len(secret) == 0 { - return "", fmt.Errorf("failed to get line content: secret empty") + return "", fmt.Errorf("secret empty") } // Truncate lineContent to max size diff --git a/engine/linecontent/linecontent_test.go b/engine/linecontent/linecontent_test.go index e9db7791..09f1fba9 100644 --- a/engine/linecontent/linecontent_test.go +++ b/engine/linecontent/linecontent_test.go @@ -24,7 +24,7 @@ func TestGetLineContent(t *testing.T) { secret: dummySecret, expected: "", error: true, - errorMessage: "failed to get line content: line empty", + errorMessage: "line empty", }, { name: "Empty secret", @@ -32,7 +32,7 @@ func TestGetLineContent(t *testing.T) { secret: "", expected: "", error: true, - errorMessage: "failed to get line content: secret empty", + errorMessage: "secret empty", }, { name: "Secret not found with line size smaller than the parse limit", diff --git a/go.mod b/go.mod index 003a49b1..4faa1b66 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.6 require ( github.com/bwmarrin/discordgo v0.27.1 github.com/gitleaks/go-gitdiff v0.9.0 + github.com/google/go-cmp v0.6.0 github.com/rs/zerolog v1.32.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.8.0 diff --git a/lib/reporting/report.go b/lib/reporting/report.go index ee127d53..2739ab09 100644 --- a/lib/reporting/report.go +++ b/lib/reporting/report.go @@ -30,7 +30,7 @@ func Init() *Report { } func (r *Report) ShowReport(format string, cfg *config.Config) error { - output, err := r.getOutput(format, cfg) + output, err := r.GetOutput(format, cfg) if err != nil { return err } @@ -53,7 +53,7 @@ func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { fileExtension := filepath.Ext(path) format := strings.TrimPrefix(fileExtension, ".") - output, err := r.getOutput(format, cfg) + output, err := r.GetOutput(format, cfg) if err != nil { return err } @@ -66,7 +66,7 @@ func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { return nil } -func (r *Report) getOutput(format string, cfg *config.Config) (string, error) { +func (r *Report) GetOutput(format string, cfg *config.Config) (string, error) { var output string var err error diff --git a/lib/reporting/report_test.go b/lib/reporting/report_test.go index 0eef1dc1..84d79047 100644 --- a/lib/reporting/report_test.go +++ b/lib/reporting/report_test.go @@ -293,7 +293,7 @@ func TestGetOutputSarif(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.arg.getOutput(sarifFormat, &config.Config{Name: "report", Version: "1"}) + got, err := tt.arg.GetOutput(sarifFormat, &config.Config{Name: "report", Version: "1"}) if tt.wantErr { assert.NotNil(t, err) return diff --git a/pkg/scan.go b/pkg/scan.go new file mode 100644 index 00000000..9300382e --- /dev/null +++ b/pkg/scan.go @@ -0,0 +1,143 @@ +package scanner + +import ( + "errors" + "fmt" + "github.com/checkmarx/2ms/lib/reporting" + "sync" + + "github.com/checkmarx/2ms/cmd" + "github.com/checkmarx/2ms/engine" +) + +type ScanConfig struct { + IgnoreResultIds []string +} + +type scanner struct{} + +func NewScanner() Scanner { + return &scanner{} +} + +func (s *scanner) Scan(scanItems []ScanItem, scanConfig ScanConfig) (*reporting.Report, error) { + itemsCh := cmd.Channels.Items + errorsCh := cmd.Channels.Errors + wg := &sync.WaitGroup{} + + // listener for errors + bufferedErrors := make(chan error, len(scanItems)+1) + go func() { + for err := range errorsCh { + if err != nil { + bufferedErrors <- err + } + } + close(bufferedErrors) + }() + + // Initialize engine configuration + engineConfig := engine.EngineConfig{IgnoredIds: scanConfig.IgnoreResultIds} + engineInstance, err := engine.Init(engineConfig) + if err != nil { + return &reporting.Report{}, fmt.Errorf("error initializing engine: %w", err) + } + + // Start processing items + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessItems(engineInstance, "custom") + + // Start processing secrets + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessSecrets() + + // Start processing secrets extras + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessSecretsExtras() + + // Start validation and scoring + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessScoreWithoutValidation(engineInstance) + + // send items to be scanned + for _, scanItem := range scanItems { + wg.Add(1) + go func(item ScanItem) { + defer wg.Done() + itemsCh <- item + }(scanItem) + } + wg.Wait() + close(itemsCh) + cmd.Channels.WaitGroup.Wait() + + close(errorsCh) + var errs []error + for err = range bufferedErrors { + if err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return &reporting.Report{}, fmt.Errorf("error(s) processing scan items:\n%w", errors.Join(errs...)) + } + + // Finalize and generate report + report := cmd.Report + return report, nil +} + +func (s *scanner) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (*reporting.Report, error) { + itemsCh := cmd.Channels.Items + errorsCh := cmd.Channels.Errors + wg := &sync.WaitGroup{} + + // Initialize engine configuration. + engineConfig := engine.EngineConfig{IgnoredIds: scanConfig.IgnoreResultIds} + engineInstance, err := engine.Init(engineConfig) + if err != nil { + return &reporting.Report{}, fmt.Errorf("error initializing engine: %w", err) + } + + // Start processing routines. + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessItems(engineInstance, "custom") + + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessSecrets() + + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessSecretsExtras() + + cmd.Channels.WaitGroup.Add(1) + go cmd.ProcessScoreWithoutValidation(engineInstance) + + // Forward scan items from itemsIn to itemsCh. + go func() { + for item := range itemsIn { + wg.Add(1) + go func(i ScanItem) { + defer wg.Done() + itemsCh <- i + }(item) + } + wg.Wait() + close(itemsCh) + }() + + // Wait for all processing routines. + cmd.Channels.WaitGroup.Wait() + close(errorsCh) + + // Check if any error occurred. + for err := range errorsCh { + if err != nil { + return &reporting.Report{}, fmt.Errorf("error processing scan items: %w", err) + } + } + + // Finalize and generate report. + report := cmd.Report + return report, nil +} diff --git a/pkg/scan_test.go b/pkg/scan_test.go new file mode 100644 index 00000000..0d3581a7 --- /dev/null +++ b/pkg/scan_test.go @@ -0,0 +1,441 @@ +package scanner + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/checkmarx/2ms/cmd" + "github.com/checkmarx/2ms/lib/reporting" + "github.com/checkmarx/2ms/lib/secrets" + "github.com/checkmarx/2ms/plugins" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" +) + +const ( + githubPatPath = "testData/secrets/github-pat.txt" + jwtPath = "testData/secrets/jwt.txt" + expectedReportPath = "testData/expectedReport.json" + expectedReportResultsIgnoredPath = "testData/expectedReportWithIgnoredResults.json" +) + +// normalizeReportData recursively traverses the report data and removes any carriage return characters. +func normalizeReportData(data interface{}) interface{} { + switch v := data.(type) { + case string: + return strings.ReplaceAll(v, "\r", "") + case []interface{}: + for i, item := range v { + v[i] = normalizeReportData(item) + } + return v + case map[string]interface{}: + for key, val := range v { + v[key] = normalizeReportData(val) + } + return v + default: + return data + } +} + +func TestScan(t *testing.T) { + t.Run("Successful Scan with Multiple Items", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + + githubPatBytes, err := os.ReadFile(githubPatPath) + assert.NoError(t, err, "failed to read github-pat file") + githubPatContent := string(githubPatBytes) + + jwtBytes, err := os.ReadFile(jwtPath) + assert.NoError(t, err, "failed to read jwt file") + jwtContent := string(jwtBytes) + + emptyContent := "" + emptyMockPath := "mockPath" + + scanItems := []ScanItem{ + { + Content: &githubPatContent, + ID: fmt.Sprintf("mock-%s", githubPatPath), + Source: githubPatPath, + }, + { + Content: &emptyContent, + ID: fmt.Sprintf("mock-%s", emptyMockPath), + Source: emptyMockPath, + }, + { + Content: &jwtContent, + ID: fmt.Sprintf("mock-%s", jwtPath), + Source: jwtPath, + }, + } + + testScanner := NewScanner() + actualReport, err := testScanner.Scan(scanItems, ScanConfig{}) + assert.NoError(t, err, "scanner encountered an error") + + expectedReportBytes, err := os.ReadFile(expectedReportPath) + assert.NoError(t, err, "failed to read expected report file") + + var expectedReport, actualReportMap map[string]interface{} + + err = json.Unmarshal(expectedReportBytes, &expectedReport) + assert.NoError(t, err, "failed to unmarshal expected report JSON") + + // Marshal actual report and unmarshal back into a map. + actualReportBytes, err := json.Marshal(actualReport) + assert.NoError(t, err, "failed to marshal actual report to JSON") + err = json.Unmarshal(actualReportBytes, &actualReportMap) + assert.NoError(t, err, "failed to unmarshal actual report JSON") + + // Normalize both expected and actual maps. + expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) + actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + + if !cmp.Equal(expectedReport, actualReportMap) { + t.Errorf("Scan report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) + } + }) + t.Run("Successful scan with multiple items and ignored results", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + + githubPatBytes, err := os.ReadFile(githubPatPath) + assert.NoError(t, err, "failed to read github-pat file") + githubPatContent := string(githubPatBytes) + + jwtBytes, err := os.ReadFile(jwtPath) + assert.NoError(t, err, "failed to read jwt file") + jwtContent := string(jwtBytes) + + emptyContent := "" + emptyMockPath := "mockPath" + + scanItems := []ScanItem{ + { + Content: &githubPatContent, + ID: fmt.Sprintf("mock-%s", githubPatPath), + Source: githubPatPath, + }, + { + Content: &emptyContent, + ID: fmt.Sprintf("mock-%s", emptyMockPath), + Source: emptyMockPath, + }, + { + Content: &jwtContent, + ID: fmt.Sprintf("mock-%s", jwtPath), + Source: jwtPath, + }, + } + + testScanner := NewScanner() + actualReport, err := testScanner.Scan(scanItems, ScanConfig{ + IgnoreResultIds: []string{ + "a0cd293e6e122a1c7384d5a56781e39ba350c54b", + "40483a2b07fa3beaf234d1a0b5d0931d7b7ae9f7", + }, + }) + assert.NoError(t, err, "scanner encountered an error") + + expectedReportBytes, err := os.ReadFile(expectedReportResultsIgnoredPath) + assert.NoError(t, err, "failed to read expected report file") + + var expectedReport, actualReportMap map[string]interface{} + + err = json.Unmarshal(expectedReportBytes, &expectedReport) + assert.NoError(t, err, "failed to unmarshal expected report JSON") + + actualReportBytes, err := json.Marshal(actualReport) + assert.NoError(t, err, "failed to marshal actual report to JSON") + err = json.Unmarshal(actualReportBytes, &actualReportMap) + assert.NoError(t, err, "failed to unmarshal actual report JSON") + + // Normalize both expected and actual maps. + expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) + actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + + if !cmp.Equal(expectedReport, actualReportMap) { + t.Errorf("Scan report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) + } + }) + t.Run("error handling should work", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + + emptyContent := "" + scanItems := []ScanItem{ + { + Content: &emptyContent, + ID: "", + Source: "", + }, + { + Content: &emptyContent, + ID: "", + Source: "", + }, + } + + testScanner := NewScanner() + cmd.Channels.Errors = make(chan error, 2) + + go func() { + cmd.Channels.Errors <- fmt.Errorf("mock processing error 1") + cmd.Channels.Errors <- fmt.Errorf("mock processing error 2") + }() + report, err := testScanner.Scan(scanItems, ScanConfig{}) + + assert.Equal(t, &reporting.Report{}, report) + assert.NotNil(t, err) + assert.Equal(t, "error(s) processing scan items:\nmock processing error 1\nmock processing error 2", err.Error()) + }) + t.Run("scan with scanItems empty", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + + testScanner := NewScanner() + actualReport, err := testScanner.Scan([]ScanItem{}, ScanConfig{}) + assert.NoError(t, err, "scanner encountered an error") + assert.Equal(t, &reporting.Report{Results: map[string][]*secrets.Secret{}}, actualReport) + }) + t.Run("scan with scanItems nil", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + + testScanner := NewScanner() + actualReport, err := testScanner.Scan(nil, ScanConfig{}) + assert.NoError(t, err, "scanner encountered an error") + assert.Equal(t, &reporting.Report{Results: map[string][]*secrets.Secret{}}, actualReport) + }) +} + +func TestScanDynamic(t *testing.T) { + t.Run("Successful ScanDynamic with Multiple Items", func(t *testing.T) { + // Reset global channels and report. + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + cmd.Channels.WaitGroup = &sync.WaitGroup{} + + // Read file contents. + githubPatBytes, err := os.ReadFile(githubPatPath) + assert.NoError(t, err, "failed to read github-pat file") + githubPatContent := string(githubPatBytes) + + jwtBytes, err := os.ReadFile(jwtPath) + assert.NoError(t, err, "failed to read jwt file") + jwtContent := string(jwtBytes) + + emptyContent := "" + emptyMockPath := "mockPath" + + scanItems := []ScanItem{ + { + Content: &githubPatContent, + ID: fmt.Sprintf("mock-%s", githubPatPath), + Source: githubPatPath, + }, + { + Content: &emptyContent, + ID: fmt.Sprintf("mock-%s", emptyMockPath), + Source: emptyMockPath, + }, + { + Content: &jwtContent, + ID: fmt.Sprintf("mock-%s", jwtPath), + Source: jwtPath, + }, + } + + // Create an input channel and feed it the scan items. + itemsIn := make(chan ScanItem, len(scanItems)) + for _, item := range scanItems { + itemsIn <- item + } + close(itemsIn) + + testScanner := NewScanner() + actualReport, err := testScanner.ScanDynamic(itemsIn, ScanConfig{}) + assert.NoError(t, err, "scanner encountered an error") + + expectedReportBytes, err := os.ReadFile(expectedReportPath) + assert.NoError(t, err, "failed to read expected report file") + + var expectedReport, actualReportMap map[string]interface{} + + err = json.Unmarshal(expectedReportBytes, &expectedReport) + assert.NoError(t, err, "failed to unmarshal expected report JSON") + + actualReportBytes, err := json.Marshal(actualReport) + assert.NoError(t, err, "failed to marshal actual report to JSON") + err = json.Unmarshal(actualReportBytes, &actualReportMap) + assert.NoError(t, err, "failed to unmarshal actual report JSON") + + // Normalize both maps. + expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) + actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + + if !cmp.Equal(expectedReport, actualReportMap) { + t.Errorf("ScanDynamic report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) + } + }) + + t.Run("Successful ScanDynamic with Multiple Items and Ignored Results", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + cmd.Channels.WaitGroup = &sync.WaitGroup{} + + githubPatBytes, err := os.ReadFile(githubPatPath) + assert.NoError(t, err, "failed to read github-pat file") + githubPatContent := string(githubPatBytes) + + jwtBytes, err := os.ReadFile(jwtPath) + assert.NoError(t, err, "failed to read jwt file") + jwtContent := string(jwtBytes) + + emptyContent := "" + emptyMockPath := "mockPath" + + scanItems := []ScanItem{ + { + Content: &githubPatContent, + ID: fmt.Sprintf("mock-%s", githubPatPath), + Source: githubPatPath, + }, + { + Content: &emptyContent, + ID: fmt.Sprintf("mock-%s", emptyMockPath), + Source: emptyMockPath, + }, + { + Content: &jwtContent, + ID: fmt.Sprintf("mock-%s", jwtPath), + Source: jwtPath, + }, + } + + itemsIn := make(chan ScanItem, len(scanItems)) + for _, item := range scanItems { + itemsIn <- item + } + close(itemsIn) + + testScanner := NewScanner() + actualReport, err := testScanner.ScanDynamic(itemsIn, ScanConfig{ + IgnoreResultIds: []string{ + "a0cd293e6e122a1c7384d5a56781e39ba350c54b", + "40483a2b07fa3beaf234d1a0b5d0931d7b7ae9f7", + }, + }) + assert.NoError(t, err, "scanner encountered an error") + + expectedReportBytes, err := os.ReadFile(expectedReportResultsIgnoredPath) + assert.NoError(t, err, "failed to read expected report file") + + var expectedReport, actualReportMap map[string]interface{} + + err = json.Unmarshal(expectedReportBytes, &expectedReport) + assert.NoError(t, err, "failed to unmarshal expected report JSON") + + actualReportBytes, err := json.Marshal(actualReport) + assert.NoError(t, err, "failed to marshal actual report to JSON") + err = json.Unmarshal(actualReportBytes, &actualReportMap) + assert.NoError(t, err, "failed to unmarshal actual report JSON") + + // Normalize both maps. + expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) + actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + + if !cmp.Equal(expectedReport, actualReportMap) { + t.Errorf("ScanDynamic report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) + } + }) + + t.Run("error handling should work", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error, 2) + cmd.Channels.WaitGroup = &sync.WaitGroup{} + + itemsIn := make(chan ScanItem) + close(itemsIn) + + go func() { + cmd.Channels.Errors <- fmt.Errorf("mock processing error 1") + cmd.Channels.Errors <- fmt.Errorf("mock processing error 2") + }() + + testScanner := NewScanner() + report, err := testScanner.ScanDynamic(itemsIn, ScanConfig{}) + + assert.Equal(t, &reporting.Report{}, report) + assert.NotNil(t, err) + expectedErrMsg := "error processing scan items: mock processing error 1" + assert.Equal(t, expectedErrMsg, err.Error()) + }) + + t.Run("scan with empty channel", func(t *testing.T) { + cmd.Report = reporting.Init() + cmd.SecretsChan = make(chan *secrets.Secret) + cmd.SecretsExtrasChan = make(chan *secrets.Secret) + cmd.ValidationChan = make(chan *secrets.Secret) + cmd.CvssScoreWithoutValidationChan = make(chan *secrets.Secret) + cmd.Channels.Items = make(chan plugins.ISourceItem) + cmd.Channels.Errors = make(chan error) + cmd.Channels.WaitGroup = &sync.WaitGroup{} + + itemsIn := make(chan ScanItem) + close(itemsIn) + + testScanner := NewScanner() + actualReport, err := testScanner.ScanDynamic(itemsIn, ScanConfig{}) + assert.NoError(t, err, "scanner encountered an error") + assert.Equal(t, &reporting.Report{Results: map[string][]*secrets.Secret{}}, actualReport) + }) +} diff --git a/pkg/scanner.go b/pkg/scanner.go new file mode 100644 index 00000000..43452fa8 --- /dev/null +++ b/pkg/scanner.go @@ -0,0 +1,33 @@ +package scanner + +import ( + "github.com/checkmarx/2ms/lib/reporting" + "github.com/checkmarx/2ms/plugins" +) + +type ScanItem struct { + Content *string + // Unique identifier of the item + ID string + // User-friendly description and/or link to the item + Source string +} + +var _ plugins.ISourceItem = (*ScanItem)(nil) + +func (i ScanItem) GetContent() *string { + return i.Content +} + +func (i ScanItem) GetID() string { + return i.ID +} + +func (i ScanItem) GetSource() string { + return i.Source +} + +type Scanner interface { + Scan(scanItems []ScanItem, scanConfig ScanConfig) (*reporting.Report, error) + ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (*reporting.Report, error) +} diff --git a/pkg/testData/expectedReport.json b/pkg/testData/expectedReport.json new file mode 100644 index 00000000..092ead53 --- /dev/null +++ b/pkg/testData/expectedReport.json @@ -0,0 +1,101 @@ +{ + "totalItemsScanned" : 3, + "totalSecretsFound" : 6, + "results" : { + "40483a2b07fa3beaf234d1a0b5d0931d7b7ae9f7" : [ { + "id" : "40483a2b07fa3beaf234d1a0b5d0931d7b7ae9f7", + "source" : "testData/secrets/github-pat.txt", + "ruleId" : "github-pat", + "startLine" : 1, + "endLine" : 1, + "lineContent" : "\n Text_Example = ghp_CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC\r", + "startColumn" : 64, + "endColumn" : 103, + "value" : "ghp_CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", + "ruleDescription" : "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "cvssScore" : 8.2 + } ], + "a0cd293e6e122a1c7384d5a56781e39ba350c54b" : [ { + "id" : "a0cd293e6e122a1c7384d5a56781e39ba350c54b", + "source" : "testData/secrets/jwt.txt", + "ruleId" : "jwt", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r", + "startColumn" : 129, + "endColumn" : 232, + "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", + "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "extraDetails" : { + "secretDetails" : { + "name" : "mockName2", + "sub" : "mockSub2" + } + }, + "cvssScore" : 8.2 + }, { + "id" : "a0cd293e6e122a1c7384d5a56781e39ba350c54b", + "source" : "testData/secrets/jwt.txt", + "ruleId" : "jwt", + "startLine" : 1, + "endLine" : 1, + "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r\n Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", + "startColumn" : 64, + "endColumn" : 166, + "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", + "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "extraDetails" : { + "secretDetails" : { + "name" : "mockName2", + "sub" : "mockSub2" + } + }, + "cvssScore" : 8.2 + } ], + "6949272451f77dc4a38d5f35d583cf56023cd2c1" : [ { + "id" : "6949272451f77dc4a38d5f35d583cf56023cd2c1", + "source" : "testData/secrets/github-pat.txt", + "ruleId" : "github-pat", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExampleghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATextExampleghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBTextExample\r", + "startColumn" : 12, + "endColumn" : 51, + "value" : "ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "ruleDescription" : "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "cvssScore" : 8.2 + } ], + "f29abe9eacc233a8e5e9c7762bca48589d9c76a2" : [ { + "id" : "f29abe9eacc233a8e5e9c7762bca48589d9c76a2", + "source" : "testData/secrets/jwt.txt", + "ruleId" : "jwt", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r", + "startColumn" : 13, + "endColumn" : 116, + "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1", + "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "extraDetails" : { + "secretDetails" : { + "name" : "mockName1", + "sub" : "mockSub1" + } + }, + "cvssScore" : 8.2 + } ], + "fc17c755f40062dcb3f16eb6299f9afc7eccbc56" : [ { + "id" : "fc17c755f40062dcb3f16eb6299f9afc7eccbc56", + "source" : "testData/secrets/github-pat.txt", + "ruleId" : "github-pat", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExampleghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATextExampleghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBTextExample\r", + "startColumn" : 63, + "endColumn" : 102, + "value" : "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + "ruleDescription" : "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "cvssScore" : 8.2 + } ] + } +} \ No newline at end of file diff --git a/pkg/testData/expectedReportWithIgnoredResults.json b/pkg/testData/expectedReportWithIgnoredResults.json new file mode 100644 index 00000000..3d99d9dc --- /dev/null +++ b/pkg/testData/expectedReportWithIgnoredResults.json @@ -0,0 +1,51 @@ +{ + "totalItemsScanned" : 3, + "totalSecretsFound" : 3, + "results" : { + "6949272451f77dc4a38d5f35d583cf56023cd2c1" : [ { + "id" : "6949272451f77dc4a38d5f35d583cf56023cd2c1", + "source" : "testData/secrets/github-pat.txt", + "ruleId" : "github-pat", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExampleghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATextExampleghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBTextExample\r", + "startColumn" : 12, + "endColumn" : 51, + "value" : "ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "ruleDescription" : "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "cvssScore" : 8.2 + } ], + "f29abe9eacc233a8e5e9c7762bca48589d9c76a2" : [ { + "id" : "f29abe9eacc233a8e5e9c7762bca48589d9c76a2", + "source" : "testData/secrets/jwt.txt", + "ruleId" : "jwt", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r", + "startColumn" : 13, + "endColumn" : 116, + "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1", + "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", + "extraDetails" : { + "secretDetails" : { + "name" : "mockName1", + "sub" : "mockSub1" + } + }, + "cvssScore" : 8.2 + } ], + "fc17c755f40062dcb3f16eb6299f9afc7eccbc56" : [ { + "id" : "fc17c755f40062dcb3f16eb6299f9afc7eccbc56", + "source" : "testData/secrets/github-pat.txt", + "ruleId" : "github-pat", + "startLine" : 0, + "endLine" : 0, + "lineContent" : "TextExampleghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATextExampleghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBTextExample\r", + "startColumn" : 63, + "endColumn" : 102, + "value" : "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + "ruleDescription" : "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + "cvssScore" : 8.2 + } ] + } +} \ No newline at end of file diff --git a/pkg/testData/secrets/github-pat.txt b/pkg/testData/secrets/github-pat.txt new file mode 100644 index 00000000..6868ba74 --- /dev/null +++ b/pkg/testData/secrets/github-pat.txt @@ -0,0 +1,2 @@ +TextExampleghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATextExampleghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBTextExample + Text_Example = ghp_CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC diff --git a/pkg/testData/secrets/jwt.txt b/pkg/testData/secrets/jwt.txt new file mode 100644 index 00000000..e8b13399 --- /dev/null +++ b/pkg/testData/secrets/jwt.txt @@ -0,0 +1,2 @@ +TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample + Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 \ No newline at end of file diff --git a/plugins/filesystem.go b/plugins/filesystem.go index fc4138ec..91d1e539 100644 --- a/plugins/filesystem.go +++ b/plugins/filesystem.go @@ -39,7 +39,7 @@ func (p *FileSystemPlugin) DefineCommand(items chan ISourceItem, errors chan err log.Info().Msg("Folder plugin started") wg := &sync.WaitGroup{} - p.getFiles(items, errors, wg) + p.GetFiles(items, errors, wg) wg.Wait() close(items) }, @@ -60,7 +60,7 @@ func (p *FileSystemPlugin) DefineCommand(items chan ISourceItem, errors chan err return cmd, nil } -func (p *FileSystemPlugin) getFiles(items chan ISourceItem, errs chan error, wg *sync.WaitGroup) { +func (p *FileSystemPlugin) GetFiles(items chan ISourceItem, errs chan error, wg *sync.WaitGroup) { fileList := make([]string, 0) err := filepath.Walk(p.Path, func(path string, fInfo os.FileInfo, err error) error { if err != nil { @@ -98,10 +98,10 @@ func (p *FileSystemPlugin) getFiles(items chan ISourceItem, errs chan error, wg return } - p.getItems(items, errs, wg, fileList) + p.GetItems(items, errs, wg, fileList) } -func (p *FileSystemPlugin) getItems(items chan ISourceItem, errs chan error, wg *sync.WaitGroup, fileList []string) { +func (p *FileSystemPlugin) GetItems(items chan ISourceItem, errs chan error, wg *sync.WaitGroup, fileList []string) { for _, filePath := range fileList { wg.Add(1) go func(filePath string) { diff --git a/plugins/filesystem_test.go b/plugins/filesystem_test.go index 2f631113..5c374431 100644 --- a/plugins/filesystem_test.go +++ b/plugins/filesystem_test.go @@ -64,7 +64,7 @@ func TestGetItems(t *testing.T) { ProjectName: "TestProject", } - plugin.getItems(itemsChan, errsChan, &wg, fileList) + plugin.GetItems(itemsChan, errsChan, &wg, fileList) wg.Wait() @@ -188,7 +188,7 @@ func TestGetFiles(t *testing.T) { errsChan := make(chan error, 10) var wg sync.WaitGroup - plugin.getFiles(itemsChan, errsChan, &wg) + plugin.GetFiles(itemsChan, errsChan, &wg) wg.Wait() close(itemsChan) close(errsChan)