fix: improve memory perfomance in large repos

.2ms.yml

@@ -46,6 +46,8 @@ ignore-result:
   - d9207d5fa344d2423e97384f45014c87c0c91d4f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
   - f858b057df65752e0030854331dd1e5e8e41856b # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
   - 670491bf5e759f4c03bf0e47f519deaccdc9ac44 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/files#diff-918450788fde1467e3fe71000c32f812131f7dcd7dafcb0310c8c9910ffea848 
-  - 46d644a014e91528b0f6d4180305d35452ed7f46 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
-  - b841441ea9d80db4ad9f21abe25cf30836a33f1d # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
-  - f1c519e1be61df80729d099a21235b64b525b280 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
+  - 1bf24590387167e7ade218952eaf168758acc0d6 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/files#diff-918450788fde1467e3fe71000c32f812131f7dcd7dafcb0310c8c9910ffea848 
+  - 30ea5ee224b162075bf512dbf5854002b6c5e727 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
+  - 51a6f4e3c7e3a79c9722abb7541b4902098e526b # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
+  - 53803ee7e880952e926898a434acff4483fec67e # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
+  - aa52405f239a8be1284d933025c557b071b24036 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4

Dockerfile

@@ -3,7 +3,7 @@
 # and "Missing User Instruction" since 2ms container is stopped after scan
 
 # Builder image
-FROM cgr.dev/chainguard/go@sha256:411f37ae52643cf040cfaca740aa78951009f3e7e399eef2ec797c153fe4c892 AS builder
+FROM cgr.dev/chainguard/go@sha256:7f9e74e1af376a6d238077d8df037a25001997581630bc121c8aecfa5c8da8b3 AS builder
 
 WORKDIR /app
 
@@ -20,7 +20,7 @@ COPY . .
 RUN GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o /app/2ms .
 
 # Runtime image
-FROM cgr.dev/chainguard/git@sha256:c893f65bcc5d3de1c327af6db17566139af7663ef89001d536e8370226dcf881
+FROM cgr.dev/chainguard/git@sha256:b0dbd0c3c6a0f44c0522663c3a7f9b47f8e62ed419c88c37199f61308f19829c
 
 WORKDIR /app
 
@@ -32,4 +32,4 @@ COPY --from=builder /app/2ms /app/2ms
 
 RUN git config --global --add safe.directory /repo
 
-ENTRYPOINT [ "/app/2ms" ]
+ENTRYPOINT [ "/app/2ms" ]

README.md

@@ -180,7 +180,7 @@ Flags:
                                       example: if 'results' is set, only engine errors will make 2ms exit code different from 0 (default none)
       --ignore-result strings         ignore specific result by id
       --ignore-rule strings           ignore rules by name or tag
-      --log-level string              log level (trace, debug, info, warn, error, fatal) (default "info")
+      --log-level string              log level (trace, debug, info, warn, error, fatal, none) (default "info")
       --max-target-megabytes int      files larger than this will be skipped.
                                       Omit or set to 0 to disable this check.
       --regex stringArray             custom regexes to apply to the scan, must be valid Go regex
@@ -376,7 +376,7 @@ The following table describes the global flags that can be used together with an
 |--ignore-on-exit |  | None | Defines which kind of non-zero exits code should be ignored. Options are: all, results, errors, none. For example, if 'results' is set, only engine errors will make 2ms exit code different from 0. |
 |--ignore-result | strings |  | Ignore specific result by ID |
 |--ignore-rule | strings |  | Ignore rules by name or tag. |
-|--log-level | string | info | Type of log to return. Options are: trace, debug, info, warn, error, fatal |
+|--log-level | string | info | Type of log to return. Options are: trace, debug, info, warn, error, fatal, none |
 |--max-target-megabytes | int |  | Files larger than than the specified threshold will be skipped. Omit or set to 0 to disable this check. |
 |--regex | stringArray |  | Custom regexes to apply to the scan. Must be valid Go regex. |
 |--report-path | strings |  | Path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif) |

cmd/config.go

@@ -13,6 +13,7 @@ import (
 )
 
 func initialize() {
+
 	configFilePath, err := rootCmd.Flags().GetString(configFileFlag)
 	if err != nil {
 		cobra.CheckErr(err)
@@ -22,6 +23,8 @@ func initialize() {
 
 	logLevel := zerolog.InfoLevel
 	switch strings.ToLower(logLevelVar) {
+	case "none":
+		logLevel = zerolog.Disabled
 	case "trace":
 		logLevel = zerolog.TraceLevel
 	case "debug":

cmd/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/checkmarx/2ms/lib/reporting"
 	"github.com/checkmarx/2ms/lib/secrets"
 	"github.com/checkmarx/2ms/plugins"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -85,7 +86,7 @@ func Execute() (int, error) {
 	cobra.OnInitialize(initialize)
 	rootCmd.PersistentFlags().StringVar(&configFilePath, configFileFlag, "", "config file path")
 	cobra.CheckErr(rootCmd.MarkPersistentFlagFilename(configFileFlag, "yaml", "yml", "json"))
-	rootCmd.PersistentFlags().StringVar(&logLevelVar, logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)")
+	rootCmd.PersistentFlags().StringVar(&logLevelVar, logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal, none)")
 	rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)")
 	rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif")
 	rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex")
@@ -165,8 +166,11 @@ func postRun(cmd *cobra.Command, args []string) error {
 	cfg := config.LoadConfig("2ms", Version)
 
 	if Report.TotalItemsScanned > 0 {
-		if err := Report.ShowReport(stdoutFormatVar, cfg); err != nil {
-			return err
+
+		if zerolog.GlobalLevel() != zerolog.Disabled {
+			if err := Report.ShowReport(stdoutFormatVar, cfg); err != nil {
+				return err
+			}
 		}
 
 		if len(reportPathVar) > 0 {

lib/reporting/json.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 )
 
-func writeJson(report Report) (string, error) {
+func writeJson(report *Report) (string, error) {
 	jsonReport, err := json.MarshalIndent(report, "", " ")
 	if err != nil {
 		return "", fmt.Errorf("failed to create Json report with error: %v", err)

lib/reporting/report.go

@@ -28,7 +28,6 @@ func Init() *Report {
 		Results: make(map[string][]*secrets.Secret),
 	}
 }
-
 func (r *Report) ShowReport(format string, cfg *config.Config) error {
 	output, err := r.GetOutput(format, cfg)
 	if err != nil {
@@ -69,14 +68,13 @@ func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error {
 func (r *Report) GetOutput(format string, cfg *config.Config) (string, error) {
 	var output string
 	var err error
-
 	switch format {
 	case jsonFormat:
-		output, err = writeJson(*r)
+		output, err = writeJson(r)
 	case longYamlFormat, shortYamlFormat:
-		output, err = writeYaml(*r)
+		output, err = writeYaml(r)
 	case sarifFormat:
-		output, err = writeSarif(*r, cfg)
+		output, err = writeSarif(r, cfg)
 	}
 	return output, err
 }

lib/reporting/report_test.go

@@ -11,7 +11,9 @@ import (
 
 	"github.com/checkmarx/2ms/lib/config"
 	"github.com/checkmarx/2ms/lib/secrets"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
 )
 
 // test input results
@@ -224,6 +226,9 @@ func TestWriteReportInNonExistingDir(t *testing.T) {
 }
 
 func TestGetOutputSarif(t *testing.T) {
+
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+
 	tests := []struct {
 		name    string
 		arg     Report
@@ -334,3 +339,107 @@ func SortResults(results1, results2 []Results) {
 		return results2[i].Message.Text < results2[j].Message.Text
 	})
 }
+
+func TestGetOutputYAML(t *testing.T) {
+	testCases := []struct {
+		name   string
+		report Report
+	}{{
+		name: "No secrets found",
+		report: Report{
+			TotalItemsScanned: 5,
+			TotalSecretsFound: 0,
+			Results:           map[string][]*secrets.Secret{},
+		},
+	},
+		{
+			name: "Single real secret in hardcodedPassword.go",
+			report: Report{
+				TotalItemsScanned: 1,
+				TotalSecretsFound: 1,
+				Results: map[string][]*secrets.Secret{
+					"c6490d749fd4670fde969011d99ea5c4c4b1c0d7": {
+						{
+							ID:               "c6490d749fd4670fde969011d99ea5c4c4b1c0d7",
+							Source:           "..\\2ms\\engine\\rules\\hardcodedPassword.go",
+							RuleID:           "generic-api-key",
+							StartLine:        45,
+							EndLine:          45,
+							LineContent:      "value",
+							StartColumn:      8,
+							EndColumn:        64,
+							Value:            "value",
+							ValidationStatus: "",
+							CvssScore:        8.2,
+							RuleDescription:  "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "Multiple real JWT secrets in jwt.txt",
+			report: Report{
+				TotalItemsScanned: 2,
+				TotalSecretsFound: 2,
+				Results: map[string][]*secrets.Secret{
+					"12fd8706491196cbfbdddd2fdcd650ed842dd963": {
+						{
+							ID:               "12fd8706491196cbfbdddd2fdcd650ed842dd963",
+							Source:           "..\\2ms\\pkg\\testData\\secrets\\jwt.txt",
+							RuleID:           "jwt",
+							StartLine:        1,
+							EndLine:          1,
+							LineContent:      "line content",
+							StartColumn:      129,
+							EndColumn:        232,
+							Value:            "value",
+							ValidationStatus: "",
+							CvssScore:        8.2,
+							RuleDescription:  "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.",
+							ExtraDetails: map[string]interface{}{
+								"secretDetails": map[string]interface{}{
+									"name": "mockName2",
+									"sub":  "mockSub2",
+								},
+							},
+						},
+						{
+							ID:               "12fd8706491196cbfbdddd2fdcd650ed842dd963",
+							Source:           "..\\2ms\\pkg\\testData\\secrets\\jwt.txt",
+							RuleID:           "jwt",
+							StartLine:        2,
+							EndLine:          2,
+							LineContent:      "line Content",
+							StartColumn:      64,
+							EndColumn:        166,
+							Value:            "value",
+							ValidationStatus: "",
+							CvssScore:        8.2,
+							RuleDescription:  "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.",
+							ExtraDetails: map[string]interface{}{
+								"secretDetails": map[string]interface{}{
+									"name": "mockName2",
+									"sub":  "mockSub2",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			output, err := tc.report.GetOutput("yaml", &config.Config{Name: "report", Version: "1"})
+			assert.NoError(t, err)
+
+			var report Report
+			err = yaml.Unmarshal([]byte(output), &report)
+			assert.NoError(t, err)
+
+			assert.Equal(t, tc.report, report)
+		})
+	}
+}

lib/reporting/sarif.go

@@ -9,7 +9,7 @@ import (
 	"github.com/checkmarx/2ms/lib/secrets"
 )
 
-func writeSarif(report Report, cfg *config.Config) (string, error) {
+func writeSarif(report *Report, cfg *config.Config) (string, error) {
 	sarif := Sarif{
 		Schema:  "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json",
 		Version: "2.1.0",
@@ -24,7 +24,7 @@ func writeSarif(report Report, cfg *config.Config) (string, error) {
 	return string(sarifReport), nil
 }
 
-func getRuns(report Report, cfg *config.Config) []Runs {
+func getRuns(report *Report, cfg *config.Config) []Runs {
 	return []Runs{
 		{
 			Tool:    getTool(report, cfg),
@@ -33,7 +33,7 @@ func getRuns(report Report, cfg *config.Config) []Runs {
 	}
 }
 
-func getTool(report Report, cfg *config.Config) Tool {
+func getTool(report *Report, cfg *config.Config) Tool {
 	tool := Tool{
 		Driver: Driver{
 			Name:            cfg.Name,
@@ -45,7 +45,7 @@ func getTool(report Report, cfg *config.Config) Tool {
 	return tool
 }
 
-func getRules(report Report) []*SarifRule {
+func getRules(report *Report) []*SarifRule {
 	uniqueRulesMap := make(map[string]*SarifRule)
 	var reportRules []*SarifRule
 	for _, reportSecrets := range report.Results {
@@ -64,15 +64,15 @@ func getRules(report Report) []*SarifRule {
 	return reportRules
 }
 
-func hasNoResults(report Report) bool {
+func hasNoResults(report *Report) bool {
 	return len(report.Results) == 0
 }
 
 func messageText(ruleName string, filePath string) string {
 	return fmt.Sprintf("%s has detected secret for file %s.", ruleName, filePath)
 }
 
-func getResults(report Report) []Results {
+func getResults(report *Report) []Results {
 	var results []Results
 
 	// if this report has no results, ensure that it is represented as [] instead of null/nil

lib/reporting/yaml.go

@@ -1,14 +1,58 @@
 package reporting
 
 import (
+	"fmt"
+	"strings"
+
 	"gopkg.in/yaml.v3"
 )
 
-func writeYaml(report Report) (string, error) {
-	yamlReport, err := yaml.Marshal(&report)
-	if err != nil {
-		return "", err
+func writeYaml(report *Report) (string, error) {
+	estimatedSize := 1024 + len(report.Results)*512
+	var builder strings.Builder
+	builder.Grow(estimatedSize)
+
+	fmt.Fprintf(&builder, "totalitemsscanned: %d\n", report.TotalItemsScanned)
+	fmt.Fprintf(&builder, "totalsecretsfound: %d\n", report.TotalSecretsFound)
+	if report.TotalSecretsFound == 0 {
+		fmt.Fprint(&builder, "results: {}\n")
+	} else {
+
+		builder.WriteString("results:\n")
+		for _, secretsList := range report.Results {
+			if len(secretsList) > 0 {
+				fmt.Fprintf(&builder, "  %s:\n", secretsList[0].ID)
+			}
+			for _, s := range secretsList {
+				fmt.Fprintf(&builder, "    - id: %s\n", s.ID)
+				fmt.Fprintf(&builder, "      source: %s\n", s.Source)
+				fmt.Fprintf(&builder, "      ruleid: %s\n", s.RuleID)
+				fmt.Fprintf(&builder, "      startline: %d\n", s.StartLine)
+				fmt.Fprintf(&builder, "      endline: %d\n", s.EndLine)
+				fmt.Fprintf(&builder, "      linecontent: %q\n", s.LineContent)
+				fmt.Fprintf(&builder, "      startcolumn: %d\n", s.StartColumn)
+				fmt.Fprintf(&builder, "      endcolumn: %d\n", s.EndColumn)
+				fmt.Fprintf(&builder, "      value: %s\n", s.Value)
+				fmt.Fprintf(&builder, "      validationstatus: %q\n", fmt.Sprintf("%v", s.ValidationStatus))
+				fmt.Fprintf(&builder, "      ruledescription: %s\n", s.RuleDescription)
+				if len(s.ExtraDetails) > 0 {
+					builder.WriteString("      extradetails:\n")
+					marshaled, err := yaml.Marshal(s.ExtraDetails)
+					if err != nil {
+						fmt.Fprintf(&builder, "        error: %v\n", err)
+					} else {
+						lines := strings.Split(string(marshaled), "\n")
+						for _, line := range lines {
+							if line != "" {
+								fmt.Fprintf(&builder, "        %s\n", line)
+							}
+						}
+					}
+				}
+				fmt.Fprintf(&builder, "      cvssscore: %.1f\n", s.CvssScore)
+			}
+		}
 	}
 
-	return string(yamlReport), nil
+	return builder.String(), nil
 }