fix: add newline in fragment

.2ms.yml

@@ -51,5 +51,22 @@ ignore-result:
   - 51a6f4e3c7e3a79c9722abb7541b4902098e526b # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
   - 53803ee7e880952e926898a434acff4483fec67e # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
   - aa52405f239a8be1284d933025c557b071b24036 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
+  - 61a50a3d783926ae08307cc9727e9b1830f4044d # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - b8fddbf33e0da0db4714425e2baedbc74865b72e # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 9d88a51fcfe0bba421e3ab285c0bcd5884889520 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - ad5cd04241f630992be8c34e2626d2372dbd7690 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 0648cbaed8d23cd128f7e9111b51d739d1f5769b # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 27ba3f4fed916199f4f65f30ffc111b8ee3dc3db # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 52ab4ec04145a57835d9ee91380c8a559b34706e # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 35a133edb564767157c6bd807f57009a9ee78349 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 0b43a67f6eb1f2d1b744b5813eec4eb9f167023d # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - ba04dd95db7fd550ebb0f295d80fce4e281529fb # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 35a133edb564767157c6bd807f57009a9ee78349 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
   - 854547fc6e35c0d1f63c0f4d426aebd4d64679fc # False positive, see https://github.com/gitleaks/gitleaks/pull/1358, found at https://github.com/Checkmarx/2ms/commit/45a5c9d35ff910dfec5e5a76cdedb8977da5dd34#diff-d712d2256df359061d691b711ca7ed30ba408199b1e3801cef289779778d8bad
   - b7c3ac03d8a24892a2c4be5810ce73ffdf6ba3ae # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - f40881f8369f0d90670fc22a719ecd0ba9cb2f02 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - 35a5080cb11d663e33e3ced8f39a24920ca44c8a # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - 7b7c1a0b1c5760490d843e0b9bfe540665d20b28 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - 92b1996f9815a2fbd9299a1997ce0bc2c153624f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - bf2e01278453a987f05b69e6c536358cab343322 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - c9ae034a5a03a540d50a2686f74fcbb5117f181c # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10

engine/engine.go

@@ -3,14 +3,15 @@ package engine
 import (
 	"crypto/sha1"
 	"fmt"
-	"github.com/checkmarx/2ms/engine/linecontent"
-	"github.com/checkmarx/2ms/engine/score"
 	"os"
 	"regexp"
 	"strings"
 	"sync"
 	"text/tabwriter"
 
+	"github.com/checkmarx/2ms/engine/linecontent"
+	"github.com/checkmarx/2ms/engine/score"
+
 	"github.com/checkmarx/2ms/engine/rules"
 	"github.com/checkmarx/2ms/engine/validation"
 	"github.com/checkmarx/2ms/lib/secrets"
@@ -80,13 +81,19 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 
 func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) {
 	defer wg.Done()
+	const CxFileEndMarker = ";cx-file-end"
 
 	fragment := detect.Fragment{
 		Raw:      *item.GetContent(),
 		FilePath: item.GetSource(),
 	}
+
+	fragment.Raw += CxFileEndMarker + "\n"
 	gitInfo := item.GetGitInfo()
-	for _, value := range e.detector.Detect(fragment) {
+
+	values := e.detector.Detect(fragment)
+
+	for idx, value := range values {
 		itemId := getFindingId(item, value)
 		var startLine, endLine int
 		var err error
@@ -103,6 +110,12 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
 			startLine = value.StartLine
 			endLine = value.EndLine
 		}
+
+		if idx == len(values)-1 && strings.HasSuffix(value.Line, CxFileEndMarker) {
+			value.Line = value.Line[:len(value.Line)-len(CxFileEndMarker)]
+			value.EndColumn--
+		}
+
 		lineContent, err := linecontent.GetLineContent(value.Line, value.Secret)
 		if err != nil {
 			errors <- fmt.Errorf("failed to get line content for source %s: %w", item.GetSource(), err)

go.mod

@@ -5,7 +5,6 @@ go 1.23.6
 require (
 	github.com/bwmarrin/discordgo v0.27.1
 	github.com/gitleaks/go-gitdiff v0.9.0
-	github.com/google/go-cmp v0.6.0
 	github.com/rs/zerolog v1.32.0
 	github.com/slack-go/slack v0.12.2
 	github.com/spf13/cobra v1.8.0

lib/utils/test_utils.go

@@ -0,0 +1,27 @@
+package utils
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// normalizeReportData recursively traverses the report data and removes any carriage return characters.
+func NormalizeReportData(data interface{}) (interface{}, error) {
+	bytes, err := json.Marshal(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal data: %w", err)
+	}
+
+	jsonStr := string(bytes)
+	jsonStr = strings.ReplaceAll(jsonStr, "\\r", "")
+
+	// Unmarshal back to a Go data structure
+	var result interface{}
+	err = json.Unmarshal([]byte(jsonStr), &result)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal data: %w", err)
+	}
+
+	return result, nil
+}

pkg/scan_test.go

@@ -4,15 +4,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"testing"
 
 	"github.com/checkmarx/2ms/cmd"
 	"github.com/checkmarx/2ms/lib/reporting"
 	"github.com/checkmarx/2ms/lib/secrets"
+	"github.com/checkmarx/2ms/lib/utils"
 	"github.com/checkmarx/2ms/plugins"
-	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -23,26 +22,6 @@ const (
 	expectedReportResultsIgnoredPath = "testData/expectedReportWithIgnoredResults.json"
 )
 
-// normalizeReportData recursively traverses the report data and removes any carriage return characters.
-func normalizeReportData(data interface{}) interface{} {
-	switch v := data.(type) {
-	case string:
-		return strings.ReplaceAll(v, "\r", "")
-	case []interface{}:
-		for i, item := range v {
-			v[i] = normalizeReportData(item)
-		}
-		return v
-	case map[string]interface{}:
-		for key, val := range v {
-			v[key] = normalizeReportData(val)
-		}
-		return v
-	default:
-		return data
-	}
-}
-
 func TestScan(t *testing.T) {
 	t.Run("Successful Scan with Multiple Items", func(t *testing.T) {
 		cmd.Report = reporting.Init()
@@ -101,12 +80,13 @@ func TestScan(t *testing.T) {
 		assert.NoError(t, err, "failed to unmarshal actual report JSON")
 
 		// Normalize both expected and actual maps.
-		expectedReport = normalizeReportData(expectedReport).(map[string]interface{})
-		actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{})
+		normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport)
+		assert.NoError(t, err, "Failed to normalize actual report")
 
-		if !cmp.Equal(expectedReport, actualReportMap) {
-			t.Errorf("Scan report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap))
-		}
+		normalizedActualReport, err := utils.NormalizeReportData(actualReportMap)
+		assert.NoError(t, err, "Failed to normalize actual report")
+
+		assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport)
 	})
 	t.Run("Successful scan with multiple items and ignored results", func(t *testing.T) {
 		cmd.Report = reporting.Init()
@@ -168,13 +148,13 @@ func TestScan(t *testing.T) {
 		err = json.Unmarshal(actualReportBytes, &actualReportMap)
 		assert.NoError(t, err, "failed to unmarshal actual report JSON")
 
-		// Normalize both expected and actual maps.
-		expectedReport = normalizeReportData(expectedReport).(map[string]interface{})
-		actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{})
+		normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport)
+		assert.NoError(t, err, "Failed to normalize actual report")
 
-		if !cmp.Equal(expectedReport, actualReportMap) {
-			t.Errorf("Scan report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap))
-		}
+		normalizedActualReport, err := utils.NormalizeReportData(actualReportMap)
+		assert.NoError(t, err, "Failed to normalize actual report")
+
+		assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport)
 	})
 	t.Run("error handling should work", func(t *testing.T) {
 		cmd.Report = reporting.Init()
@@ -309,12 +289,13 @@ func TestScanDynamic(t *testing.T) {
 		assert.NoError(t, err, "failed to unmarshal actual report JSON")
 
 		// Normalize both maps.
-		expectedReport = normalizeReportData(expectedReport).(map[string]interface{})
-		actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{})
+		normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport)
+		assert.NoError(t, err, "Failed to normalize actual report")
 
-		if !cmp.Equal(expectedReport, actualReportMap) {
-			t.Errorf("ScanDynamic report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap))
-		}
+		normalizedActualReport, err := utils.NormalizeReportData(actualReportMap)
+		assert.NoError(t, err, "Failed to normalize actual report")
+
+		assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport)
 	})
 
 	t.Run("Successful ScanDynamic with Multiple Items and Ignored Results", func(t *testing.T) {
@@ -385,12 +366,13 @@ func TestScanDynamic(t *testing.T) {
 		assert.NoError(t, err, "failed to unmarshal actual report JSON")
 
 		// Normalize both maps.
-		expectedReport = normalizeReportData(expectedReport).(map[string]interface{})
-		actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{})
+		normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport)
+		assert.NoError(t, err, "Failed to normalize actual report")
 
-		if !cmp.Equal(expectedReport, actualReportMap) {
-			t.Errorf("ScanDynamic report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap))
-		}
+		normalizedActualReport, err := utils.NormalizeReportData(actualReportMap)
+		assert.NoError(t, err, "Failed to normalize actual report")
+
+		assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport)
 	})
 
 	t.Run("error handling should work", func(t *testing.T) {

pkg/testData/expectedReport.json

@@ -39,7 +39,7 @@
    "ruleId" : "jwt",
    "startLine" : 1,
    "endLine" : 1,
-   "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r\n           Text_Example =                                     eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2",
+   "lineContent": "\n           Text_Example =                                     eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2",
    "startColumn" : 64,
    "endColumn" : 166,
    "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2",

tests/e2e_test.go

@@ -1,6 +1,15 @@
 package tests
 
-import "testing"
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/checkmarx/2ms/lib/utils"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
 
 func TestIntegration(t *testing.T) {
 	if testing.Short() {
@@ -79,3 +88,80 @@ func TestIntegration(t *testing.T) {
 		}
 	})
 }
+
+func TestSecretsScans(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping edge cases test")
+	}
+
+	tests := []struct {
+		Name               string
+		ScanTarget         string
+		TargetPath         string
+		ExpectedReportPath string
+	}{
+		{
+			Name:               "secret at end without newline",
+			ScanTarget:         "filesystem",
+			TargetPath:         "testData/input/secret_at_end.txt",
+			ExpectedReportPath: "testData/expectedReport/secret_at_end_report.json",
+		},
+		{
+			Name:               "multi line secret ",
+			ScanTarget:         "filesystem",
+			TargetPath:         "testData/input/multi_line_secret.txt",
+			ExpectedReportPath: "testData/expectedReport/multi_line_secret_report.json",
+		},
+		{
+			Name:               "secret at end with newline ",
+			ScanTarget:         "filesystem",
+			TargetPath:         "testData/input/secret_at_end_with_newline.txt",
+			ExpectedReportPath: "testData/expectedReport/secret_at_end_with_newline_report.json",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.Name, func(t *testing.T) {
+			executable, err := createCLI(t.TempDir())
+			require.Nil(t, err, "failed to build CLI")
+
+			args := []string{tc.ScanTarget}
+			if tc.ScanTarget == "filesystem" {
+				args = append(args, "--path", tc.TargetPath)
+			} else {
+				args = append(args, tc.TargetPath)
+			}
+			args = append(args, "--ignore-on-exit", "results")
+
+			if err := executable.run(args[0], args[1:]...); err != nil {
+				t.Fatalf("error running scan with args: %v, got: %v", args, err)
+			}
+
+			actualReport, err := executable.getReport()
+			require.NoError(t, err, "failed to get report")
+
+			expectedBytes, err := os.ReadFile(tc.ExpectedReportPath)
+			assert.NoError(t, err, "failed to read expected report")
+
+			var expectedReportMap map[string]interface{}
+			err = json.Unmarshal(expectedBytes, &expectedReportMap)
+			assert.NoError(t, err, "failed to unmarshal expected report JSON")
+
+			actualReportBytes, err := json.Marshal(actualReport)
+			assert.NoError(t, err, "failed to marshal actual report to JSON")
+
+			var actualReportMap map[string]interface{}
+
+			err = json.Unmarshal(actualReportBytes, &actualReportMap)
+			assert.NoError(t, err, "failed to unmarshal actual report JSON")
+
+			normalizedExpectedReport, err := utils.NormalizeReportData(expectedReportMap)
+			assert.NoError(t, err, "Failed to normalize expected report")
+
+			normalizedActualReport, err := utils.NormalizeReportData(actualReportMap)
+			assert.NoError(t, err, "Failed to normalize expected report")
+
+			assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport)
+		})
+	}
+}

tests/testData/expectedReport/multi_line_secret_report.json

@@ -0,0 +1,51 @@
+{
+    "totalItemsScanned": 1,
+    "totalSecretsFound": 3,
+    "results": {
+     "047d26912b890e89c7f01b7ec9e926390224e4f0": [
+      {
+       "id": "047d26912b890e89c7f01b7ec9e926390224e4f0",
+       "source": "testData/input/multi_line_secret.txt",
+       "ruleId": "private-key",
+       "startLine": 3,
+       "endLine": 4,
+       "lineContent": "\n        -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n        vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----\r",
+       "startColumn": 10,
+       "endColumn": 377,
+       "value": "-----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n        vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----",
+       "ruleDescription": "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.",
+       "cvssScore": 8.2
+      }
+     ],
+     "58e5a02e5571db6dc1f9c0fdba8d86e254225bf1": [
+      {
+       "id": "58e5a02e5571db6dc1f9c0fdba8d86e254225bf1",
+       "source": "testData/input/multi_line_secret.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 1,
+       "endLine": 1,
+       "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r",
+       "startColumn": 3,
+       "endColumn": 81,
+       "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ],
+     "ed47a9a9052d119d91763ce84d689370fdbccf1f": [
+      {
+       "id": "ed47a9a9052d119d91763ce84d689370fdbccf1f",
+       "source": "testData/input/multi_line_secret.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 2,
+       "endLine": 2,
+       "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`\r",
+       "startColumn": 6,
+       "endColumn": 88,
+       "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ]
+    }
+   }