fix: add newline in fragment

.2ms.yml

@@ -51,5 +51,16 @@ ignore-result:
   - 51a6f4e3c7e3a79c9722abb7541b4902098e526b # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
   - 53803ee7e880952e926898a434acff4483fec67e # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
   - aa52405f239a8be1284d933025c557b071b24036 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
+  - 61a50a3d783926ae08307cc9727e9b1830f4044d # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - b8fddbf33e0da0db4714425e2baedbc74865b72e # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 9d88a51fcfe0bba421e3ab285c0bcd5884889520 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - ad5cd04241f630992be8c34e2626d2372dbd7690 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 0648cbaed8d23cd128f7e9111b51d739d1f5769b # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 27ba3f4fed916199f4f65f30ffc111b8ee3dc3db # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 52ab4ec04145a57835d9ee91380c8a559b34706e # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 35a133edb564767157c6bd807f57009a9ee78349 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 0b43a67f6eb1f2d1b744b5813eec4eb9f167023d # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - ba04dd95db7fd550ebb0f295d80fce4e281529fb # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
+  - 35a133edb564767157c6bd807f57009a9ee78349 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0
   - 854547fc6e35c0d1f63c0f4d426aebd4d64679fc # False positive, see https://github.com/gitleaks/gitleaks/pull/1358, found at https://github.com/Checkmarx/2ms/commit/45a5c9d35ff910dfec5e5a76cdedb8977da5dd34#diff-d712d2256df359061d691b711ca7ed30ba408199b1e3801cef289779778d8bad
-  - b7c3ac03d8a24892a2c4be5810ce73ffdf6ba3ae # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - b7c3ac03d8a24892a2c4be5810ce73ffdf6ba3ae # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10

engine/engine.go

@@ -3,14 +3,15 @@
 import (
 	"crypto/sha1"
 	"fmt"
-	"github.com/checkmarx/2ms/engine/linecontent"
-	"github.com/checkmarx/2ms/engine/score"
 	"os"
 	"regexp"
 	"strings"
 	"sync"
 	"text/tabwriter"
 
+	"github.com/checkmarx/2ms/engine/linecontent"
+	"github.com/checkmarx/2ms/engine/score"
+
 	"github.com/checkmarx/2ms/engine/rules"
 	"github.com/checkmarx/2ms/engine/validation"
 	"github.com/checkmarx/2ms/lib/secrets"
@@ -85,7 +86,10 @@
 		Raw:      *item.GetContent(),
 		FilePath: item.GetSource(),
 	}
+
+	fragment.Raw += "\n"
 	gitInfo := item.GetGitInfo()
+
 	for _, value := range e.detector.Detect(fragment) {
 		itemId := getFindingId(item, value)
 		var startLine, endLine int

tests/2ms.json

@@ -0,0 +1,36 @@
+{
+ "totalItemsScanned": 1,
+ "totalSecretsFound": 2,
+ "results": {
+  "565296f55a43023fbfaff6d1df3c75ff9184e4b8": [
+   {
+    "id": "565296f55a43023fbfaff6d1df3c75ff9184e4b8",
+    "source": "./testData/input/secret_at_end_with_newline.txt",
+    "ruleId": "generic-api-key",
+    "startLine": 2,
+    "endLine": 2,
+    "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`\r",
+    "startColumn": 6,
+    "endColumn": 88,
+    "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",
+    "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+    "cvssScore": 8.2
+   }
+  ],
+  "e1e313e377f4b84f4608b6b450aa4f7854113c7a": [
+   {
+    "id": "e1e313e377f4b84f4608b6b450aa4f7854113c7a",
+    "source": "./testData/input/secret_at_end_with_newline.txt",
+    "ruleId": "generic-api-key",
+    "startLine": 1,
+    "endLine": 1,
+    "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r",
+    "startColumn": 3,
+    "endColumn": 81,
+    "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506",
+    "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+    "cvssScore": 8.2
+   }
+  ]
+ }
+}

tests/e2e.go

@@ -9,7 +9,9 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/checkmarx/2ms/lib/reporting"
 )
@@ -20,7 +22,7 @@ type cli struct {
 }
 
 func createCLI(outputDir string) (cli, error) {
-	executable := path.Join(outputDir, "2ms")
+	executable := filepath.Join(outputDir, "2ms")
 	lib, err := build.Import("github.com/checkmarx/2ms", "", build.FindOnly)
 	if err != nil {
 		return cli{}, fmt.Errorf("failed to import 2ms: %s", err)
@@ -77,3 +79,23 @@ func (c *cli) getReport() (reporting.Report, error) {
 
 	return *report, nil
 }
+
+// normalizeReportData recursively traverses the report data and removes any carriage return characters.
+func normalizeReportData(data interface{}) interface{} {
+	switch v := data.(type) {
+	case string:
+		return strings.ReplaceAll(v, "\r", "")
+	case []interface{}:
+		for i, item := range v {
+			v[i] = normalizeReportData(item)
+		}
+		return v
+	case map[string]interface{}:
+		for key, val := range v {
+			v[key] = normalizeReportData(val)
+		}
+		return v
+	default:
+		return data
+	}
+}

tests/e2e_test.go

@@ -1,6 +1,12 @@
 package tests
 
-import "testing"
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
 
 func TestIntegration(t *testing.T) {
 	if testing.Short() {
@@ -79,3 +85,81 @@ func TestIntegration(t *testing.T) {
 		}
 	})
 }
+
+func TestSecretsEdgeCases(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping edge cases test")
+	}
+
+	tests := []struct {
+		Name               string
+		ScanTarget         string
+		TargetPath         string
+		ExpectedReportPath string
+	}{
+		{
+			Name:               "secret at end without newline (filesystem)",
+			ScanTarget:         "filesystem",
+			TargetPath:         "testData/input/secret_at_end.txt",
+			ExpectedReportPath: "testData/expectedReport/secret_at_end_report.json",
+		},
+		{
+			Name:               "secret at end with multiLine (filesystem)",
+			ScanTarget:         "filesystem",
+			TargetPath:         "testData/input/multi_line_secret.txt",
+			ExpectedReportPath: "testData/expectedReport/multi_line_secret_report.json",
+		},
+		{
+			Name:               "secret at end with backspace in newline (filesystem)",
+			ScanTarget:         "filesystem",
+			TargetPath:         "testData/input/secret_at_end_with_newline.txt",
+			ExpectedReportPath: "testData/expectedReport/secret_at_end_with_newline_report.json",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.Name, func(t *testing.T) {
+			executable, err := createCLI(t.TempDir())
+			if err != nil {
+				t.Fatalf("failed to build CLI: %s", err)
+			}
+
+			args := []string{tc.ScanTarget}
+			if tc.ScanTarget == "filesystem" {
+				args = append(args, "--path", tc.TargetPath)
+			} else {
+				args = append(args, tc.TargetPath)
+			}
+			args = append(args, "--ignore-on-exit", "results")
+
+			if err := executable.run(args[0], args[1:]...); err != nil {
+				t.Fatalf("error running scan with args: %v, got: %v", args, err)
+			}
+
+			actualReport, err := executable.getReport()
+			if err != nil {
+				t.Fatalf("failed to get report: %s", err)
+			}
+
+			expectedBytes, err := os.ReadFile(tc.ExpectedReportPath)
+			assert.NoError(t, err, "failed to read expected report")
+
+			var expectedReportMap map[string]interface{}
+			err = json.Unmarshal(expectedBytes, &expectedReportMap)
+			assert.NoError(t, err, "failed to unmarshal expected report JSON")
+
+			actualReportBytes, err := json.Marshal(actualReport)
+			assert.NoError(t, err, "failed to marshal actual report to JSON")
+
+			var actualReportMap map[string]interface{}
+			err = json.Unmarshal(actualReportBytes, &actualReportMap)
+			assert.NoError(t, err, "failed to unmarshal actual report JSON")
+
+			normalizedExpectedReport := normalizeReportData(expectedReportMap).(map[string]interface{})
+			normalizedActualReport := normalizeReportData(actualReportMap).(map[string]interface{})
+
+			assert.EqualValuesf(t, normalizedExpectedReport, normalizedActualReport, "Test Fail")
+
+		})
+	}
+}

tests/testData/expectedReport/multi_line_secret_report.json

@@ -0,0 +1,51 @@
+{
+    "totalItemsScanned": 1,
+    "totalSecretsFound": 3,
+    "results": {
+     "047d26912b890e89c7f01b7ec9e926390224e4f0": [
+      {
+       "id": "047d26912b890e89c7f01b7ec9e926390224e4f0",
+       "source": "testData/input/multi_line_secret.txt",
+       "ruleId": "private-key",
+       "startLine": 3,
+       "endLine": 4,
+       "lineContent": "\n        -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n        vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----\r",
+       "startColumn": 10,
+       "endColumn": 377,
+       "value": "-----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n        vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----",
+       "ruleDescription": "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.",
+       "cvssScore": 8.2
+      }
+     ],
+     "58e5a02e5571db6dc1f9c0fdba8d86e254225bf1": [
+      {
+       "id": "58e5a02e5571db6dc1f9c0fdba8d86e254225bf1",
+       "source": "testData/input/multi_line_secret.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 1,
+       "endLine": 1,
+       "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r",
+       "startColumn": 3,
+       "endColumn": 81,
+       "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ],
+     "ed47a9a9052d119d91763ce84d689370fdbccf1f": [
+      {
+       "id": "ed47a9a9052d119d91763ce84d689370fdbccf1f",
+       "source": "testData/input/multi_line_secret.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 2,
+       "endLine": 2,
+       "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`\r",
+       "startColumn": 6,
+       "endColumn": 88,
+       "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ]
+    }
+   }

tests/testData/expectedReport/secret_at_end_report.json

@@ -0,0 +1,36 @@
+{
+    "totalItemsScanned": 1,
+    "totalSecretsFound": 2,
+    "results": {
+     "6a3e642795e27b989c54ac0c91147fe8e9a405b4": [
+      {
+       "id": "6a3e642795e27b989c54ac0c91147fe8e9a405b4",
+       "source": "testData/input/secret_at_end.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 2,
+       "endLine": 2,
+       "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`",
+       "startColumn": 6,
+       "endColumn": 88,
+       "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ],
+     "84bc054139c2363b37538209055a2d9c23026fab": [
+      {
+       "id": "84bc054139c2363b37538209055a2d9c23026fab",
+       "source": "testData/input/secret_at_end.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 1,
+       "endLine": 1,
+       "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r",
+       "startColumn": 3,
+       "endColumn": 81,
+       "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ]
+    }
+   }

tests/testData/expectedReport/secret_at_end_with_newline_report.json

@@ -0,0 +1,36 @@
+{
+    "totalItemsScanned": 1,
+    "totalSecretsFound": 2,
+    "results": {
+     "6af9b6df67e2971f45e6e27d4e068c2a515d2961": [
+      {
+       "id": "6af9b6df67e2971f45e6e27d4e068c2a515d2961",
+       "source": "testData/input/secret_at_end_with_newline.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 2,
+       "endLine": 2,
+       "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`\r",
+       "startColumn": 6,
+       "endColumn": 88,
+       "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ],
+     "f4b4bf79a4000811227225e3c556ea3862cfcb1a": [
+      {
+       "id": "f4b4bf79a4000811227225e3c556ea3862cfcb1a",
+       "source": "testData/input/secret_at_end_with_newline.txt",
+       "ruleId": "generic-api-key",
+       "startLine": 1,
+       "endLine": 1,
+       "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r",
+       "startColumn": 3,
+       "endColumn": 81,
+       "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506",
+       "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+       "cvssScore": 8.2
+      }
+     ]
+    }
+   }

tests/testData/input/multi_line_secret.txt

@@ -0,0 +1,5 @@
+`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
+		`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`
+        -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq
+        vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----
+

tests/testData/input/secret_at_end.txt

@@ -0,0 +1,2 @@
+`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
+		`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`

tests/testData/input/secret_at_end_with_newline.txt

@@ -0,0 +1,2 @@
+`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
+		`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`