From 2cdf66865f2bdf006869b8a84f448bec3525bfa0 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Wed, 7 May 2025 15:13:13 +0100 Subject: [PATCH 1/6] Add e2e tests --- engine/engine.go | 7 +- tests/e2e.go | 7 +- tests/e2e_test.go | 80 ++++++++++++++++++- tests/testData/expectedReport/report1.json | 36 +++++++++ tests/testData/expectedReport/report2.json | 51 ++++++++++++ tests/testData/expectedReport/report3.json | 36 +++++++++ tests/testData/input/multi_line_secret.txt | 5 ++ tests/testData/input/secret_at_end.txt | 2 + .../input/secret_at_end_with_newline.txt | 3 + 9 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 tests/testData/expectedReport/report1.json create mode 100644 tests/testData/expectedReport/report2.json create mode 100644 tests/testData/expectedReport/report3.json create mode 100644 tests/testData/input/multi_line_secret.txt create mode 100644 tests/testData/input/secret_at_end.txt create mode 100644 tests/testData/input/secret_at_end_with_newline.txt diff --git a/engine/engine.go b/engine/engine.go index a0581b57..721f15ee 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -3,14 +3,15 @@ package engine import ( "crypto/sha1" "fmt" - "github.com/checkmarx/2ms/engine/linecontent" - "github.com/checkmarx/2ms/engine/score" "os" "regexp" "strings" "sync" "text/tabwriter" + "github.com/checkmarx/2ms/engine/linecontent" + "github.com/checkmarx/2ms/engine/score" + "github.com/checkmarx/2ms/engine/rules" "github.com/checkmarx/2ms/engine/validation" "github.com/checkmarx/2ms/lib/secrets" @@ -85,6 +86,8 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S Raw: *item.GetContent(), FilePath: item.GetSource(), } + + fragment.Raw += "\n" for _, value := range e.detector.Detect(fragment) { itemId := getFindingId(item, value) var startLine, endLine int diff --git a/tests/e2e.go b/tests/e2e.go index 7af45f1f..28dbf7b1 100644 --- a/tests/e2e.go +++ b/tests/e2e.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "runtime" "github.com/checkmarx/2ms/lib/reporting" @@ -20,7 +21,11 @@ type cli struct { } func createCLI(outputDir string) (cli, error) { - executable := path.Join(outputDir, "2ms") + executableName := "2ms" + if runtime.GOOS == "windows" { + executableName += ".exe" + } + executable := filepath.Join(outputDir, executableName) lib, err := build.Import("github.com/checkmarx/2ms", "", build.FindOnly) if err != nil { return cli{}, fmt.Errorf("failed to import 2ms: %s", err) diff --git a/tests/e2e_test.go b/tests/e2e_test.go index 9448dbb1..f7f08691 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -1,6 +1,13 @@ package tests -import "testing" +import ( + "encoding/json" + "os" + "testing" + + "github.com/checkmarx/2ms/lib/reporting" + "github.com/google/go-cmp/cmp" +) func TestIntegration(t *testing.T) { if testing.Short() { @@ -79,3 +86,74 @@ func TestIntegration(t *testing.T) { } }) } + +func TestSecretsEdgeCases(t *testing.T) { + if testing.Short() { + t.Skip("skipping edge cases test") + } + + tests := []struct { + Name string + ScanTarget string + TargetPath string + ExpectedReportPath string + }{ + { + Name: "secret at end without newline (filesystem)", + ScanTarget: "filesystem", + TargetPath: "testData/input/secret_at_end.txt", + ExpectedReportPath: "testData/expectedReport/report1.json", + }, + { + Name: "secret at end with multiLine (filesystem)", + ScanTarget: "filesystem", + TargetPath: "testData/input/multi_line_secret.txt", + ExpectedReportPath: "testData/expectedReport/report2.json", + }, + { + Name: "secret at end with backspace in newline (filesystem)", + ScanTarget: "filesystem", + TargetPath: "testData/input/secret_at_end_with_newline.txt", + ExpectedReportPath: "testData/expectedReport/report3.json", + }, + } + + for _, tc := range tests { + t.Run(tc.Name, func(t *testing.T) { + executable, err := createCLI(t.TempDir()) + if err != nil { + t.Fatalf("failed to build CLI: %s", err) + } + + args := []string{tc.ScanTarget} + if tc.ScanTarget == "filesystem" { + args = append(args, "--path", tc.TargetPath) + } else { + args = append(args, tc.TargetPath) + } + args = append(args, "--ignore-on-exit", "results") + + if err := executable.run(args[0], args[1:]...); err != nil { + t.Fatalf("error running scan with args: %v, got: %v", args, err) + } + + actualReport, err := executable.getReport() + if err != nil { + t.Fatalf("failed to get report: %s", err) + } + + expectedBytes, err := os.ReadFile(tc.ExpectedReportPath) + if err != nil { + t.Fatalf("failed to read expected report: %s", err) + } + var expectedReport reporting.Report + if err := json.Unmarshal(expectedBytes, &expectedReport); err != nil { + t.Fatalf("failed to unmarshal expected report: %s", err) + } + + if !cmp.Equal(expectedReport, actualReport) { + t.Errorf("Scan report does not match expected report:\n%s", cmp.Diff(expectedReport, actualReport)) + } + }) + } +} diff --git a/tests/testData/expectedReport/report1.json b/tests/testData/expectedReport/report1.json new file mode 100644 index 00000000..9ddc1c42 --- /dev/null +++ b/tests/testData/expectedReport/report1.json @@ -0,0 +1,36 @@ +{ + "totalItemsScanned": 1, + "totalSecretsFound": 2, + "results": { + "6a3e642795e27b989c54ac0c91147fe8e9a405b4": [ + { + "id": "6a3e642795e27b989c54ac0c91147fe8e9a405b4", + "source": "testData/input/secret_at_end.txt", + "ruleId": "generic-api-key", + "startLine": 2, + "endLine": 2, + "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`", + "startColumn": 6, + "endColumn": 88, + "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ], + "84bc054139c2363b37538209055a2d9c23026fab": [ + { + "id": "84bc054139c2363b37538209055a2d9c23026fab", + "source": "testData/input/secret_at_end.txt", + "ruleId": "generic-api-key", + "startLine": 1, + "endLine": 1, + "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r", + "startColumn": 3, + "endColumn": 81, + "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ] + } + } \ No newline at end of file diff --git a/tests/testData/expectedReport/report2.json b/tests/testData/expectedReport/report2.json new file mode 100644 index 00000000..c2b54fee --- /dev/null +++ b/tests/testData/expectedReport/report2.json @@ -0,0 +1,51 @@ +{ + "totalItemsScanned": 1, + "totalSecretsFound": 3, + "results": { + "1ef1c6a736cb0725175ac969776c3fe0b4602389": [ + { + "id": "1ef1c6a736cb0725175ac969776c3fe0b4602389", + "source": "testData/input/multi_line_secret.txt", + "ruleId": "private-key", + "startLine": 3, + "endLine": 4, + "lineContent": "\n -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----\r", + "startColumn": 10, + "endColumn": 377, + "value": "-----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----", + "ruleDescription": "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.", + "cvssScore": 8.2 + } + ], + "58e5a02e5571db6dc1f9c0fdba8d86e254225bf1": [ + { + "id": "58e5a02e5571db6dc1f9c0fdba8d86e254225bf1", + "source": "testData/input/multi_line_secret.txt", + "ruleId": "generic-api-key", + "startLine": 1, + "endLine": 1, + "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r", + "startColumn": 3, + "endColumn": 81, + "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ], + "ed47a9a9052d119d91763ce84d689370fdbccf1f": [ + { + "id": "ed47a9a9052d119d91763ce84d689370fdbccf1f", + "source": "testData/input/multi_line_secret.txt", + "ruleId": "generic-api-key", + "startLine": 2, + "endLine": 2, + "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`\r", + "startColumn": 6, + "endColumn": 88, + "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ] + } + } \ No newline at end of file diff --git a/tests/testData/expectedReport/report3.json b/tests/testData/expectedReport/report3.json new file mode 100644 index 00000000..9f9a870d --- /dev/null +++ b/tests/testData/expectedReport/report3.json @@ -0,0 +1,36 @@ +{ + "totalItemsScanned": 1, + "totalSecretsFound": 2, + "results": { + "6af9b6df67e2971f45e6e27d4e068c2a515d2961": [ + { + "id": "6af9b6df67e2971f45e6e27d4e068c2a515d2961", + "source": "testData/input/secret_at_end_with_newline.txt", + "ruleId": "generic-api-key", + "startLine": 2, + "endLine": 2, + "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`\r", + "startColumn": 6, + "endColumn": 88, + "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ], + "f4b4bf79a4000811227225e3c556ea3862cfcb1a": [ + { + "id": "f4b4bf79a4000811227225e3c556ea3862cfcb1a", + "source": "testData/input/secret_at_end_with_newline.txt", + "ruleId": "generic-api-key", + "startLine": 1, + "endLine": 1, + "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r", + "startColumn": 3, + "endColumn": 81, + "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ] + } + } \ No newline at end of file diff --git a/tests/testData/input/multi_line_secret.txt b/tests/testData/input/multi_line_secret.txt new file mode 100644 index 00000000..75873920 --- /dev/null +++ b/tests/testData/input/multi_line_secret.txt @@ -0,0 +1,5 @@ +`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, + `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` + -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq + vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY----- + \ No newline at end of file diff --git a/tests/testData/input/secret_at_end.txt b/tests/testData/input/secret_at_end.txt new file mode 100644 index 00000000..6a155497 --- /dev/null +++ b/tests/testData/input/secret_at_end.txt @@ -0,0 +1,2 @@ +`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, + `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` \ No newline at end of file diff --git a/tests/testData/input/secret_at_end_with_newline.txt b/tests/testData/input/secret_at_end_with_newline.txt new file mode 100644 index 00000000..81b86001 --- /dev/null +++ b/tests/testData/input/secret_at_end_with_newline.txt @@ -0,0 +1,3 @@ +`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, + `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` + \ No newline at end of file From d069b615e2dcd3241643ad7b104f3a487fb31095 Mon Sep 17 00:00:00 2001 From: cx-leonardo-fontes <204389152+cx-leonardo-fontes@users.noreply.github.com> Date: Wed, 21 May 2025 15:56:05 +0100 Subject: [PATCH 2/6] update checkmarx scan workflow --- .github/workflows/ast-scan.yml | 25 ------------------------- .github/workflows/cx-one-scan.yaml | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/ast-scan.yml create mode 100644 .github/workflows/cx-one-scan.yaml diff --git a/.github/workflows/ast-scan.yml b/.github/workflows/ast-scan.yml deleted file mode 100644 index 805a6348..00000000 --- a/.github/workflows/ast-scan.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Checkmarx One Scan - -on: - workflow_dispatch: - pull_request: - push: - branches: - - main - schedule: - - cron: '00 7 * * *' - -jobs: - cx-scan: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Checkmarx One CLI Action - uses: checkmarx/ast-github-action@dd0f9365942f29a99c3be5bdb308958ede8f906b # v.2.0.25 - with: - base_uri: ${{ secrets.AST_RND_SCANS_BASE_URI }} - cx_tenant: ${{ secrets.AST_RND_SCANS_TENANT }} - cx_client_id: ${{ secrets.AST_RND_SCANS_CLIENT_ID }} - cx_client_secret: ${{ secrets.AST_RND_SCANS_CLIENT_SECRET }} - additional_params: --tags scs --threshold "sast-high=1; sast-medium=1; sast-low=1; sca-high=1; sca-medium=1; iac-security-high=1; iac-security-medium=1; iac-security-low=1" diff --git a/.github/workflows/cx-one-scan.yaml b/.github/workflows/cx-one-scan.yaml new file mode 100644 index 00000000..926337b8 --- /dev/null +++ b/.github/workflows/cx-one-scan.yaml @@ -0,0 +1,26 @@ +name: cx-one-scan + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + schedule: + - cron: '00 7 * * *' + +jobs: + cx-one-scan: + name: cx-one-scan + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Checkmarx One CLI Action + uses: checkmarx/ast-github-action@86e9ae570a811f9a1fb85903647a307aa3bf6253 # 2.0.44 + with: + base_uri: ${{ secrets.AST_RND_SCANS_BASE_URI }} + cx_tenant: ${{ secrets.AST_RND_SCANS_TENANT }} + cx_client_id: ${{ secrets.AST_RND_SCANS_CLIENT_ID }} + cx_client_secret: ${{ secrets.AST_RND_SCANS_CLIENT_SECRET }} + additional_params: --tags scs --threshold "sast-critical=1; sast-high=1; sast-medium=1; sast-low=1; sca-critical=1; sca-high=1; sca-medium=1; sca-low=1; iac-security-critical=1; iac-security-high=1; iac-security-medium=1;iac-security-low=1" From 38b33108b5e624777b29389a2144b7471f9d8d1b Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Wed, 21 May 2025 16:39:33 +0100 Subject: [PATCH 3/6] fix: new line in fragment and e2e tests --- .2ms.yml | 27 ++ cmd/workers_test.go | 4 + engine/engine.go | 19 +- engine/engine_test.go | 4 + go.mod | 28 +- go.sum | 62 ++--- lib/reporting/report_test.go | 6 +- lib/reporting/sarif.go | 12 +- lib/reporting/sarif_test.go | 53 ++++ lib/utils/test_utils.go | 27 ++ pkg/scan.go | 17 +- pkg/scan_test.go | 68 ++--- pkg/scanner.go | 4 + pkg/testData/expectedReport.json | 4 +- plugins/git.go | 162 ++++++++++-- plugins/git_test.go | 248 ++++++++++++++++++ plugins/plugins.go | 8 +- tests/e2e.go | 7 +- tests/e2e_test.go | 58 ++-- ...rt2.json => multi_line_secret_report.json} | 4 +- tests/testData/expectedReport/report1.json | 36 --- .../expectedReport/secret_at_end_report.json | 36 +++ ...=> secret_at_end_with_newline_report.json} | 0 .../input/secret_at_end_with_newline.txt | 1 - 24 files changed, 682 insertions(+), 213 deletions(-) create mode 100644 lib/reporting/sarif_test.go create mode 100644 lib/utils/test_utils.go rename tests/testData/expectedReport/{report2.json => multi_line_secret_report.json} (96%) delete mode 100644 tests/testData/expectedReport/report1.json create mode 100644 tests/testData/expectedReport/secret_at_end_report.json rename tests/testData/expectedReport/{report3.json => secret_at_end_with_newline_report.json} (100%) diff --git a/.2ms.yml b/.2ms.yml index 2c13a979..ac9ac59b 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -51,3 +51,30 @@ ignore-result: - 51a6f4e3c7e3a79c9722abb7541b4902098e526b # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 - 53803ee7e880952e926898a434acff4483fec67e # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 - aa52405f239a8be1284d933025c557b071b24036 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 + - 61a50a3d783926ae08307cc9727e9b1830f4044d # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - b8fddbf33e0da0db4714425e2baedbc74865b72e # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 9d88a51fcfe0bba421e3ab285c0bcd5884889520 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - ad5cd04241f630992be8c34e2626d2372dbd7690 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 0648cbaed8d23cd128f7e9111b51d739d1f5769b # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 27ba3f4fed916199f4f65f30ffc111b8ee3dc3db # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 52ab4ec04145a57835d9ee91380c8a559b34706e # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 35a133edb564767157c6bd807f57009a9ee78349 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 0b43a67f6eb1f2d1b744b5813eec4eb9f167023d # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - ba04dd95db7fd550ebb0f295d80fce4e281529fb # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 35a133edb564767157c6bd807f57009a9ee78349 # value used for testing, found at https://github.com/Checkmarx/2ms/pull/288/commits/2cdf66865f2bdf006869b8a84f448bec3525bfa0 + - 854547fc6e35c0d1f63c0f4d426aebd4d64679fc # False positive, see https://github.com/gitleaks/gitleaks/pull/1358, found at https://github.com/Checkmarx/2ms/commit/45a5c9d35ff910dfec5e5a76cdedb8977da5dd34#diff-d712d2256df359061d691b711ca7ed30ba408199b1e3801cef289779778d8bad + - ae0f7e65c291d7f0ea998dfa77485bfc632e5d62 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 99f9ffb901cb72a0282ce32cf7dc050e5225cd81 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - bdd20706ea03aa38c8c9f3f87200cf6ab9010a53 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 1bd84965941175ee61639964adbff6170bea7703 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - f86543794ab8c77a54adc91581dcf72bfef6bf78 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 0f80a32cc85ea5c04b65dbf7d6db6ddb8c2e4d29 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 29a593e19a06c138d63468b8a028696ccdfc7eb2 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 8149f62cd847f3c4ba5ffc502bdcb8d66e800c7f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - e3b354d102fe73cd4f4016e1ee17e468256d2ae8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 5c2e640a480ca64c809133e1b157fd97960356bf # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 92b1996f9815a2fbd9299a1997ce0bc2c153624f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 7b7c1a0b1c5760490d843e0b9bfe540665d20b28 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - c9ae034a5a03a540d50a2686f74fcbb5117f181c # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - b7c3ac03d8a24892a2c4be5810ce73ffdf6ba3ae # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - f40881f8369f0d90670fc22a719ecd0ba9cb2f02 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 diff --git a/cmd/workers_test.go b/cmd/workers_test.go index 8152ae20..f3f35129 100644 --- a/cmd/workers_test.go +++ b/cmd/workers_test.go @@ -30,6 +30,10 @@ func (i *mockItem) GetSource() string { return i.source } +func (i *mockItem) GetGitInfo() *plugins.GitInfo { + return nil +} + func TestProcessItems(t *testing.T) { totalItemsToProcess := 5 engineConfig := engine.EngineConfig{} diff --git a/engine/engine.go b/engine/engine.go index 721f15ee..1d2c80d2 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -81,23 +81,38 @@ func Init(engineConfig EngineConfig) (*Engine, error) { func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) { defer wg.Done() + const CxFileEndMarker = ";cx-file-end" fragment := detect.Fragment{ Raw: *item.GetContent(), FilePath: item.GetSource(), } - fragment.Raw += "\n" - for _, value := range e.detector.Detect(fragment) { + fragment.Raw += CxFileEndMarker + "\n" + gitInfo := item.GetGitInfo() + + values := e.detector.Detect(fragment) + + for _, value := range values { itemId := getFindingId(item, value) var startLine, endLine int + var err error if pluginName == "filesystem" { startLine = value.StartLine + 1 endLine = value.EndLine + 1 + } else if pluginName == "git" { + startLine, endLine, err = plugins.GetGitStartAndEndLine(gitInfo, value.StartLine, value.EndLine) + if err != nil { + errors <- fmt.Errorf("failed to get git lines for source %s: %w", item.GetSource(), err) + return + } } else { startLine = value.StartLine endLine = value.EndLine } + + value.Line = strings.TrimSuffix(value.Line, CxFileEndMarker) + lineContent, err := linecontent.GetLineContent(value.Line, value.Secret) if err != nil { errors <- fmt.Errorf("failed to get line content for source %s: %w", item.GetSource(), err) diff --git a/engine/engine_test.go b/engine/engine_test.go index d31fa39e..8fb67a96 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -195,3 +195,7 @@ func (i item) GetSource() string { } return "test" } + +func (i item) GetGitInfo() *plugins.GitInfo { + return nil +} diff --git a/go.mod b/go.mod index 35524222..046f0ef1 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,12 @@ go 1.23.6 require ( github.com/bwmarrin/discordgo v0.27.1 github.com/gitleaks/go-gitdiff v0.9.0 - github.com/google/go-cmp v0.6.0 github.com/rs/zerolog v1.32.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.8.0 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e - github.com/stretchr/testify v1.9.0 + github.com/spf13/pflag v1.0.6 + github.com/spf13/viper v1.20.1 + github.com/stretchr/testify v1.10.0 github.com/zricethezav/gitleaks/v8 v8.18.2 golang.org/x/time v0.5.0 gopkg.in/yaml.v3 v3.0.1 @@ -23,36 +22,31 @@ require ( github.com/charmbracelet/lipgloss v0.7.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/semgroup v1.2.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/h2non/filetype v1.1.3 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect - github.com/pelletier/go-toml/v2 v2.2.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/cast v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.35.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/sync v0.11.0 // indirect + golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/text v0.23.0 // indirect ) //transitive dependency not applied but also reported as not used by go mod why diff --git a/go.sum b/go.sum index b79fd945..8e77827a 100644 --- a/go.sum +++ b/go.sum @@ -8,7 +8,6 @@ github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZ github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -16,12 +15,14 @@ github.com/fatih/semgroup v1.2.0 h1:h/OLXwEM+3NNyAdZEpMiH1OzfplU09i2qXPVThGZvyg= github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS879kgm8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg= github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -31,8 +32,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -43,8 +42,6 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb h1:w1g9wNDIE/pHSTmAaUhv4TZQuPBS6GV3mMz5hkgziIU= github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -54,14 +51,12 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= -github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= -github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -76,34 +71,26 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= +github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ= github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e h1:NghP/tGibNy35kVZJjwUR8HjoHOW1TUucmO/DUikdRs= -github.com/spf13/viper v1.18.2-0.20240419203757-d539b7a2462e/go.mod h1:Hqr8J4/Q1O00v/4zIIggDIidAoD4w8Oqtzc+Ew8QO+I= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -116,8 +103,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -129,8 +114,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -148,8 +133,8 @@ golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -162,8 +147,5 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/lib/reporting/report_test.go b/lib/reporting/report_test.go index 7a106487..38a8fd7e 100644 --- a/lib/reporting/report_test.go +++ b/lib/reporting/report_test.go @@ -84,7 +84,7 @@ var ( // sarif results result1Sarif = Results{ Message: Message{ - Text: messageText(result1.RuleID, result1.Source), + Text: createMessageText(result1.RuleID, result1.Source), }, RuleId: ruleID1, Locations: []Locations{ @@ -115,7 +115,7 @@ var ( } result2Sarif = Results{ Message: Message{ - Text: messageText(result2.RuleID, result2.Source), + Text: createMessageText(result2.RuleID, result2.Source), }, RuleId: ruleID2, Locations: []Locations{ @@ -146,7 +146,7 @@ var ( } result3Sarif = Results{ Message: Message{ - Text: messageText(result3.RuleID, result3.Source), + Text: createMessageText(result3.RuleID, result3.Source), }, RuleId: ruleID1, Locations: []Locations{ diff --git a/lib/reporting/sarif.go b/lib/reporting/sarif.go index 261e23ef..1a9b8025 100644 --- a/lib/reporting/sarif.go +++ b/lib/reporting/sarif.go @@ -68,7 +68,15 @@ func hasNoResults(report *Report) bool { return len(report.Results) == 0 } -func messageText(ruleName string, filePath string) string { +func createMessageText(ruleName string, filePath string) string { + // maintain only the filename if the scan target is git + if strings.HasPrefix(filePath, "git show ") { + filePathParts := strings.SplitN(filePath, ":", 2) + if len(filePathParts) == 2 { + filePath = filePathParts[1] + } + } + return fmt.Sprintf("%s has detected secret for file %s.", ruleName, filePath) } @@ -85,7 +93,7 @@ func getResults(report *Report) []Results { for _, secret := range secrets { r := Results{ Message: Message{ - Text: messageText(secret.RuleID, secret.Source), + Text: createMessageText(secret.RuleID, secret.Source), }, RuleId: secret.RuleID, Locations: getLocation(secret), diff --git a/lib/reporting/sarif_test.go b/lib/reporting/sarif_test.go new file mode 100644 index 00000000..3b903810 --- /dev/null +++ b/lib/reporting/sarif_test.go @@ -0,0 +1,53 @@ +package reporting + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreateMessageText(t *testing.T) { + ruleName := "Test Rule" + messagePrefix := ruleName + " has detected secret for file %s." + + tests := []struct { + Name string + FilePath string + ExpectedMessage string + }{ + { + Name: "Filesystem file name", + FilePath: "folder/filename.txt", + ExpectedMessage: fmt.Sprintf(messagePrefix, "folder/filename.txt"), + }, + { + Name: "Simple git filename", + FilePath: "git show 1a9f3c87b4d029f54e8c72d8b11a78f6a3c29d2e:folder/filename.txt", + ExpectedMessage: fmt.Sprintf(messagePrefix, "folder/filename.txt"), + }, + { + Name: "Broken git file name with no commit hash", + FilePath: "git show folder/filename.txt", + ExpectedMessage: fmt.Sprintf(messagePrefix, "git show folder/filename.txt"), + }, + { + Name: "Git file name with one colon character", + FilePath: "git show d8e914f06d8d4494bd4f9ab2a2c9c88f78ef25ad:folder/filename:secondpart.txt", + ExpectedMessage: fmt.Sprintf(messagePrefix, "folder/filename:secondpart.txt"), + }, + { + Name: "Git file name with multiple colon character", + FilePath: "git show a73b5cf94f0b29e1cc6e71a092f6b8ebc1d0e002:folder:secondfolderpart/filename:secondpart.txt", + ExpectedMessage: fmt.Sprintf(messagePrefix, "folder:secondfolderpart/filename:secondpart.txt"), + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + message := createMessageText(ruleName, tt.FilePath) + fmt.Printf("%v", message) + assert.Equal(t, tt.ExpectedMessage, message) + }) + } +} diff --git a/lib/utils/test_utils.go b/lib/utils/test_utils.go new file mode 100644 index 00000000..94e52286 --- /dev/null +++ b/lib/utils/test_utils.go @@ -0,0 +1,27 @@ +package utils + +import ( + "encoding/json" + "fmt" + "strings" +) + +// normalizeReportData recursively traverses the report data and removes any carriage return characters. +func NormalizeReportData(data interface{}) (interface{}, error) { + bytes, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("failed to marshal data: %w", err) + } + + jsonStr := string(bytes) + jsonStr = strings.ReplaceAll(jsonStr, "\\r", "") + + // Unmarshal back to a Go data structure + var result interface{} + err = json.Unmarshal([]byte(jsonStr), &result) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data: %w", err) + } + + return result, nil +} diff --git a/pkg/scan.go b/pkg/scan.go index 9300382e..27899eb1 100644 --- a/pkg/scan.go +++ b/pkg/scan.go @@ -91,7 +91,6 @@ func (s *scanner) Scan(scanItems []ScanItem, scanConfig ScanConfig) (*reporting. func (s *scanner) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (*reporting.Report, error) { itemsCh := cmd.Channels.Items errorsCh := cmd.Channels.Errors - wg := &sync.WaitGroup{} // Initialize engine configuration. engineConfig := engine.EngineConfig{IgnoredIds: scanConfig.IgnoreResultIds} @@ -113,18 +112,10 @@ func (s *scanner) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (* cmd.Channels.WaitGroup.Add(1) go cmd.ProcessScoreWithoutValidation(engineInstance) - // Forward scan items from itemsIn to itemsCh. - go func() { - for item := range itemsIn { - wg.Add(1) - go func(i ScanItem) { - defer wg.Done() - itemsCh <- i - }(item) - } - wg.Wait() - close(itemsCh) - }() + for item := range itemsIn { + itemsCh <- item + } + close(itemsCh) // Wait for all processing routines. cmd.Channels.WaitGroup.Wait() diff --git a/pkg/scan_test.go b/pkg/scan_test.go index 0d3581a7..75ebad25 100644 --- a/pkg/scan_test.go +++ b/pkg/scan_test.go @@ -4,15 +4,14 @@ import ( "encoding/json" "fmt" "os" - "strings" "sync" "testing" "github.com/checkmarx/2ms/cmd" "github.com/checkmarx/2ms/lib/reporting" "github.com/checkmarx/2ms/lib/secrets" + "github.com/checkmarx/2ms/lib/utils" "github.com/checkmarx/2ms/plugins" - "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" ) @@ -23,26 +22,6 @@ const ( expectedReportResultsIgnoredPath = "testData/expectedReportWithIgnoredResults.json" ) -// normalizeReportData recursively traverses the report data and removes any carriage return characters. -func normalizeReportData(data interface{}) interface{} { - switch v := data.(type) { - case string: - return strings.ReplaceAll(v, "\r", "") - case []interface{}: - for i, item := range v { - v[i] = normalizeReportData(item) - } - return v - case map[string]interface{}: - for key, val := range v { - v[key] = normalizeReportData(val) - } - return v - default: - return data - } -} - func TestScan(t *testing.T) { t.Run("Successful Scan with Multiple Items", func(t *testing.T) { cmd.Report = reporting.Init() @@ -101,12 +80,13 @@ func TestScan(t *testing.T) { assert.NoError(t, err, "failed to unmarshal actual report JSON") // Normalize both expected and actual maps. - expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) - actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport) + assert.NoError(t, err, "Failed to normalize actual report") - if !cmp.Equal(expectedReport, actualReportMap) { - t.Errorf("Scan report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) - } + normalizedActualReport, err := utils.NormalizeReportData(actualReportMap) + assert.NoError(t, err, "Failed to normalize actual report") + + assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport) }) t.Run("Successful scan with multiple items and ignored results", func(t *testing.T) { cmd.Report = reporting.Init() @@ -168,13 +148,13 @@ func TestScan(t *testing.T) { err = json.Unmarshal(actualReportBytes, &actualReportMap) assert.NoError(t, err, "failed to unmarshal actual report JSON") - // Normalize both expected and actual maps. - expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) - actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport) + assert.NoError(t, err, "Failed to normalize actual report") - if !cmp.Equal(expectedReport, actualReportMap) { - t.Errorf("Scan report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) - } + normalizedActualReport, err := utils.NormalizeReportData(actualReportMap) + assert.NoError(t, err, "Failed to normalize actual report") + + assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport) }) t.Run("error handling should work", func(t *testing.T) { cmd.Report = reporting.Init() @@ -309,12 +289,13 @@ func TestScanDynamic(t *testing.T) { assert.NoError(t, err, "failed to unmarshal actual report JSON") // Normalize both maps. - expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) - actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport) + assert.NoError(t, err, "Failed to normalize actual report") - if !cmp.Equal(expectedReport, actualReportMap) { - t.Errorf("ScanDynamic report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) - } + normalizedActualReport, err := utils.NormalizeReportData(actualReportMap) + assert.NoError(t, err, "Failed to normalize actual report") + + assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport) }) t.Run("Successful ScanDynamic with Multiple Items and Ignored Results", func(t *testing.T) { @@ -385,12 +366,13 @@ func TestScanDynamic(t *testing.T) { assert.NoError(t, err, "failed to unmarshal actual report JSON") // Normalize both maps. - expectedReport = normalizeReportData(expectedReport).(map[string]interface{}) - actualReportMap = normalizeReportData(actualReportMap).(map[string]interface{}) + normalizedExpectedReport, err := utils.NormalizeReportData(expectedReport) + assert.NoError(t, err, "Failed to normalize actual report") - if !cmp.Equal(expectedReport, actualReportMap) { - t.Errorf("ScanDynamic report does not match the expected report:\n%s", cmp.Diff(expectedReport, actualReportMap)) - } + normalizedActualReport, err := utils.NormalizeReportData(actualReportMap) + assert.NoError(t, err, "Failed to normalize actual report") + + assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport) }) t.Run("error handling should work", func(t *testing.T) { diff --git a/pkg/scanner.go b/pkg/scanner.go index 43452fa8..9c273781 100644 --- a/pkg/scanner.go +++ b/pkg/scanner.go @@ -27,6 +27,10 @@ func (i ScanItem) GetSource() string { return i.Source } +func (i ScanItem) GetGitInfo() *plugins.GitInfo { + return nil +} + type Scanner interface { Scan(scanItems []ScanItem, scanConfig ScanConfig) (*reporting.Report, error) ScanDynamic(itemsIn <-chan ScanItem, scanConfig ScanConfig) (*reporting.Report, error) diff --git a/pkg/testData/expectedReport.json b/pkg/testData/expectedReport.json index 092ead53..d9f646f4 100644 --- a/pkg/testData/expectedReport.json +++ b/pkg/testData/expectedReport.json @@ -39,9 +39,9 @@ "ruleId" : "jwt", "startLine" : 1, "endLine" : 1, - "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r\n Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", + "lineContent": "\n Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", "startColumn" : 64, - "endColumn" : 166, + "endColumn" : 167, "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", "extraDetails" : { diff --git a/plugins/git.go b/plugins/git.go index 62b16e79..9e0bede8 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -13,10 +13,18 @@ import ( git "github.com/zricethezav/gitleaks/v8/sources" ) +type DiffType int + +const ( + AddedContent DiffType = iota + RemovedContent +) + const ( argDepth = "depth" argScanAllBranches = "all-branches" argProjectName = "project-name" + unknownCommit = "unknown" ) type GitPlugin struct { @@ -27,6 +35,11 @@ type GitPlugin struct { projectName string } +type GitInfo struct { + Hunks []*gitdiff.TextFragment + ContentType DiffType +} + func (p *GitPlugin) GetName() string { return "git" } @@ -69,36 +82,91 @@ func (p *GitPlugin) buildScanOptions() string { } func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan ISourceItem, errChan chan error) { - diffs, close := p.readGitLog(path, scanOptions, errChan) - defer close() + diffs, wait := p.readGitLog(path, scanOptions, errChan) + defer wait() for file := range diffs { - if file.PatchHeader == nil { - // While parsing the PatchHeader, the token size limit may be exceeded, resulting in a nil value. - // This scenario is unlikely, but it causes the scan to never complete. - file.PatchHeader = &gitdiff.PatchHeader{} + p.processFileDiff(file, itemsChan) + } +} + +// processFileDiff handles processing a single diff file. +func (p *GitPlugin) processFileDiff(file *gitdiff.File, itemsChan chan ISourceItem) { + if file.PatchHeader == nil { + // When parsing the PatchHeader, the token size limit may be exceeded, resulting in a nil value + // This scenario is unlikely but may cause the scan to never complete + file.PatchHeader = &gitdiff.PatchHeader{} + file.PatchHeader.SHA = unknownCommit + } + + log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) + + // Skip binary files + if file.IsBinary { + return + } + + // Extract the changes (added and removed) from the text fragments + addedChanges, removedChanges := extractChanges(file.TextFragments) + + var fileName string + if file.IsDelete { + fileName = file.OldName + } else { + fileName = file.NewName + } + id := fmt.Sprintf("%s-%s-%s-%s", p.GetName(), p.projectName, file.PatchHeader.SHA, fileName) + source := fmt.Sprintf("git show %s:%s", file.PatchHeader.SHA, fileName) + + // If there are added changes, send an item with added content + if addedChanges != "" { + itemsChan <- item{ + Content: &addedChanges, + ID: id, + Source: source, + GitInfo: &GitInfo{ + Hunks: file.TextFragments, + ContentType: AddedContent, + }, } + } - log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) - if file.IsBinary || file.IsDelete { - continue + // If there are removed changes, send an item with removed content + if removedChanges != "" { + itemsChan <- item{ + Content: &removedChanges, + ID: id, + Source: source, + GitInfo: &GitInfo{ + Hunks: file.TextFragments, + ContentType: RemovedContent, + }, } + } +} - fileChanges := "" - for _, textFragment := range file.TextFragments { - if textFragment != nil { - raw := textFragment.Raw(gitdiff.OpAdd) - fileChanges += raw - } +// extractChanges iterates over the text fragments to compile added and removed changes +func extractChanges(fragments []*gitdiff.TextFragment) (added string, removed string) { + var addedBuilder, removedBuilder strings.Builder + + for _, tf := range fragments { + if tf == nil { + continue } - if fileChanges != "" { - itemsChan <- item{ - Content: &fileChanges, - ID: fmt.Sprintf("%s-%s-%s-%s", p.GetName(), p.projectName, file.PatchHeader.SHA, file.NewName), - Source: fmt.Sprintf("git show %s:%s", file.PatchHeader.SHA, file.NewName), + for i := range tf.Lines { + switch tf.Lines[i].Op { + case gitdiff.OpAdd: + addedBuilder.WriteString(tf.Lines[i].Line) + case gitdiff.OpDelete: + removedBuilder.WriteString(tf.Lines[i].Line) + default: } + // Clean up the line content to free memory + tf.Lines[i].Line = "" } } + + return addedBuilder.String(), removedBuilder.String() } func (p *GitPlugin) readGitLog(path string, scanOptions string, errChan chan error) (<-chan *gitdiff.File, func()) { @@ -138,3 +206,57 @@ func validGitRepoArgs(cmd *cobra.Command, args []string) error { } return nil } + +// GetGitStartAndEndLine walks the diff hunks and discover the actual start and end lines of the file +func GetGitStartAndEndLine(gitInfo *GitInfo, localStartLine, localEndLine int) (int, int, error) { + hunkPosition, hunkCount, relevantOp, err := getHunkPosAndCount(gitInfo) + if err != nil { + return 0, 0, fmt.Errorf("failed to get hunk position and count: %w", err) + } + + diffLines := 0 // Tracks how many lines have been processed in the diff + for _, hunk := range gitInfo.Hunks { + // If the hunk ends before the start line in the file diff, skip it + totalLines := hunkCount(hunk) + if diffLines+totalLines <= localStartLine { + diffLines += totalLines + continue + } + + // Get the start line of the hunk in the file diff and walk through its lines to find the actual start line + fileStartLine := hunkPosition(hunk) - 1 + for _, line := range hunk.Lines { + switch line.Op { + case relevantOp: + fileStartLine += 1 + if diffLines == localStartLine { + fileEndLine := fileStartLine + (localEndLine - localStartLine) + return fileStartLine, fileEndLine, nil + } + diffLines += 1 + case gitdiff.OpContext: // Context lines are not counted in the diff + fileStartLine += 1 + default: + } + } + } + // Did not find the start line in any hunk + return 0, 0, fmt.Errorf("failed to find start line %d in hunks", localStartLine) +} + +// getHunkPosAndCount returns the functions to get the position and count of hunks based on the content type +func getHunkPosAndCount(gitInfo *GitInfo) (hunkPos func(h *gitdiff.TextFragment) int, hunkCount func(h *gitdiff.TextFragment) int, matchOp gitdiff.LineOp, err error) { + switch gitInfo.ContentType { + case AddedContent: + hunkPos = func(h *gitdiff.TextFragment) int { return int(h.NewPosition) } + hunkCount = func(h *gitdiff.TextFragment) int { return int(h.LinesAdded) } + matchOp = gitdiff.OpAdd + case RemovedContent: + hunkPos = func(h *gitdiff.TextFragment) int { return int(h.OldPosition) } + hunkCount = func(h *gitdiff.TextFragment) int { return int(h.LinesDeleted) } + matchOp = gitdiff.OpDelete + default: + err = fmt.Errorf("unknown content type: %d", gitInfo.ContentType) + } + return +} diff --git a/plugins/git_test.go b/plugins/git_test.go index 85ac71cf..8a71aff7 100644 --- a/plugins/git_test.go +++ b/plugins/git_test.go @@ -3,6 +3,7 @@ package plugins import ( "errors" "fmt" + "github.com/gitleaks/go-gitdiff/gitdiff" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "os" @@ -164,3 +165,250 @@ func TestValidGitRepoArgs(t *testing.T) { }) } } + +func TestGetGitStartAndEndLine(t *testing.T) { + tests := []struct { + name string + gitInfo *GitInfo + localStartLine int + localEndLine int + expectedFileStartLine int + expectedFileEndLine int + }{ + { + name: "Secret in added content without context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(9, 0, 10, 3, 3, 0, nil), + createMockHunk(49, 0, 53, 3, 3, 0, nil), + createMockHunk(55, 0, 62, 2, 2, 0, nil), + createMockHunk(58, 0, 67, 1, 1, 0, nil), + createMockHunk(103, 11, 112, 1, 1, 11, nil), + }, + ContentType: AddedContent, + }, + localStartLine: 9, + localEndLine: 9, + expectedFileStartLine: 112, + expectedFileEndLine: 112, + }, + { + name: "Secret in removed content without context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(10, 2, 10, 1, 1, 2, nil), + createMockHunk(29, 0, 29, 1, 1, 0, nil), + createMockHunk(46, 8, 46, 1, 1, 8, nil), + createMockHunk(57, 2, 50, 1, 1, 2, nil), + createMockHunk(63, 2, 55, 2, 2, 2, nil), + createMockHunk(106, 0, 99, 1, 1, 0, nil), + createMockHunk(108, 8, 101, 3, 3, 8, nil), + }, + ContentType: RemovedContent, + }, + localStartLine: 18, + localEndLine: 18, + expectedFileStartLine: 112, + expectedFileEndLine: 112, + }, + { + name: "Secret in added content with context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(7, 8, 7, 7, 1, 2, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + createMockHunk(27, 6, 26, 7, 1, 0, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + }, + ContentType: AddedContent, + }, + localStartLine: 1, + localEndLine: 1, + expectedFileStartLine: 29, + expectedFileEndLine: 29, + }, + { + name: "Secret in removed content with context lines", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(475, 8, 475, 8, 2, 2, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + createMockHunk(512, 8, 512, 8, 2, 2, []gitdiff.Line{ + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpDelete, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpAdd, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + { + Op: gitdiff.OpContext, + }, + }), + }, + ContentType: RemovedContent, + }, + localStartLine: 3, + localEndLine: 3, + expectedFileStartLine: 516, + expectedFileEndLine: 516, + }, + { + name: "validate skip hunk when secret is found immediately after the hunk before in added content", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(975, 0, 976, 3, 3, 0, nil), + createMockHunk(977, 4, 980, 1, 1, 4, nil), + }, + ContentType: AddedContent, + }, + localStartLine: 3, + localEndLine: 3, + expectedFileStartLine: 980, + expectedFileEndLine: 980, + }, + { + name: "validate skip hunk when secret is found immediately after the hunk before in removed content", + gitInfo: &GitInfo{ + Hunks: []*gitdiff.TextFragment{ + createMockHunk(976, 3, 975, 0, 0, 3, nil), + createMockHunk(980, 1, 977, 4, 4, 1, nil), + }, + ContentType: RemovedContent, + }, + localStartLine: 3, + localEndLine: 3, + expectedFileStartLine: 980, + expectedFileEndLine: 980, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actualFileStartLine, actualFileEndLine, err := GetGitStartAndEndLine(tt.gitInfo, tt.localStartLine, tt.localEndLine) + if err != nil { + t.Fatalf("GetGitStartAndEndLine() error = %v", err) + } + assert.Equal(t, tt.expectedFileStartLine, actualFileStartLine) + assert.Equal(t, tt.expectedFileEndLine, actualFileEndLine) + }) + } +} + +func createMockHunk(oldPos, oldLines, newPos, newLines, linesAdded, linesDeleted int64, lines []gitdiff.Line) *gitdiff.TextFragment { + if lines == nil { + for i := int64(0); i < linesDeleted; i++ { + lines = append(lines, gitdiff.Line{Op: gitdiff.OpDelete}) + } + for i := int64(0); i < linesAdded; i++ { + lines = append(lines, gitdiff.Line{Op: gitdiff.OpAdd}) + } + } + return &gitdiff.TextFragment{ + OldPosition: oldPos, + OldLines: oldLines, + NewPosition: newPos, + NewLines: newLines, + LinesAdded: linesAdded, + LinesDeleted: linesDeleted, + Lines: lines, + } +} diff --git a/plugins/plugins.go b/plugins/plugins.go index 9777b0b2..b78ff249 100644 --- a/plugins/plugins.go +++ b/plugins/plugins.go @@ -10,6 +10,7 @@ type ISourceItem interface { GetContent() *string GetID() string GetSource() string + GetGitInfo() *GitInfo } type item struct { @@ -17,7 +18,8 @@ type item struct { // Unique identifier of the item ID string // User friendly description and/or link to the item - Source string + Source string + GitInfo *GitInfo } var _ ISourceItem = (*item)(nil) @@ -34,6 +36,10 @@ func (i item) GetSource() string { return i.Source } +func (i item) GetGitInfo() *GitInfo { + return i.GitInfo +} + type Plugin struct { ID string Limit chan struct{} diff --git a/tests/e2e.go b/tests/e2e.go index 28dbf7b1..7af45f1f 100644 --- a/tests/e2e.go +++ b/tests/e2e.go @@ -9,7 +9,6 @@ import ( "os" "os/exec" "path" - "path/filepath" "runtime" "github.com/checkmarx/2ms/lib/reporting" @@ -21,11 +20,7 @@ type cli struct { } func createCLI(outputDir string) (cli, error) { - executableName := "2ms" - if runtime.GOOS == "windows" { - executableName += ".exe" - } - executable := filepath.Join(outputDir, executableName) + executable := path.Join(outputDir, "2ms") lib, err := build.Import("github.com/checkmarx/2ms", "", build.FindOnly) if err != nil { return cli{}, fmt.Errorf("failed to import 2ms: %s", err) diff --git a/tests/e2e_test.go b/tests/e2e_test.go index f7f08691..ee8d14ec 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -5,8 +5,10 @@ import ( "os" "testing" - "github.com/checkmarx/2ms/lib/reporting" - "github.com/google/go-cmp/cmp" + "github.com/checkmarx/2ms/lib/utils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestIntegration(t *testing.T) { @@ -87,7 +89,7 @@ func TestIntegration(t *testing.T) { }) } -func TestSecretsEdgeCases(t *testing.T) { +func TestSecretsScans(t *testing.T) { if testing.Short() { t.Skip("skipping edge cases test") } @@ -99,31 +101,29 @@ func TestSecretsEdgeCases(t *testing.T) { ExpectedReportPath string }{ { - Name: "secret at end without newline (filesystem)", + Name: "secret at end without newline", ScanTarget: "filesystem", TargetPath: "testData/input/secret_at_end.txt", - ExpectedReportPath: "testData/expectedReport/report1.json", + ExpectedReportPath: "testData/expectedReport/secret_at_end_report.json", }, { - Name: "secret at end with multiLine (filesystem)", + Name: "multi line secret ", ScanTarget: "filesystem", TargetPath: "testData/input/multi_line_secret.txt", - ExpectedReportPath: "testData/expectedReport/report2.json", + ExpectedReportPath: "testData/expectedReport/multi_line_secret_report.json", }, { - Name: "secret at end with backspace in newline (filesystem)", + Name: "secret at end with newline ", ScanTarget: "filesystem", TargetPath: "testData/input/secret_at_end_with_newline.txt", - ExpectedReportPath: "testData/expectedReport/report3.json", + ExpectedReportPath: "testData/expectedReport/secret_at_end_with_newline_report.json", }, } for _, tc := range tests { t.Run(tc.Name, func(t *testing.T) { executable, err := createCLI(t.TempDir()) - if err != nil { - t.Fatalf("failed to build CLI: %s", err) - } + require.Nil(t, err, "failed to build CLI") args := []string{tc.ScanTarget} if tc.ScanTarget == "filesystem" { @@ -138,22 +138,30 @@ func TestSecretsEdgeCases(t *testing.T) { } actualReport, err := executable.getReport() - if err != nil { - t.Fatalf("failed to get report: %s", err) - } + require.NoError(t, err, "failed to get report") expectedBytes, err := os.ReadFile(tc.ExpectedReportPath) - if err != nil { - t.Fatalf("failed to read expected report: %s", err) - } - var expectedReport reporting.Report - if err := json.Unmarshal(expectedBytes, &expectedReport); err != nil { - t.Fatalf("failed to unmarshal expected report: %s", err) - } + assert.NoError(t, err, "failed to read expected report") - if !cmp.Equal(expectedReport, actualReport) { - t.Errorf("Scan report does not match expected report:\n%s", cmp.Diff(expectedReport, actualReport)) - } + var expectedReportMap map[string]interface{} + err = json.Unmarshal(expectedBytes, &expectedReportMap) + assert.NoError(t, err, "failed to unmarshal expected report JSON") + + actualReportBytes, err := json.Marshal(actualReport) + assert.NoError(t, err, "failed to marshal actual report to JSON") + + var actualReportMap map[string]interface{} + + err = json.Unmarshal(actualReportBytes, &actualReportMap) + assert.NoError(t, err, "failed to unmarshal actual report JSON") + + normalizedExpectedReport, err := utils.NormalizeReportData(expectedReportMap) + assert.NoError(t, err, "Failed to normalize expected report") + + normalizedActualReport, err := utils.NormalizeReportData(actualReportMap) + assert.NoError(t, err, "Failed to normalize expected report") + + assert.EqualValues(t, normalizedExpectedReport, normalizedActualReport) }) } } diff --git a/tests/testData/expectedReport/report2.json b/tests/testData/expectedReport/multi_line_secret_report.json similarity index 96% rename from tests/testData/expectedReport/report2.json rename to tests/testData/expectedReport/multi_line_secret_report.json index c2b54fee..2b195e52 100644 --- a/tests/testData/expectedReport/report2.json +++ b/tests/testData/expectedReport/multi_line_secret_report.json @@ -2,9 +2,9 @@ "totalItemsScanned": 1, "totalSecretsFound": 3, "results": { - "1ef1c6a736cb0725175ac969776c3fe0b4602389": [ + "047d26912b890e89c7f01b7ec9e926390224e4f0": [ { - "id": "1ef1c6a736cb0725175ac969776c3fe0b4602389", + "id": "047d26912b890e89c7f01b7ec9e926390224e4f0", "source": "testData/input/multi_line_secret.txt", "ruleId": "private-key", "startLine": 3, diff --git a/tests/testData/expectedReport/report1.json b/tests/testData/expectedReport/report1.json deleted file mode 100644 index 9ddc1c42..00000000 --- a/tests/testData/expectedReport/report1.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "totalItemsScanned": 1, - "totalSecretsFound": 2, - "results": { - "6a3e642795e27b989c54ac0c91147fe8e9a405b4": [ - { - "id": "6a3e642795e27b989c54ac0c91147fe8e9a405b4", - "source": "testData/input/secret_at_end.txt", - "ruleId": "generic-api-key", - "startLine": 2, - "endLine": 2, - "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`", - "startColumn": 6, - "endColumn": 88, - "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", - "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", - "cvssScore": 8.2 - } - ], - "84bc054139c2363b37538209055a2d9c23026fab": [ - { - "id": "84bc054139c2363b37538209055a2d9c23026fab", - "source": "testData/input/secret_at_end.txt", - "ruleId": "generic-api-key", - "startLine": 1, - "endLine": 1, - "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r", - "startColumn": 3, - "endColumn": 81, - "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506", - "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", - "cvssScore": 8.2 - } - ] - } - } \ No newline at end of file diff --git a/tests/testData/expectedReport/secret_at_end_report.json b/tests/testData/expectedReport/secret_at_end_report.json new file mode 100644 index 00000000..083229a1 --- /dev/null +++ b/tests/testData/expectedReport/secret_at_end_report.json @@ -0,0 +1,36 @@ +{ + "totalItemsScanned": 1, + "totalSecretsFound": 2, + "results": { + "6a3e642795e27b989c54ac0c91147fe8e9a405b4": [ + { + "id": "6a3e642795e27b989c54ac0c91147fe8e9a405b4", + "source": "testData/input/secret_at_end.txt", + "ruleId": "generic-api-key", + "startLine": 2, + "endLine": 2, + "lineContent": "\n\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`", + "startColumn": 6, + "endColumn": 88, + "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ], + "84bc054139c2363b37538209055a2d9c23026fab": [ + { + "id": "84bc054139c2363b37538209055a2d9c23026fab", + "source": "testData/input/secret_at_end.txt", + "ruleId": "generic-api-key", + "startLine": 1, + "endLine": 1, + "lineContent": "`\"client_id\" : \"0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506\"`,\r", + "startColumn": 3, + "endColumn": 81, + "value": "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506", + "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "cvssScore": 8.2 + } + ] + } +} \ No newline at end of file diff --git a/tests/testData/expectedReport/report3.json b/tests/testData/expectedReport/secret_at_end_with_newline_report.json similarity index 100% rename from tests/testData/expectedReport/report3.json rename to tests/testData/expectedReport/secret_at_end_with_newline_report.json diff --git a/tests/testData/input/secret_at_end_with_newline.txt b/tests/testData/input/secret_at_end_with_newline.txt index 81b86001..e9ee9345 100644 --- a/tests/testData/input/secret_at_end_with_newline.txt +++ b/tests/testData/input/secret_at_end_with_newline.txt @@ -1,3 +1,2 @@ `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` - \ No newline at end of file From 2e719b56ef352d79aee3438370a4e2758bf7d77e Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Wed, 21 May 2025 17:00:47 +0100 Subject: [PATCH 4/6] chore: ignore secrets to run secret scanning --- .2ms.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.2ms.yml b/.2ms.yml index d3b36d51..904dd016 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -85,3 +85,5 @@ ignore-result: - 92b1996f9815a2fbd9299a1997ce0bc2c153624f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - bf2e01278453a987f05b69e6c536358cab343322 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - c9ae034a5a03a540d50a2686f74fcbb5117f181c # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - f9e5e0b35a39914c67ee1660191a356d3c7ab1db # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 777f3d460d69a70e2ce760ca757b18f2aa984392 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 From 34aa9982ef28ee6ac6be0fa05c660d8fc8e8cf32 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Wed, 21 May 2025 17:02:16 +0100 Subject: [PATCH 5/6] chore: ignore false positives --- .2ms.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.2ms.yml b/.2ms.yml index 904dd016..fa5c5b40 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -87,3 +87,5 @@ ignore-result: - c9ae034a5a03a540d50a2686f74fcbb5117f181c # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - f9e5e0b35a39914c67ee1660191a356d3c7ab1db # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - 777f3d460d69a70e2ce760ca757b18f2aa984392 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 8f0e0442b01c18b02cfb8e59555103f8233fc7bf # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - e392318c730d4cd0a04340f1e3d41d4c61f6eb20 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 From fb8450dfcccccbec1076f39fe96f95eb4795f650 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Wed, 21 May 2025 17:32:59 +0100 Subject: [PATCH 6/6] chore: change results --- .2ms.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.2ms.yml b/.2ms.yml index fa5c5b40..7d346dc3 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -87,5 +87,5 @@ ignore-result: - c9ae034a5a03a540d50a2686f74fcbb5117f181c # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - f9e5e0b35a39914c67ee1660191a356d3c7ab1db # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - 777f3d460d69a70e2ce760ca757b18f2aa984392 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - - 8f0e0442b01c18b02cfb8e59555103f8233fc7bf # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - e392318c730d4cd0a04340f1e3d41d4c61f6eb20 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 8f0e0442b01c18b02cfb8e59555103f8233fc7bf # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 \ No newline at end of file