From 2e9dfd645f41fac45a86af70f61e30d4b1438467 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Thu, 29 May 2025 16:08:49 +0100 Subject: [PATCH 1/5] fix: github-app-toke and Clojar not being caught --- .2ms.yml | 3 +- engine/rules/clojars.go | 24 +++++++++++ engine/rules/github.go | 89 +++++++++++++++++++++++++++++++++++++++++ engine/rules/rules.go | 4 +- 4 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 engine/rules/clojars.go create mode 100644 engine/rules/github.go diff --git a/.2ms.yml b/.2ms.yml index 7d346dc3..d643fc01 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -88,4 +88,5 @@ ignore-result: - f9e5e0b35a39914c67ee1660191a356d3c7ab1db # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - 777f3d460d69a70e2ce760ca757b18f2aa984392 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - e392318c730d4cd0a04340f1e3d41d4c61f6eb20 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 - - 8f0e0442b01c18b02cfb8e59555103f8233fc7bf # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 \ No newline at end of file + - 8f0e0442b01c18b02cfb8e59555103f8233fc7bf # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10 + - 353627158f2e7fa5bb60271cee17da80e5fbba17 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/292/commits/cc44c8f8bee92250bdcd69bf9fbaffabf0eb442a \ No newline at end of file diff --git a/engine/rules/clojars.go b/engine/rules/clojars.go new file mode 100644 index 00000000..dde57ef1 --- /dev/null +++ b/engine/rules/clojars.go @@ -0,0 +1,24 @@ +package rules + +import ( + "regexp" + + "github.com/zricethezav/gitleaks/v8/cmd/generate/secrets" + "github.com/zricethezav/gitleaks/v8/config" +) + +func Clojars() *config.Rule { + // define rule + r := config.Rule{ + Description: "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation.", + RuleID: "clojars-api-token", + Regex: regexp.MustCompile(`(?i)(CLOJARS_)([a-z0-9]{60})`), + Keywords: []string{"clojars"}, + } + + // validate + tps := []string{ + generateSampleSecret("clojars", "CLOJARS_"+secrets.NewSecret(alphaNumeric("60"))), + } + return validate(r, tps, nil) +} diff --git a/engine/rules/github.go b/engine/rules/github.go new file mode 100644 index 00000000..d408d996 --- /dev/null +++ b/engine/rules/github.go @@ -0,0 +1,89 @@ +package rules + +import ( + "regexp" + + "github.com/zricethezav/gitleaks/v8/cmd/generate/secrets" + "github.com/zricethezav/gitleaks/v8/config" +) + +func GitHubPat() *config.Rule { + // define rule + r := config.Rule{ + Description: "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", + RuleID: "github-pat", + Regex: regexp.MustCompile(`ghp_[0-9a-zA-Z]{36}`), + Keywords: []string{"ghp_"}, + } + + // validate + tps := []string{ + generateSampleSecret("github", "ghp_"+secrets.NewSecret(alphaNumeric("36"))), + } + return validate(r, tps, nil) +} + +func GitHubFineGrainedPat() *config.Rule { + // define rule + r := config.Rule{ + Description: "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation.", + RuleID: "github-fine-grained-pat", + Regex: regexp.MustCompile(`github_pat_[0-9a-zA-Z_]{82}`), + Keywords: []string{"github_pat_"}, + } + + // validate + tps := []string{ + generateSampleSecret("github", "github_pat_"+secrets.NewSecret(alphaNumeric("82"))), + } + return validate(r, tps, nil) +} + +func GitHubOauth() *config.Rule { + // define rule + r := config.Rule{ + Description: "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks.", + RuleID: "github-oauth", + Regex: regexp.MustCompile(`gho_[0-9a-zA-Z]{36}`), + Keywords: []string{"gho_"}, + } + + // validate + tps := []string{ + generateSampleSecret("github", "gho_"+secrets.NewSecret(alphaNumeric("36"))), + } + return validate(r, tps, nil) +} + +func GitHubApp() *config.Rule { + // define rule + r := config.Rule{ + Description: "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security.", + RuleID: "github-app-token", + Regex: regexp.MustCompile(`ghu_[0-9a-zA-Z]{36}|ghs_[0-9a-zA-Z]{36}`), + Keywords: []string{"ghu_", "ghs_"}, + } + + // validate + tps := []string{ + generateSampleSecret("github", "ghu_"+secrets.NewSecret(alphaNumeric("36"))), + generateSampleSecret("github", "ghs_"+secrets.NewSecret(alphaNumeric("36"))), + } + return validate(r, tps, nil) +} + +func GitHubRefresh() *config.Rule { + // define rule + r := config.Rule{ + Description: "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services.", + RuleID: "github-refresh-token", + Regex: regexp.MustCompile(`ghr_[0-9a-zA-Z]{36}`), + Keywords: []string{"ghr_"}, + } + + // validate + tps := []string{ + generateSampleSecret("github", "ghr_"+secrets.NewSecret(alphaNumeric("36"))), + } + return validate(r, tps, nil) +} diff --git a/engine/rules/rules.go b/engine/rules/rules.go index f53ac610..c2eb08ee 100644 --- a/engine/rules/rules.go +++ b/engine/rules/rules.go @@ -92,7 +92,7 @@ func getDefaultRules() *[]Rule { {Rule: *rules.Beamer(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryNewsAndMedia, RuleType: 4}}, {Rule: *rules.CodecovAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategorySecurity, RuleType: 4}}, {Rule: *rules.CoinbaseAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryCryptocurrencyExchange, RuleType: 4}}, - {Rule: *rules.Clojars(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryPackageManagement, RuleType: 4}}, + {Rule: *Clojars(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryPackageManagement, RuleType: 4}}, {Rule: *rules.ConfluentAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategorySocialMedia, RuleType: 4}}, {Rule: *rules.ConfluentSecretKey(), Tags: []string{TagSecretKey}, ScoreParameters: ScoreParameters{Category: CategorySocialMedia, RuleType: 4}}, {Rule: *rules.Contentful(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryContentManagementSystem, RuleType: 4}}, @@ -131,7 +131,7 @@ func getDefaultRules() *[]Rule { {Rule: *rules.GitHubPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryDevelopmentPlatform, RuleType: 4}}, {Rule: *rules.GitHubFineGrainedPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}}, {Rule: *rules.GitHubOauth(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAuthenticationAndAuthorization, RuleType: 4}}, - {Rule: *rules.GitHubApp(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryCICD, RuleType: 4}}, + {Rule: *GitHubApp(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryCICD, RuleType: 4}}, {Rule: *rules.GitHubRefresh(), Tags: []string{TagRefreshToken}, ScoreParameters: ScoreParameters{Category: CategoryAuthenticationAndAuthorization, RuleType: 4}}, {Rule: *rules.GitlabPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategorySourceCodeManagement, RuleType: 4}}, {Rule: *rules.GitlabPipelineTriggerToken(), Tags: []string{TagTriggerToken}, ScoreParameters: ScoreParameters{Category: CategoryCICD, RuleType: 4}}, From 28614f3e79cbbb5a7b4a84267c997bdbda336a79 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Thu, 29 May 2025 16:24:20 +0100 Subject: [PATCH 2/5] fix: change just githubApp to our side --- engine/rules/github.go | 64 --------------------------------------- engine/rules/rule_test.go | 4 ++- 2 files changed, 3 insertions(+), 65 deletions(-) diff --git a/engine/rules/github.go b/engine/rules/github.go index d408d996..94cf7d75 100644 --- a/engine/rules/github.go +++ b/engine/rules/github.go @@ -7,54 +7,6 @@ import ( "github.com/zricethezav/gitleaks/v8/config" ) -func GitHubPat() *config.Rule { - // define rule - r := config.Rule{ - Description: "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", - RuleID: "github-pat", - Regex: regexp.MustCompile(`ghp_[0-9a-zA-Z]{36}`), - Keywords: []string{"ghp_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "ghp_"+secrets.NewSecret(alphaNumeric("36"))), - } - return validate(r, tps, nil) -} - -func GitHubFineGrainedPat() *config.Rule { - // define rule - r := config.Rule{ - Description: "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation.", - RuleID: "github-fine-grained-pat", - Regex: regexp.MustCompile(`github_pat_[0-9a-zA-Z_]{82}`), - Keywords: []string{"github_pat_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "github_pat_"+secrets.NewSecret(alphaNumeric("82"))), - } - return validate(r, tps, nil) -} - -func GitHubOauth() *config.Rule { - // define rule - r := config.Rule{ - Description: "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks.", - RuleID: "github-oauth", - Regex: regexp.MustCompile(`gho_[0-9a-zA-Z]{36}`), - Keywords: []string{"gho_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "gho_"+secrets.NewSecret(alphaNumeric("36"))), - } - return validate(r, tps, nil) -} - func GitHubApp() *config.Rule { // define rule r := config.Rule{ @@ -71,19 +23,3 @@ func GitHubApp() *config.Rule { } return validate(r, tps, nil) } - -func GitHubRefresh() *config.Rule { - // define rule - r := config.Rule{ - Description: "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services.", - RuleID: "github-refresh-token", - Regex: regexp.MustCompile(`ghr_[0-9a-zA-Z]{36}`), - Keywords: []string{"ghr_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "ghr_"+secrets.NewSecret(alphaNumeric("36"))), - } - return validate(r, tps, nil) -} diff --git a/engine/rules/rule_test.go b/engine/rules/rule_test.go index 6ed8330c..d3e6f012 100644 --- a/engine/rules/rule_test.go +++ b/engine/rules/rule_test.go @@ -15,10 +15,12 @@ func Test2msRules(t *testing.T) { validate func() *config.Rule }{ {name: "AuthenticatedURL", validate: rules.AuthenticatedURL}, + {name: "Clojars", validate: rules.Clojars}, + {name: "GenericCredential", validate: rules.GenericCredential}, + {name: "GitHubApp", validate: rules.GitHubApp}, {name: "HardcodedPassword", validate: rules.HardcodedPassword}, {name: "PlaidAccessID", validate: rules.PlaidAccessID}, {name: "PrivateKey", validate: rules.PrivateKey}, - {name: "VaultServiceToken", validate: rules.VaultServiceToken}, } for _, tRule := range testsRules { From cb84ecb455024590359224ea6e70b18bc4db84e0 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Thu, 29 May 2025 16:24:20 +0100 Subject: [PATCH 3/5] fix: change regex to gitleak new version --- engine/rules/github.go | 64 --------------------------------------- engine/rules/rule_test.go | 4 ++- 2 files changed, 3 insertions(+), 65 deletions(-) diff --git a/engine/rules/github.go b/engine/rules/github.go index d408d996..94cf7d75 100644 --- a/engine/rules/github.go +++ b/engine/rules/github.go @@ -7,54 +7,6 @@ import ( "github.com/zricethezav/gitleaks/v8/config" ) -func GitHubPat() *config.Rule { - // define rule - r := config.Rule{ - Description: "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", - RuleID: "github-pat", - Regex: regexp.MustCompile(`ghp_[0-9a-zA-Z]{36}`), - Keywords: []string{"ghp_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "ghp_"+secrets.NewSecret(alphaNumeric("36"))), - } - return validate(r, tps, nil) -} - -func GitHubFineGrainedPat() *config.Rule { - // define rule - r := config.Rule{ - Description: "Found a GitHub Fine-Grained Personal Access Token, risking unauthorized repository access and code manipulation.", - RuleID: "github-fine-grained-pat", - Regex: regexp.MustCompile(`github_pat_[0-9a-zA-Z_]{82}`), - Keywords: []string{"github_pat_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "github_pat_"+secrets.NewSecret(alphaNumeric("82"))), - } - return validate(r, tps, nil) -} - -func GitHubOauth() *config.Rule { - // define rule - r := config.Rule{ - Description: "Discovered a GitHub OAuth Access Token, posing a risk of compromised GitHub account integrations and data leaks.", - RuleID: "github-oauth", - Regex: regexp.MustCompile(`gho_[0-9a-zA-Z]{36}`), - Keywords: []string{"gho_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "gho_"+secrets.NewSecret(alphaNumeric("36"))), - } - return validate(r, tps, nil) -} - func GitHubApp() *config.Rule { // define rule r := config.Rule{ @@ -71,19 +23,3 @@ func GitHubApp() *config.Rule { } return validate(r, tps, nil) } - -func GitHubRefresh() *config.Rule { - // define rule - r := config.Rule{ - Description: "Detected a GitHub Refresh Token, which could allow prolonged unauthorized access to GitHub services.", - RuleID: "github-refresh-token", - Regex: regexp.MustCompile(`ghr_[0-9a-zA-Z]{36}`), - Keywords: []string{"ghr_"}, - } - - // validate - tps := []string{ - generateSampleSecret("github", "ghr_"+secrets.NewSecret(alphaNumeric("36"))), - } - return validate(r, tps, nil) -} diff --git a/engine/rules/rule_test.go b/engine/rules/rule_test.go index 6ed8330c..d3e6f012 100644 --- a/engine/rules/rule_test.go +++ b/engine/rules/rule_test.go @@ -15,10 +15,12 @@ func Test2msRules(t *testing.T) { validate func() *config.Rule }{ {name: "AuthenticatedURL", validate: rules.AuthenticatedURL}, + {name: "Clojars", validate: rules.Clojars}, + {name: "GenericCredential", validate: rules.GenericCredential}, + {name: "GitHubApp", validate: rules.GitHubApp}, {name: "HardcodedPassword", validate: rules.HardcodedPassword}, {name: "PlaidAccessID", validate: rules.PlaidAccessID}, {name: "PrivateKey", validate: rules.PrivateKey}, - {name: "VaultServiceToken", validate: rules.VaultServiceToken}, } for _, tRule := range testsRules { From 59bc386dedfe4b128b2ce7760f90c983122b4c54 Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Thu, 29 May 2025 16:50:36 +0100 Subject: [PATCH 4/5] fix: updated regex to last version of gitleaks --- engine/rules/clojars.go | 2 +- engine/rules/github.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/rules/clojars.go b/engine/rules/clojars.go index dde57ef1..ed5c5bdb 100644 --- a/engine/rules/clojars.go +++ b/engine/rules/clojars.go @@ -12,7 +12,7 @@ func Clojars() *config.Rule { r := config.Rule{ Description: "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation.", RuleID: "clojars-api-token", - Regex: regexp.MustCompile(`(?i)(CLOJARS_)([a-z0-9]{60})`), + Regex: regexp.MustCompile(`(?i)CLOJARS_[a-z0-9]{60}`), Keywords: []string{"clojars"}, } diff --git a/engine/rules/github.go b/engine/rules/github.go index 94cf7d75..85fe04dd 100644 --- a/engine/rules/github.go +++ b/engine/rules/github.go @@ -12,7 +12,7 @@ func GitHubApp() *config.Rule { r := config.Rule{ Description: "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security.", RuleID: "github-app-token", - Regex: regexp.MustCompile(`ghu_[0-9a-zA-Z]{36}|ghs_[0-9a-zA-Z]{36}`), + Regex: regexp.MustCompile(`(?:ghu|ghs)_[0-9a-zA-Z]{36}`), Keywords: []string{"ghu_", "ghs_"}, } From 206565ce9ee8a8c3852ac287d626295e93edf9ba Mon Sep 17 00:00:00 2001 From: Miguel Neiva Date: Thu, 29 May 2025 16:52:50 +0100 Subject: [PATCH 5/5] fix: add VaultServiceToken to rule test --- engine/rules/rule_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/rules/rule_test.go b/engine/rules/rule_test.go index d3e6f012..b58c5543 100644 --- a/engine/rules/rule_test.go +++ b/engine/rules/rule_test.go @@ -21,6 +21,7 @@ func Test2msRules(t *testing.T) { {name: "HardcodedPassword", validate: rules.HardcodedPassword}, {name: "PlaidAccessID", validate: rules.PlaidAccessID}, {name: "PrivateKey", validate: rules.PrivateKey}, + {name: "VaultServiceToken", validate: rules.VaultServiceToken}, } for _, tRule := range testsRules {