Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 20 additions & 20 deletions .2ms.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,23 +29,23 @@ ignore-result:
- 33a14f1d1e4a1201a3e0062ebf09079fe8c84714 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go
- da610673906f695e3e85bda6fc0a916762f01a70 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go
- f8da5c56428cf708773be38269932c46aaf44cd4 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go
- 0d49f4953e8c5b2e04cca54d40bd2a91c079926b # value used for testing
- 5fb857fa72e8d568e6cfd96119d6b2db87c1e9b2 # value used for testing
- 6b92e79146584c6263671b7bcaac79a9c0852465 # value used for testing
- 22a792422372ef239494839d11c188258d18abc8 # value used for testing
- 29ce1990ca4555a207e77a66ffc26d46575a7911 # value used for testing
- 98a2f843609061bba58b69d4d31b70624de299ee # value used for testing
- 0188f28d26c2ae3f87df20092ab39c4465d6bbba # value used for testing
- 468bdfec08e1660b6ec73d78d15f03c320c68078 # value used for testing
- 5586d6fb77d9fa54224604ab158c2ceda4ab0995 # value used for testing
- 6403ca0ffb2abf3f1c9f70202474fb8f6564c4d7 # value used for testing
- a6fe66dfd9531c5415c1d1fed28b71f13a855a46 # value used for testing
- aaf4ba87a3bdbaf9346c0229f404eb86c0e6aabe # value used for testing
- b09e3219bca2cbcc4d7bc34f46e394e1f80d6574 # value used for testing
- c00a0d0af6bac8b20572bbb3b0b2cbea70476a0d # value used for testing
- c94ccae65acb14fdd2b9db7c9119e58875346a3b # value used for testing
- d4ac7947e0a7a4b387bf46279daa74e9dbe7f66f # value used for testing
- dd2e802e4c3205e57e291a89dfd469946531292b # value used for testing
- e475d6cf0a94469ea1717db008936a4e8749fe6a # value used for testing
- eed7c634d36422d7276cd8623c149e4c8d874f95 # value used for testing
- ff933778f18c92254c15369564b7d359f44018b5 # value used for testing
- 0fed4aed1a63c270c410840e01c957025fe2e7b9 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 1e829f2d6236716cecb4683af3d26236fe4d5467 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 2d2c9ddd21109c7d4909e166c18d570eadc95812 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 2fbcd323e560798376c79e73a78938b8edd294d8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 4e243aa393011da09ff2bc4a598c2879e967abb8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 09f677494966f740534bc70d6194529a96d59a69 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 10abdf40e0d14bdfb965f9a5be9056a09c7eb40d # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 4632bbfa56424359e6386b56d05bcec49b16548a # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 489123e817ab17accc3d47a6d39cdb17e37b6c97 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 14472805c7de45b2c3b5caf92f19effcba98c386 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- acf7fa74f45c792f8ab3f701609a0b6ba4283b91 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- bc457e70eee50d9edd5ec8194bc7749fa95fde34 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- ca6b0f5b7f055a1ae52c2f34ad5d0409f7625c85 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- d7cf637e896d585946966d95d7b04910569a0191 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- d9207d5fa344d2423e97384f45014c87c0c91d4f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- f858b057df65752e0030854331dd1e5e8e41856b # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600
- 670491bf5e759f4c03bf0e47f519deaccdc9ac44 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/files#diff-918450788fde1467e3fe71000c32f812131f7dcd7dafcb0310c8c9910ffea848
- 46d644a014e91528b0f6d4180305d35452ed7f46 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
- b841441ea9d80db4ad9f21abe25cf30836a33f1d # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
- f1c519e1be61df80729d099a21235b64b525b280 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4
78 changes: 78 additions & 0 deletions engine/rules/generic-key.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package rules

import (
"github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules"
"github.com/zricethezav/gitleaks/v8/config"
)

func GenericCredential() *config.Rule {
// define rule
r := config.Rule{
RuleID: "generic-api-key",
Description: "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
Regex: generateSemiGenericRegexIncludingXml([]string{
"key",
"api",
"token",
"secret",
"client",
"passwd",
"password",
"auth",
"access",
}, `[0-9a-z\-_.=]{10,150}`, true),
Keywords: []string{
"key",
"api",
"token",
"secret",
"client",
"passwd",
"password",
"auth",
"access",
},
Entropy: 3.5,
Allowlist: config.Allowlist{
StopWords: rules.DefaultStopWords,
},
}

// validate
tps := []string{
generateSampleSecret("generic", "CLOJARS_34bf0e88955ff5a1c328d6a7491acc4f48e865a7b8dd4d70a70749037443"),
generateSampleSecret("generic", "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"),
`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`,

`<key>client_secret</key>
<string>6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde</string>`,

`<key>password</key>
<string>bW9jLmxpYW1nQGhjaW5pbW1pay5ucmVvamI=</string>`,

`<key>password</key>
<string>bW9jLmxpYW1nQGhjaW5pbW1pay5ucmVvamI=</string>`,

`<key>access_key_FOR_X_SERVICES</key>
<string>kgfur834kmjfdoi34i9</string>`,
}
fps := []string{
`client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`,
`password combination.

R5: Regulatory--21`,

`<key>AD_UNIT_ID_FOR_BANNER_TEST</key>
<string>ca-app-pub-3940256099942544/2934735716</string>`,
`<key>AD_UNIT_ID_FOR_INTERSTITIAL_TEST</key>
<string>ca-app-pub-3940256099942544/4411468910</string>`,
`<key>CLIENT_ID</key>
<string>407966239993-b1h97alknrmf0g846um5pr3a25s9qmeu.apps.googleusercontent.com</string>`,
`<key>REVERSED_CLIENT_ID</key>
<string>com.googleusercontent.apps.407966239993-b1h97alknrmf0g846um5pr3a25s9qmeu</string>`,
`<key>GOOGLE_APP_ID</key>
<string>1:407966239993:ios:0d7534f14f8cfe19</string>`,
}
return validate(r, tps, fps)
}
2 changes: 1 addition & 1 deletion engine/rules/rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func getDefaultRules() *[]Rule {
{Rule: *rules.FrameIO(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryNewsAndMedia, RuleType: 4}},
{Rule: *rules.FreshbooksAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryFinancialServices, RuleType: 4}},
{Rule: *rules.GCPAPIKey(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryCloudPlatform, RuleType: 4}},
{Rule: *rules.GenericCredential(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
{Rule: *GenericCredential(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
{Rule: *rules.GitHubPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryDevelopmentPlatform, RuleType: 4}},
{Rule: *rules.GitHubFineGrainedPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}},
{Rule: *rules.GitHubOauth(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAuthenticationAndAuthorization, RuleType: 4}},
Expand Down
33 changes: 30 additions & 3 deletions engine/rules/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,17 @@ const (
identifierCaseInsensitiveSuffix = `)`
identifierPrefix = `(?:`
identifierSuffix = `)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}`
identifierSuffixIncludingXml = `)(?:[0-9a-z\-_\t .]{0,20})(?:<\/key>\s{0,10}<string)?(?:[\s|']|[\s|"]){0,3}`

// commonly used assignment operators or function call
operator = `(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)`

// boundaries for the secret
// \x60 = `
secretPrefixUnique = `\b(`
secretPrefix = `(?:'|\"|\s|=|\x60){0,5}(`
secretSuffix = `)(?:['|\"|\n|\r|\s|\x60|;]|$)`
secretPrefixUnique = `\b(`
secretPrefix = `(?:'|\"|\s|=|\x60){0,5}(`
secretSuffix = `)(?:['|\"|\n|\r|\s|\x60|;]|$)`
secretSuffixIncludingXml = `)(?:['|\"|\n|\r|\s|\x60|;]|$|\s{0,10}<\/string>)`
)

func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp {
Expand All @@ -44,6 +46,31 @@ func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseIn
sb.WriteString(secretSuffix)
return regexp.MustCompile(sb.String())
}
func generateSemiGenericRegexIncludingXml(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp {
var sb strings.Builder
// The identifiers should always be case-insensitive.
// This is inelegant but prevents an extraneous `(?i:)` from being added to the pattern; it could be removed.
if isCaseInsensitive {
sb.WriteString(caseInsensitive)
writeIdentifiersIncludinXml(&sb, identifiers)
} else {
sb.WriteString(identifierCaseInsensitivePrefix)
writeIdentifiers(&sb, identifiers)
sb.WriteString(identifierCaseInsensitiveSuffix)
}
sb.WriteString(operator)
sb.WriteString(secretPrefix)
sb.WriteString(secretRegex)
sb.WriteString(secretSuffixIncludingXml)

return regexp.MustCompile(sb.String())
}

func writeIdentifiersIncludinXml(sb *strings.Builder, identifiers []string) {
sb.WriteString(identifierPrefix)
sb.WriteString(strings.Join(identifiers, "|"))
sb.WriteString(identifierSuffixIncludingXml)
}

func writeIdentifiers(sb *strings.Builder, identifiers []string) {
sb.WriteString(identifierPrefix)
Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@ require (
github.com/spf13/cast v1.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.32.0 // indirect
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.22.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)

Expand Down
12 changes: 8 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,9 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
Expand All @@ -127,8 +128,9 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand All @@ -137,15 +139,17 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand Down