diff --git a/.2ms.yml b/.2ms.yml index 36fa6e70..4a582504 100644 --- a/.2ms.yml +++ b/.2ms.yml @@ -29,23 +29,23 @@ ignore-result: - 33a14f1d1e4a1201a3e0062ebf09079fe8c84714 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go - da610673906f695e3e85bda6fc0a916762f01a70 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go - f8da5c56428cf708773be38269932c46aaf44cd4 # value used for testing, found at https://github.com/Checkmarx/2ms/commits/d093b7ca36fdacd2f895dd9afd088fad05d77600/cmd/workers_test.go - - 0d49f4953e8c5b2e04cca54d40bd2a91c079926b # value used for testing - - 5fb857fa72e8d568e6cfd96119d6b2db87c1e9b2 # value used for testing - - 6b92e79146584c6263671b7bcaac79a9c0852465 # value used for testing - - 22a792422372ef239494839d11c188258d18abc8 # value used for testing - - 29ce1990ca4555a207e77a66ffc26d46575a7911 # value used for testing - - 98a2f843609061bba58b69d4d31b70624de299ee # value used for testing - - 0188f28d26c2ae3f87df20092ab39c4465d6bbba # value used for testing - - 468bdfec08e1660b6ec73d78d15f03c320c68078 # value used for testing - - 5586d6fb77d9fa54224604ab158c2ceda4ab0995 # value used for testing - - 6403ca0ffb2abf3f1c9f70202474fb8f6564c4d7 # value used for testing - - a6fe66dfd9531c5415c1d1fed28b71f13a855a46 # value used for testing - - aaf4ba87a3bdbaf9346c0229f404eb86c0e6aabe # value used for testing - - b09e3219bca2cbcc4d7bc34f46e394e1f80d6574 # value used for testing - - c00a0d0af6bac8b20572bbb3b0b2cbea70476a0d # value used for testing - - c94ccae65acb14fdd2b9db7c9119e58875346a3b # value used for testing - - d4ac7947e0a7a4b387bf46279daa74e9dbe7f66f # value used for testing - - dd2e802e4c3205e57e291a89dfd469946531292b # value used for testing - - e475d6cf0a94469ea1717db008936a4e8749fe6a # value used for testing - - eed7c634d36422d7276cd8623c149e4c8d874f95 # value used for testing - - ff933778f18c92254c15369564b7d359f44018b5 # value used for testing + - 0fed4aed1a63c270c410840e01c957025fe2e7b9 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 1e829f2d6236716cecb4683af3d26236fe4d5467 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 2d2c9ddd21109c7d4909e166c18d570eadc95812 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 2fbcd323e560798376c79e73a78938b8edd294d8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 4e243aa393011da09ff2bc4a598c2879e967abb8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 09f677494966f740534bc70d6194529a96d59a69 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 10abdf40e0d14bdfb965f9a5be9056a09c7eb40d # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 4632bbfa56424359e6386b56d05bcec49b16548a # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 489123e817ab17accc3d47a6d39cdb17e37b6c97 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 14472805c7de45b2c3b5caf92f19effcba98c386 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - acf7fa74f45c792f8ab3f701609a0b6ba4283b91 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - bc457e70eee50d9edd5ec8194bc7749fa95fde34 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - ca6b0f5b7f055a1ae52c2f34ad5d0409f7625c85 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - d7cf637e896d585946966d95d7b04910569a0191 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - d9207d5fa344d2423e97384f45014c87c0c91d4f # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - f858b057df65752e0030854331dd1e5e8e41856b # value used for testing, found at https://github.com/Checkmarx/2ms/commit/d093b7ca36fdacd2f895dd9afd088fad05d77600 + - 670491bf5e759f4c03bf0e47f519deaccdc9ac44 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/files#diff-918450788fde1467e3fe71000c32f812131f7dcd7dafcb0310c8c9910ffea848 + - 46d644a014e91528b0f6d4180305d35452ed7f46 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 + - b841441ea9d80db4ad9f21abe25cf30836a33f1d # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 + - f1c519e1be61df80729d099a21235b64b525b280 # value used as true positive, found at https://github.com/Checkmarx/2ms/pull/280/commits/829d4260f43f399499fa78031eda897e8d5fc1a4 \ No newline at end of file diff --git a/engine/rules/generic-key.go b/engine/rules/generic-key.go new file mode 100644 index 00000000..b9de9626 --- /dev/null +++ b/engine/rules/generic-key.go @@ -0,0 +1,78 @@ +package rules + +import ( + "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" + "github.com/zricethezav/gitleaks/v8/config" +) + +func GenericCredential() *config.Rule { + // define rule + r := config.Rule{ + RuleID: "generic-api-key", + Description: "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + Regex: generateSemiGenericRegexIncludingXml([]string{ + "key", + "api", + "token", + "secret", + "client", + "passwd", + "password", + "auth", + "access", + }, `[0-9a-z\-_.=]{10,150}`, true), + Keywords: []string{ + "key", + "api", + "token", + "secret", + "client", + "passwd", + "password", + "auth", + "access", + }, + Entropy: 3.5, + Allowlist: config.Allowlist{ + StopWords: rules.DefaultStopWords, + }, + } + + // validate + tps := []string{ + generateSampleSecret("generic", "CLOJARS_34bf0e88955ff5a1c328d6a7491acc4f48e865a7b8dd4d70a70749037443"), + generateSampleSecret("generic", "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"), + `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, + `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`, + + `client_secret + 6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde`, + + `password + bW9jLmxpYW1nQGhjaW5pbW1pay5ucmVvamI=`, + + `password + bW9jLmxpYW1nQGhjaW5pbW1pay5ucmVvamI=`, + + `access_key_FOR_X_SERVICES + kgfur834kmjfdoi34i9`, + } + fps := []string{ + `client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`, + `password combination. + +R5: Regulatory--21`, + + `AD_UNIT_ID_FOR_BANNER_TEST +ca-app-pub-3940256099942544/2934735716`, + `AD_UNIT_ID_FOR_INTERSTITIAL_TEST +ca-app-pub-3940256099942544/4411468910`, + `CLIENT_ID +407966239993-b1h97alknrmf0g846um5pr3a25s9qmeu.apps.googleusercontent.com`, + `REVERSED_CLIENT_ID +com.googleusercontent.apps.407966239993-b1h97alknrmf0g846um5pr3a25s9qmeu`, + `GOOGLE_APP_ID +1:407966239993:ios:0d7534f14f8cfe19`, + } + return validate(r, tps, fps) +} diff --git a/engine/rules/rules.go b/engine/rules/rules.go index a99b2e38..f53ac610 100644 --- a/engine/rules/rules.go +++ b/engine/rules/rules.go @@ -127,7 +127,7 @@ func getDefaultRules() *[]Rule { {Rule: *rules.FrameIO(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryNewsAndMedia, RuleType: 4}}, {Rule: *rules.FreshbooksAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryFinancialServices, RuleType: 4}}, {Rule: *rules.GCPAPIKey(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryCloudPlatform, RuleType: 4}}, - {Rule: *rules.GenericCredential(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}}, + {Rule: *GenericCredential(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}}, {Rule: *rules.GitHubPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryDevelopmentPlatform, RuleType: 4}}, {Rule: *rules.GitHubFineGrainedPat(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}}, {Rule: *rules.GitHubOauth(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAuthenticationAndAuthorization, RuleType: 4}}, diff --git a/engine/rules/utils.go b/engine/rules/utils.go index 5406ec7f..9c086e6f 100644 --- a/engine/rules/utils.go +++ b/engine/rules/utils.go @@ -15,15 +15,17 @@ const ( identifierCaseInsensitiveSuffix = `)` identifierPrefix = `(?:` identifierSuffix = `)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}` + identifierSuffixIncludingXml = `)(?:[0-9a-z\-_\t .]{0,20})(?:<\/key>\s{0,10}|:{1,3}=|\|\|:|<=|=>|:|\?=)` // boundaries for the secret // \x60 = ` - secretPrefixUnique = `\b(` - secretPrefix = `(?:'|\"|\s|=|\x60){0,5}(` - secretSuffix = `)(?:['|\"|\n|\r|\s|\x60|;]|$)` + secretPrefixUnique = `\b(` + secretPrefix = `(?:'|\"|\s|=|\x60){0,5}(` + secretSuffix = `)(?:['|\"|\n|\r|\s|\x60|;]|$)` + secretSuffixIncludingXml = `)(?:['|\"|\n|\r|\s|\x60|;]|$|\s{0,10}<\/string>)` ) func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp { @@ -44,6 +46,31 @@ func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseIn sb.WriteString(secretSuffix) return regexp.MustCompile(sb.String()) } +func generateSemiGenericRegexIncludingXml(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp { + var sb strings.Builder + // The identifiers should always be case-insensitive. + // This is inelegant but prevents an extraneous `(?i:)` from being added to the pattern; it could be removed. + if isCaseInsensitive { + sb.WriteString(caseInsensitive) + writeIdentifiersIncludinXml(&sb, identifiers) + } else { + sb.WriteString(identifierCaseInsensitivePrefix) + writeIdentifiers(&sb, identifiers) + sb.WriteString(identifierCaseInsensitiveSuffix) + } + sb.WriteString(operator) + sb.WriteString(secretPrefix) + sb.WriteString(secretRegex) + sb.WriteString(secretSuffixIncludingXml) + + return regexp.MustCompile(sb.String()) +} + +func writeIdentifiersIncludinXml(sb *strings.Builder, identifiers []string) { + sb.WriteString(identifierPrefix) + sb.WriteString(strings.Join(identifiers, "|")) + sb.WriteString(identifierSuffixIncludingXml) +} func writeIdentifiers(sb *strings.Builder, identifiers []string) { sb.WriteString(identifierPrefix) diff --git a/go.mod b/go.mod index 4faa1b66..35524222 100644 --- a/go.mod +++ b/go.mod @@ -47,11 +47,11 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.32.0 // indirect + golang.org/x/crypto v0.35.0 // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 1b8ae1ce..b79fd945 100644 --- a/go.sum +++ b/go.sum @@ -113,8 +113,9 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -127,8 +128,9 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -137,15 +139,17 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=