Pattern improvements. Version Up to 1.14.7 (#809)

* Improve sql password rule

* `get` improved with 'or'

Author: babenek

.ci/benchmark.txt

@@ -226,7 +226,7 @@ FileType           FileNumber    ValidLines    Positives    Negatives
 .zsh                        6           872                        11
 .zsh-theme                  1            97                         1
 TOTAL:                  11375      16999171        17141        53736
-credsweeper result_cnt : 16978, lost_cnt : 0, true_cnt : 16827, false_cnt : 151
+credsweeper result_cnt : 16979, lost_cnt : 0, true_cnt : 16828, false_cnt : 151
 Rules                             Positives    Negatives    Reported     TP    FP     TN    FN       FPR       FNR       ACC       PRC       RCL        F1
 ------------------------------  -----------  -----------  ----------  -----  ----  -----  ----  --------  --------  --------  --------  --------  --------
 API                                     243         4009         242    236     6   4003     7  0.001497  0.028807  0.996943  0.975207  0.971193  0.973196
@@ -270,7 +270,7 @@ Nonce                                   131          109         128    127
 OTP / 2FA Secret                         64            3          56     54     2      1    10  0.666667  0.156250  0.820896  0.964286  0.843750  0.900000
 Other                                     0           20                  0     0     20     0  0.000000            1.000000
 PEM Private Key                        1157           72        1157   1157     0     72     0  0.000000  0.000000  1.000000  1.000000  1.000000  1.000000
-Password                               2595        11366        2527   2517    10  11356    78  0.000880  0.030058  0.993697  0.996043  0.969942  0.982819
+Password                               2595        11366        2528   2518    10  11356    77  0.000880  0.029672  0.993768  0.996044  0.970328  0.983018
 Perplexity API Key                        2            0           2      2     0      0     0            0.000000  1.000000  1.000000  1.000000  1.000000
 Postman Credentials                       2            0           2      2     0      0     0            0.000000  1.000000  1.000000  1.000000  1.000000
 SQL Password                             44           14          41     40     1     13     4  0.071429  0.090909  0.913793  0.975610  0.909091  0.941176
@@ -284,4 +284,4 @@ Token                                  1144         5285        1072   1066
 Twilio Credentials                       30           39          30     30     0     39     0  0.000000  0.000000  1.000000  1.000000  1.000000  1.000000
 URL Credentials                         225          401         221    220     1    400     5  0.002494  0.022222  0.990415  0.995475  0.977778  0.986547
 UUID                                   2517         3716        2554   2494    60   3656    23  0.016146  0.009138  0.986684  0.976507  0.990862  0.983632
-                                      17141        53736       16979  16827   151  53585   314  0.002810  0.018319  0.993439  0.991106  0.981681  0.986371
+                                      17141        53736       16980  16828   151  53585   313  0.002810  0.018260  0.993453  0.991107  0.981740  0.986401

.github/workflows/check.yml

@@ -92,7 +92,7 @@ jobs:
         run: |
           banner="$(python -m credsweeper --banner | head -1)"
           echo "banner = '${banner}'"
-          if [ "CredSweeper 1.14.6 crc32:765e27c6" != "${banner}" ]; then
+          if [ "CredSweeper 1.14.7 crc32:fdae340d" != "${banner}" ]; then
             echo "Update the check for '${banner}'"
             exit 1
           fi

credsweeper/__init__.py

@@ -24,4 +24,4 @@
     "__version__"
 ]
 
-__version__ = "1.14.6"
+__version__ = "1.14.7"

credsweeper/common/keyword_pattern.py

@@ -26,7 +26,7 @@ class KeywordPattern:
            r"\s*" \
            r"(\[(?!\])|\((?!\))|\{(?!\}))" \
            r"(\s|\\{1,8}[tnr])*" \
-           r"(?(get)('[^']{1,31}'|\"[^\"]{1,31}\")\s*,\s*|)" \
+           r"(?(get)('[^']{1,31}'|\"[^\"]{1,31}\")\s*(,|\)\s*or)\s*|)" \
            r"([0-9a-z_]{1,32}\s*[:=]\s*)?" \
            r"){1,8})?"
     string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[\"'`])))?"

credsweeper/rules/config.yaml

@@ -3,7 +3,7 @@
   confidence: weak
   type: pattern
   values:
-    - (?P<variable>(\w*(?i:비밀번호|비번|패스워드|키|암호화?|토큰|(?<!by)pass(?!e[dns]|ing|ion|age)|\bpwd?\b|token|secret|key|cred)\w*)\s*(설정은|[=:!]{1,3}))?\s*([._0-9A-Za-z\[\]]*get(env)?\s*\(\s*(?(variable)[^,]+|[\"'\\]*(\\*([\"']|&(quot|apos|#3[49]);)){0,4}(\w*(?i:(?<!by)pass(?!e[dns]|ing|ion|age|\s+[a-z]{3,80})|\bpwd?\b|token|secret|key|cred)\w*))(\\*([\"']|&(quot|apos|#3[49]);)){0,4})\s*,\s*(default\s*=\s*)?([brufl@]{1,2}(?=\\*[\"'&]))?(?P<lq>(\\*([\"']|&(quot|apos|#3[49]);)){1,4})(?P<value>(.(?!(?P=lq))){4,80}.?)
+    - (?P<variable>(\w*(?i:비밀번호|비번|패스워드|키|암호화?|토큰|(?<!by)pass(?!e[dns]|ing|ion|age)|\bpwd?\b|token|secret|key|cred)\w*)\s*(설정은|[=:!]{1,3}))?\s*([._0-9A-Za-z\[\]]*get(env)?\s*\(\s*(?(variable)[^,]+|[\"'\\]*(\\*([\"']|&(quot|apos|#3[49]);)){0,4}(\w*(?i:(?<!by)pass(?!e[dns]|ing|ion|age|\s+[a-z]{3,80})|\bpwd?\b|token|secret|key|cred)\w*))(\\*([\"']|&(quot|apos|#3[49]);)){0,4})\s*(,(\s*default\s*=)?|\)\s*or)\s*([brufl@]{1,2}(?=\\*[\"'&]))?(?P<lq>(\\*([\"']|&(quot|apos|#3[49]);)){1,4})(?P<value>(.(?!(?P=lq))){4,80}.?)
   filter_type:
     - ValueAllowlistCheck
     - LineGitBinaryCheck
@@ -993,9 +993,9 @@
   confidence: moderate
   type: pattern
   values:
-    - (?:^|/|[^\\0-9A-Za-z+_-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>sl\.(u\.)?[0-9A-Za-z_-]{135})(?![0-9A-Za-z_-])
+    - (?:^|/|[^\\0-9A-Za-z+_-]|\\[0abfnrtv]|(?:%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu][0-9A-Fa-f]{4}|\x1B\[[0-9;]{0,80}m)(?P<value>sl\.(u\.)?[0-9A-Za-z_-]{77,177})(?![0-9A-Za-z_-])
   filter_type: TokenPattern
-  min_line_len: 138
+  min_line_len: 80
   required_substrings:
     - sl.
   target:
@@ -1536,7 +1536,7 @@
   confidence: weak
   type: pattern
   values:
-    - (\\[nrt]|\b)(?i:(?P<variable>(CREATE|ALTER|SET\s{1,8}PASSWORD|INSERT(\s{1,8}IGNORE)?|UPDATE\s{1,8}[^\s;]{1,80})\s{1,8}(LOGIN|USER|ROLE|FOR|INTO|SET)\s{1,8}([^\s;]{1,80}\s{1,8}|VALUES\s*\(){1,8}(IDENTIFIED((\s{1,8}WITH\s{1,8}\S{1,80})?\s{1,8}(BY|AS))|(=|WITH)?\s*PASSWORD\b(\s*=)?)))\s*(?P<wrap>[(]\s*)?(?P<value_leftquote>((?P<esq>\\{1,8})?([\"'`]|&(quot|apos|#3[49]);)){1,4})?(?P<value>(?(value_leftquote)((?!(?P=value_leftquote))(?(esq)((?!(?P=esq)([\"'`]|&(quot|apos|#3[49]);)).)|((?!(?P=value_leftquote)).)))|(?!&(quot|apos|#3[49]);)(\\+([ tnr]|[^\s\"'`])|[^\s\"'`,;\\])){3,80})(?(value_leftquote)(?P<value_rightquote>(?<!\\)(?P=value_leftquote))|(?(wrap)[)]|[\s\"'`,;]))
+    - (\\[nrt]|\b)(?i:(?P<variable>(CREATE|ALTER|SET\s{1,8}PASSWORD|INSERT(\s{1,8}IGNORE)?|UPDATE\s{1,8}[^\s;]{1,80})\s{1,8}(LOGIN|USER|ROLE|FOR|INTO|SET)\s{1,8}((?!IDENTIFIED|PASSWORD)[^\s;]{1,80}\s{1,8}|VALUES\s{0,8}\(){1,8}(IDENTIFIED((\s{1,8}WITH\s{1,8}\S{1,80})?\s{1,8}(BY|AS))|(=|WITH)?\s{0,8}PASSWORD\b(\s{0,8}=)?)))\s{0,8}(?P<wrap>[(]\s{0,8})?(?P<value_leftquote>((?P<esq>\\{1,8})?([\"'`]|&(quot|apos|#3[49]);)){1,4})?(?P<value>(?(value_leftquote)((?!(?P=value_leftquote))(?(esq)((?!(?P=esq)([\"'`]|&(quot|apos|#3[49]);)).)|((?!(?P=value_leftquote)).)))|(?!&(quot|apos|#3[49]);)(\\{1,8}([ tnr]|[^\s\"'`])|[^\s\"'`,;\\])){3,80})(?(value_leftquote)(?P<value_rightquote>(?<!\\)(?P=value_leftquote))|(?(wrap)[)]|[\s\"'`,;]))
   filter_type:
     - ValueAllowlistCheck
     - ValuePatternCheck

tests/__init__.py

@@ -10,16 +10,16 @@
 ZERO_ML_THRESHOLD = 0.0
 
 # with option --doc & NEGLIGIBLE_ML_THRESHOLD
-SAMPLES_IN_DOC = 924
+SAMPLES_IN_DOC = 925
 
 # credentials count after scan without filters and ML validations
-SAMPLES_REGEX_COUNT = 655
+SAMPLES_REGEX_COUNT = 656
 
 # credentials count after scan with filters and without ML validation
-SAMPLES_FILTERED_COUNT = 544
+SAMPLES_FILTERED_COUNT = 545
 
 # credentials count after default post-processing
-SAMPLES_POST_CRED_COUNT = 497
+SAMPLES_POST_CRED_COUNT = 499
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 138

tests/common/test_keyword_pattern.py

@@ -44,6 +44,7 @@ def test_separator_p(self, config: Config, file_path: pytest.fixture, line: str)
             # ['''password=f"\\"secret=2\\""''', '''\\"secret=2\\"'''],  # todo
             # ['''password=r"\\\\"secret=3\\\\""''', '''\\"secret=3\\"'''],  # todo
             # ['''"password = 'sec;$2`\\'[\\/*;ret';";''', '''sec;$2`\\'[\\/*;ret'''],  # todo
+            ['PASSWORD = os.environ.get("PASSWORD") or "at5G6zi!m"', "at5G6zi!m"],
             ["deFINE \\n\\t('DB_PASSWORD',\\n\\t'devSeCrEt');", "devSeCrEt"],
             ['''...log=1;User ID=X3;password=Quantum42!\\""''', '''Quantum42!'''],
             [

tests/data/depth_3_pedantic.json

@@ -1806,6 +1806,27 @@
             }
         ]
     },
+    {
+        "rule": "Password",
+        "severity": "high",
+        "confidence": "moderate",
+        "ml_probability": 0.999,
+        "line_data_list": [
+            {
+                "line": "PASSWORD = os.environ.get(\"PASSWORD\") or \"at5G6zi!m\"",
+                "line_num": 5,
+                "path": "./tests/samples/doc_get_p",
+                "info": "FILE:./tests/samples/doc_get_p|RAW",
+                "variable": "PASSWORD",
+                "variable_start": 0,
+                "variable_end": 8,
+                "value": "at5G6zi!m",
+                "value_start": 42,
+                "value_end": 51,
+                "entropy": 3.16993
+            }
+        ]
+    },
     {
         "rule": "Password",
         "severity": "high",
@@ -14906,6 +14927,27 @@
             }
         ]
     },
+    {
+        "rule": "SQL Password",
+        "severity": "medium",
+        "confidence": "weak",
+        "ml_probability": 0.996,
+        "line_data_list": [
+            {
+                "line": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*92FD05ADCA2EC9D1E10C096DEB1618BC2470E409' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK; -- old MySQL hash scheme with sha1",
+                "line_num": 31,
+                "path": "./tests/samples/sql_password",
+                "info": "FILE:./tests/samples/sql_password|RAW",
+                "variable": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS",
+                "variable_start": 0,
+                "variable_end": 70,
+                "value": "*92FD05ADCA2EC9D1E10C096DEB1618BC2470E409",
+                "value_start": 72,
+                "value_end": 113,
+                "entropy": 3.78769
+            }
+        ]
+    },
     {
         "rule": "SQL Password",
         "severity": "medium",

tests/data/doc.json

@@ -1642,6 +1642,27 @@
             }
         ]
     },
+    {
+        "rule": "DOC_GET",
+        "severity": "medium",
+        "confidence": "weak",
+        "ml_probability": 0.969,
+        "line_data_list": [
+            {
+                "line": "PASSWORD = os.environ.get(\"PASSWORD\") or \"at5G6zi!m\"",
+                "line_num": 5,
+                "path": "./tests/samples/doc_get_p",
+                "info": "FILE:./tests/samples/doc_get_p|RAW",
+                "variable": "PASSWORD =",
+                "variable_start": 0,
+                "variable_end": 10,
+                "value": "at5G6zi!m",
+                "value_start": 42,
+                "value_end": 51,
+                "entropy": 3.16993
+            }
+        ]
+    },
     {
         "rule": "DOC_CREDENTIALS",
         "severity": "medium",
@@ -19962,20 +19983,20 @@
         "rule": "SQL Password",
         "severity": "medium",
         "confidence": "weak",
-        "ml_probability": 0.0,
+        "ml_probability": 0.996,
         "line_data_list": [
             {
-                "line": "\"ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*SqLpa5sW0rD' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCO",
+                "line": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*92FD05ADCA2EC9D1E10C096DEB1618BC2470E409' REQUIRE NONE",
                 "line_num": 31,
                 "path": "./tests/samples/sql_password",
                 "info": "FILE:./tests/samples/sql_password|RAW",
-                "variable": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*SqLpa5sW0rD' REQUIRE NONE PASSWORD",
-                "variable_start": 1,
-                "variable_end": 108,
-                "value": "EXPIRE",
-                "value_start": 109,
-                "value_end": 115,
-                "entropy": 2.25163
+                "variable": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS",
+                "variable_start": 0,
+                "variable_end": 70,
+                "value": "*92FD05ADCA2EC9D1E10C096DEB1618BC2470E409",
+                "value_start": 72,
+                "value_end": 113,
+                "entropy": 3.78769
             }
         ]
     },

tests/data/no_filters_no_ml.json

@@ -2038,6 +2038,27 @@
             }
         ]
     },
+    {
+        "rule": "Password",
+        "severity": "high",
+        "confidence": "moderate",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "PASSWORD = os.environ.get(\"PASSWORD\") or \"at5G6zi!m\"",
+                "line_num": 5,
+                "path": "./tests/samples/doc_get_p",
+                "info": "",
+                "variable": "PASSWORD",
+                "variable_start": 0,
+                "variable_end": 8,
+                "value": "at5G6zi!m",
+                "value_start": 42,
+                "value_end": 51,
+                "entropy": 3.16993
+            }
+        ]
+    },
     {
         "rule": "Password",
         "severity": "high",
@@ -13034,17 +13055,17 @@
         "ml_probability": null,
         "line_data_list": [
             {
-                "line": "\"ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*SqLpa5sW0rD' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK;\",",
+                "line": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*92FD05ADCA2EC9D1E10C096DEB1618BC2470E409' REQUIRE NONE PASSWORD EXPIRE DEFAULT ACCOUNT UNLOCK; -- old MySQL hash scheme with sha1",
                 "line_num": 31,
                 "path": "./tests/samples/sql_password",
                 "info": "",
-                "variable": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS '*SqLpa5sW0rD' REQUIRE NONE PASSWORD",
-                "variable_start": 1,
-                "variable_end": 108,
-                "value": "EXPIRE",
-                "value_start": 109,
-                "value_end": 115,
-                "entropy": 2.25163
+                "variable": "ALTER USER 'test'@'1.1.1.1' IDENTIFIED WITH 'mysql_native_password' AS",
+                "variable_start": 0,
+                "variable_end": 70,
+                "value": "*92FD05ADCA2EC9D1E10C096DEB1618BC2470E409",
+                "value_start": 72,
+                "value_end": 113,
+                "entropy": 3.78769
             }
         ]
     },