Basic & bearer authorization patterns

.github/workflows/benchmark.yml

@@ -30,8 +30,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
-          repository: Samsung/CredData
-          ref: d425c1b7600407ca5a82f2379fdc8627d194fb39
+          repository: babenek/CredData
+          ref: 10a12bd8171a5daf5f106f988e0d1ab8c321d894
 
       - name: Markup hashing
         run: |
@@ -86,8 +86,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
-          repository: Samsung/CredData
-          ref: d425c1b7600407ca5a82f2379fdc8627d194fb39
+          repository: babenek/CredData
+          ref: 10a12bd8171a5daf5f106f988e0d1ab8c321d894
 
       - name: Markup hashing
         run: |
@@ -189,8 +189,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
-          repository: Samsung/CredData
-          ref: d425c1b7600407ca5a82f2379fdc8627d194fb39
+          repository: babenek/CredData
+          ref: 10a12bd8171a5daf5f106f988e0d1ab8c321d894
 
       - name: Markup hashing
         run: |
@@ -377,8 +377,8 @@ jobs:
       - name: Checkout CredData
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
-          repository: Samsung/CredData
-          ref: d425c1b7600407ca5a82f2379fdc8627d194fb39
+          repository: babenek/CredData
+          ref: 10a12bd8171a5daf5f106f988e0d1ab8c321d894
 
       - name: Markup hashing
         run: |

credsweeper/filters/__init__.py

@@ -10,6 +10,7 @@
 from credsweeper.filters.value_base64_encoded_pem_check import ValueBase64EncodedPem
 from credsweeper.filters.value_base64_key_check import ValueBase64KeyCheck
 from credsweeper.filters.value_base64_part_check import ValueBase64PartCheck
+from credsweeper.filters.value_basic_auth_check import ValueBasicAuthCheck
 from credsweeper.filters.value_blocklist_check import ValueBlocklistCheck
 from credsweeper.filters.value_camel_case_check import ValueCamelCaseCheck
 from credsweeper.filters.value_couple_keyword_check import ValueCoupleKeywordCheck

credsweeper/filters/value_basic_auth_check.py

@@ -0,0 +1,37 @@
+import contextlib
+import logging
+
+from credsweeper.common.constants import DEFAULT_PATTERN_LEN, UTF_8
+from credsweeper.config.config import Config
+from credsweeper.credentials.line_data import LineData
+from credsweeper.file_handler.analysis_target import AnalysisTarget
+from credsweeper.filters.filter import Filter
+from credsweeper.utils.util import Util
+
+
+class ValueBasicAuthCheck(Filter):
+    """Check that candidate have a known structure"""
+
+    def __init__(self, config: Config = None) -> None:
+        pass
+
+    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
+        """Run filter checks on received token which might be structured.
+
+        Args:
+            line_data: credential candidate data
+            target: multiline target from which line data was obtained
+
+        Return:
+            True, if need to filter candidate and False if left
+
+        """
+        value = line_data.value
+        with contextlib.suppress(Exception):
+            # Basic encoding -> login:password
+            decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True)
+            delimiter_pos = decoded.find(b':')
+            # check whether the delimiter exists and all chars are decoded
+            if 0 < delimiter_pos < len(decoded) - DEFAULT_PATTERN_LEN and decoded.decode(UTF_8):
+                return False
+        return True

credsweeper/rules/config.yaml

@@ -1465,6 +1465,36 @@
     - code
     - doc
 
+- name: Basic Authorization
+  severity: medium
+  confidence: strong
+  type: pattern
+  values:
+    - (?P<variable>(?i:basic))(?P<separator>[^\\0-9A-Za-z_/+-]+)(?P<value>[=0-9A-Za-z_/+-]{8,8000})(?![0-9A-Za-z_/+-])
+  min_line_len: 18
+  filter_type:
+    - ValueBasicAuthCheck
+  required_substrings:
+    - basic
+  target:
+    - code
+    - doc
+
+- name: Bearer Authorization
+  severity: medium
+  confidence: strong
+  type: pattern
+  values:
+    - (?P<variable>(?i:bearer|ntlm))(?P<separator>[^\\0-9A-Za-z_/+-]+)(?P<value>[.0-9A-Za-z_/+-]{32,8000}=*)(?![0-9A-Za-z_/+-])
+  min_line_len: 37
+  filter_type: GeneralKeyword
+  required_substrings:
+    - bearer
+    - ntlm
+  target:
+    - code
+    - doc
+
 - name: API
   severity: low
   confidence: moderate

tests/__init__.py

@@ -7,7 +7,7 @@
 NEGLIGIBLE_ML_THRESHOLD = 0.0001
 
 # credentials count after scan with negligible ML threshold
-SAMPLES_CRED_COUNT = 506
+SAMPLES_CRED_COUNT = 506 + 3
 
 # Number of filtered credentials with ML
 ML_FILTERED = 22
@@ -16,7 +16,7 @@
 SAMPLES_POST_CRED_COUNT = SAMPLES_CRED_COUNT - ML_FILTERED
 
 # with option --doc & NEGLIGIBLE_ML_THRESHOLD
-SAMPLES_IN_DOC = 858
+SAMPLES_IN_DOC = 858 + 3
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 128

tests/data/depth_3.json

@@ -408,24 +408,66 @@
             }
         ]
     },
+    {
+        "rule": "Bearer Authorization",
+        "severity": "medium",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "Authorization: NTLM TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "line_num": 4,
+                "path": "./tests/samples/auth.hs",
+                "info": "FILE:./tests/samples/auth.hs|RAW",
+                "variable": "NTLM",
+                "variable_start": 15,
+                "variable_end": 19,
+                "value": "TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "value_start": 20,
+                "value_end": 59,
+                "entropy": 4.00235
+            }
+        ]
+    },
     {
         "rule": "Auth",
         "severity": "medium",
         "confidence": "moderate",
         "ml_probability": 1.0,
         "line_data_list": [
             {
-                "line": "curl -H \"Authorization: Basic WxhZGRpVuc2VzYW1lbjYp12vcG\" http://localhost:8080/.",
+                "line": "curl -H \"Authorization: Basic R2hyZG5oYzpycWVpIGVuZ2xiZg==\" http://localhost:8080/.",
                 "line_num": 8,
                 "path": "./tests/samples/auth_n.template",
                 "info": "FILE:./tests/samples/auth_n.template|RAW",
                 "variable": "Authorization",
                 "variable_start": 9,
                 "variable_end": 22,
-                "value": "WxhZGRpVuc2VzYW1lbjYp12vcG",
+                "value": "R2hyZG5oYzpycWVpIGVuZ2xiZg==",
                 "value_start": 30,
-                "value_end": 56,
-                "entropy": 4.08506
+                "value_end": 58,
+                "entropy": 4.20897
+            }
+        ]
+    },
+    {
+        "rule": "Basic Authorization",
+        "severity": "medium",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "curl -H \"Authorization: Basic R2hyZG5oYzpycWVpIGVuZ2xiZg==\" http://localhost:8080/.",
+                "line_num": 8,
+                "path": "./tests/samples/auth_n.template",
+                "info": "FILE:./tests/samples/auth_n.template|RAW",
+                "variable": "Basic",
+                "variable_start": 24,
+                "variable_end": 29,
+                "value": "R2hyZG5oYzpycWVpIGVuZ2xiZg==",
+                "value_start": 30,
+                "value_end": 58,
+                "entropy": 4.20897
             }
         ]
     },
@@ -450,6 +492,27 @@
             }
         ]
     },
+    {
+        "rule": "Bearer Authorization",
+        "severity": "medium",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.",
+                "line_num": 9,
+                "path": "./tests/samples/auth_n.template",
+                "info": "FILE:./tests/samples/auth_n.template|RAW",
+                "variable": "Bearer",
+                "variable_start": 24,
+                "variable_end": 30,
+                "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj",
+                "value_start": 31,
+                "value_end": 65,
+                "entropy": 4.53585
+            }
+        ]
+    },
     {
         "rule": "AWS Client ID",
         "severity": "high",

tests/data/doc.json

@@ -167,6 +167,69 @@
             }
         ]
     },
+    {
+        "rule": "Bearer Authorization",
+        "severity": "medium",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "Authorization: NTLM TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "line_num": 4,
+                "path": "./tests/samples/auth.hs",
+                "info": "FILE:./tests/samples/auth.hs|RAW",
+                "variable": "NTLM",
+                "variable_start": 15,
+                "variable_end": 19,
+                "value": "TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
+                "value_start": 20,
+                "value_end": 59,
+                "entropy": 4.00235
+            }
+        ]
+    },
+    {
+        "rule": "Basic Authorization",
+        "severity": "medium",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "curl -H \"Authorization: Basic R2hyZG5oYzpycWVpIGVuZ2xiZg==\" http://localhost:8080/.",
+                "line_num": 8,
+                "path": "./tests/samples/auth_n.template",
+                "info": "FILE:./tests/samples/auth_n.template|RAW",
+                "variable": "Basic",
+                "variable_start": 24,
+                "variable_end": 29,
+                "value": "R2hyZG5oYzpycWVpIGVuZ2xiZg==",
+                "value_start": 30,
+                "value_end": 58,
+                "entropy": 4.20897
+            }
+        ]
+    },
+    {
+        "rule": "Bearer Authorization",
+        "severity": "medium",
+        "confidence": "strong",
+        "ml_probability": null,
+        "line_data_list": [
+            {
+                "line": "curl -H \"Authorization: Bearer eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj\" http://localhost:8080/.",
+                "line_num": 9,
+                "path": "./tests/samples/auth_n.template",
+                "info": "FILE:./tests/samples/auth_n.template|RAW",
+                "variable": "Bearer",
+                "variable_start": 24,
+                "variable_end": 30,
+                "value": "eyJGRpVu1c2VzY2-823r_db32hbf4W1lbj",
+                "value_start": 31,
+                "value_end": 65,
+                "entropy": 4.53585
+            }
+        ]
+    },
     {
         "rule": "AWS Client ID",
         "severity": "high",