Address comments

Author: RobGeada

detectors/built_in/app.py

@@ -27,7 +27,7 @@ def detect_content(request: ContentAnalysisHttpRequest):
                 try:
                     message_detections += detector_registry.handle_request(content, request.detector_params)
                 except Exception as e:
-                    raise HTTPException(status_code=500, detail=str(e))
+                    raise HTTPException(status_code=500) from e
         detections.append(message_detections)
     return ContentsAnalysisResponse(root=detections)
 

detectors/built_in/file_type_detectors.py

@@ -176,8 +176,10 @@ def __init__(self):
 
     def handle_request(self, content: str, detector_params: dict) -> List[ContentAnalysisResponse]:
         detections = []
-        if "file_type" in detector_params and isinstance(detector_params["file_type"], list):
-            for file_type in detector_params["file_type"]:
+        if "file_type" in detector_params and isinstance(detector_params["file_type"], (list, str)):
+            file_types = detector_params["file_type"]
+            file_types = [file_types] if isinstance(file_types, str) else file_types
+            for file_type in file_types:
                 if file_type.startswith("json-with-schema"):
                     result = is_valid_json_schema(content, file_type.split("json-with-schema:")[1])
                     if result is not None:

detectors/built_in/regex_detectors.py

@@ -11,9 +11,57 @@ def email_address_detector(string: str) -> List[ContentAnalysisResponse]:
     return get_regex_detections(string, pattern, "pii", "email_address")
 
 def credit_card_detector(string: str) -> List[ContentAnalysisResponse]:
-    """Detect credit cards in the text contents"""
-    pattern = r"\b(?:4\d{3}|5[0-5]\d{2}|6\d{3}|1\d{3}|3\d{3})[- ]\d{4}[- ]\d{4}[- ]\d{4}\b"
-    return get_regex_detections(string, pattern, "pii", "credit_card")
+    """Detect credit cards in the text contents (Visa, MasterCard, Amex, Discover, Diners Club, JCB) with Luhn check"""
+    # Match major card types with separators (space or dash) between groups, not continuous digits
+    pattern = (
+        r"\b(?:"
+        r"4\d{3}([- ])\d{4}\1\d{4}\1\d{4}"                # Visa 16-digit with separators
+        r"|4\d{15}"                                       # Visa 16-digit continuous
+        r"|5[1-5]\d{2}([- ])\d{4}\2\d{4}\2\d{4}"           # MasterCard 16-digit with separators
+        r"|5[1-5]\d{14}"                                  # MasterCard 16-digit continuous
+        r"|3[47]\d{2}([- ])\d{6}\3\d{5}"                   # Amex 15-digit with separators
+        r"|3[47]\d{13}"                                   # Amex 15-digit continuous
+        r"|6(?:011|5\d{2})([- ])\d{4}\4\d{4}\4\d{4}"       # Discover 16-digit with separators
+        r"|6(?:011|5\d{2})\d{12}"                         # Discover 16-digit continuous
+        r"|3(?:0[0-5]|[68]\d)\d([- ])\d{6}\5\d{4}"         # Diners Club 14-digit with separators
+        r"|3(?:0[0-5]|[68]\d)\d{11}"                      # Diners Club 14-digit continuous
+        r"|35\d{2}([- ])\d{4}\6\d{4}\6\d{4}"               # JCB 16-digit with separators
+        r"|35\d{14}"                                      # JCB 16-digit continuous
+        r")\b"
+    )
+    # Find all matches and filter with Luhn check
+    detections = []
+    for match in re.finditer(pattern, string):
+        cc_number = match.group(0).replace(" ", "").replace("-", "")
+        print(cc_number)
+        if is_luhn_valid(cc_number):
+            detections.append(
+                ContentAnalysisResponse(
+                    start=match.start(),
+                    end=match.end(),
+                    text=match.group(0),
+                    detection_type="pii",
+                    detection="credit_card",
+                    score=1.0
+                )
+            )
+    return detections
+
+def luhn_checksum(card_number: str):
+    card_number = "".join(c for c in card_number if c in "0123456789")
+    def digits_of(n):
+        return [int(d) for d in str(n)]
+    digits = digits_of(card_number)
+    odd_digits = digits[-1::-2]
+    even_digits = digits[-2::-2]
+    checksum = 0
+    checksum += sum(odd_digits)
+    for d in even_digits:
+        checksum += sum(digits_of(d*2))
+    return checksum % 10
+
+def is_luhn_valid(card_number):
+    return luhn_checksum(card_number) == 0
 
 
 def ipv4_detector(string: str) -> List[ContentAnalysisResponse]:
@@ -85,8 +133,10 @@ def __init__(self):
 
     def handle_request(self, content: str, detector_params: dict) -> List[ContentAnalysisResponse]:
         detections = []
-        if "regex" in detector_params and isinstance(detector_params["regex"], list):
-            for regex in detector_params["regex"]:
+        if "regex" in detector_params and isinstance(detector_params["regex"], (list, str)):
+            regexes = detector_params["regex"]
+            regexes = [regexes] if isinstance(regexes, str) else regexes
+            for regex in regexes:
                 if regex == "$CUSTOM_REGEX":
                     pass
                 elif regex in self.registry:

tests/detectors/builtIn/test_filetype.py

@@ -78,7 +78,6 @@ def test_json_schema_invalid(self, client: TestClient, jsonschema):
         assert detections[0]["detection"] == "json_schema_mismatch"
 
     def test_json_schema_invalid_json(self, client: TestClient, jsonschema):
-        import json
         payload = {
             "contents": ['{a: 1}'],
             "detector_params": {"file_type": [f"json-with-schema:{jsonschema}"]}
@@ -91,9 +90,7 @@ def test_json_schema_invalid_json(self, client: TestClient, jsonschema):
 
     def test_json_schema_invalid_json_schema(self, client: TestClient):
         # The schema expects an object with a required integer property "a"
-        invalid_schema = {
-            "notvalidjson: {"
-        }
+        invalid_schema = '{"notvalidjson": {'
         payload = {
             "contents": [json.dumps({"a": 1})],
             "detector_params": {"file_type": [f"json-with-schema:{invalid_schema}"]}
@@ -218,6 +215,17 @@ def test_detect_content_unrecognized_filetype(self, client: TestClient):
         assert "message" in data
         assert "Unrecognized file type" in data["message"]
 
+    def test_detect_content_single_filetype(self, client: TestClient):
+        payload = {
+            "contents": ['{a: 1, b: 2}'],
+            "detector_params": {"file_type": "json"}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        detections = resp.json()[0]
+        assert detections[0]["detection"] == "invalid_json"
+
+
     def test_multiple_filetype_valid_and_invalid(self, client: TestClient):
         import json
         schema = {

tests/detectors/builtIn/test_regex.py

@@ -11,8 +11,6 @@ def client(self):
         "regex,content,expected",
         [
             ("email", "Contact me at [email protected]", "[email protected]"),
-            ("credit-card", "Card: 4111-1111-1111-1111", "4111-1111-1111-1111"),
-            ("credit-card", "Card: 4111 1111 1111 1111", "4111 1111 1111 1111"),
             ("ipv4", "My IP is 192.168.1.1", "192.168.1.1"),
             ("us-social-security-number", "SSN: 123-45-6789", "123-45-6789"),
             ("us-social-security-number", "SSN: 123 45 6789", "123 45 6789"),
@@ -37,8 +35,7 @@ def test_builtin_regex_detectors(self, client, regex, content, expected):
     @pytest.mark.parametrize(
         "regex,content",
         [
-            ("email", "Contact me at [email protected] "),
-            ("credit-card", "Card: 4111111111111111"),
+            ("email", "Contact me at [email protected] "), # invalid luhn card
             ("us-social-security-number", "SSN: 123456789"),
             ("us-phone-number", "Call 1234567890"),
         ]
@@ -52,6 +49,73 @@ def test_builtin_regex_detectors_should_not_match(self, client, regex, content):
         assert resp.status_code == 200
         assert resp.json()[0] == []
 
+    @pytest.mark.parametrize(
+        "content,expected",
+        [
+            # Visa
+            ("Card: 4111 1111 1111 1111", "4111 1111 1111 1111"),
+            ("Card: 4111-1111-1111-1111", "4111-1111-1111-1111"),
+            ("Card: 4111111111111111", "4111111111111111"),
+            # MasterCard
+            ("Card: 5555 5555 5555 4444", "5555 5555 5555 4444"),
+            ("Card: 5555-5555-5555-4444", "5555-5555-5555-4444"),
+            ("Card: 5555555555554444", "5555555555554444"),
+            # Amex
+            ("Card: 3782 822463 10005", "3782 822463 10005"),
+            ("Card: 3782-822463-10005", "3782-822463-10005"),
+            ("Card: 378282246310005", "378282246310005"),
+            # Discover
+            ("Card: 6011 1111 1111 1117", "6011 1111 1111 1117"),
+            ("Card: 6011-1111-1111-1117", "6011-1111-1111-1117"),
+            ("Card: 6011111111111117", "6011111111111117"),
+            # Diners Club
+            ("Card: 3056 930902 5904", "3056 930902 5904"),
+            ("Card: 3056-930902-5904", "3056-930902-5904"),
+            ("Card: 30569309025904", "30569309025904"),
+            # JCB
+            ("Card: 3530 1113 3330 0000", "3530 1113 3330 0000"),
+            ("Card: 3530-1113-3330-0000", "3530-1113-3330-0000"),
+            ("Card: 3530111333300000", "3530111333300000"),
+        ]
+    )
+    def test_credit_card_detector_patterns(self, client, content, expected):
+        payload = {
+            "contents": [content],
+            "detector_params": {"regex": ["credit-card"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        found = [d["text"] for d in resp.json()[0]]
+        assert expected in found
+
+    @pytest.mark.parametrize(
+        "content",
+        [
+            # Invalid Luhn
+            "Card: 4111 1111 1111 1112",
+            "Card: 5555-5555-5555-4440",
+            "Card: 3782 822463 10006",
+            "Card: 6011-1111-1111-1110",
+            "Card: 3056-930902-5900",
+            "Card: 3530-1113-3330-0001",
+            # Not enough digits
+            "Card: 4111 1111 1111",
+            "Card: 5555-5555-5555",
+            "Card: 3782 822463",
+            "Card: 6011-1111-1111",
+            "Card: 3056-930902",
+            "Card: 3530-1113-3330",
+        ]
+    )
+    def test_credit_card_detector_invalid(self, client, content):
+        payload = {
+            "contents": [content],
+            "detector_params": {"regex": ["credit-card"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        assert resp.json()[0] == []
+
 
     def test_multiple_regexes(self, client):
         payload = {
@@ -116,5 +180,18 @@ def test_multiple_contents(self, client):
         assert any("[email protected]" in d["text"] for d in results[0])
         assert any("123-45-6789" in d["text"] for d in results[1])
 
+    def test_single_detector(self, client):
+        payload = {
+            "contents": [
+                "Email: [email protected]",
+                "SSN: 123-45-6789"
+            ],
+            "detector_params": {"regex": "email"}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        results = resp.json()
+        assert any("[email protected]" in d["text"] for d in results[0])
+