Merge pull request trustyai-explainability#29 from saichandrapandraju/multi-detector-state

Refactor detector management to store multiple detector types

Author: saichandrapandraju

detectors/built_in/app.py

@@ -1,5 +1,5 @@
 from fastapi import HTTPException
-
+from contextlib import asynccontextmanager
 from base_detector_registry import BaseDetectorRegistry
 from regex_detectors import RegexDetectorRegistry
 from file_type_detectors import FileTypeDetectorRegistry
@@ -8,21 +8,32 @@
 from detectors.common.scheme import ContentAnalysisHttpRequest,  ContentsAnalysisResponse
 from detectors.common.app import DetectorBaseAPI as FastAPI
 
-app = FastAPI()
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    app.set_detector(RegexDetectorRegistry(), "regex")
+    app.set_detector(FileTypeDetectorRegistry(), "file_type")
+    yield
+    
+    app.cleanup_detector()
+
+
+app = FastAPI(lifespan=lifespan)
 Instrumentator().instrument(app).expose(app)
 
 
-registry : dict[str, BaseDetectorRegistry] = {
-    "regex": RegexDetectorRegistry(),
-    "file_type": FileTypeDetectorRegistry(),
-}
+# registry : dict[str, BaseDetectorRegistry] = {
+#     "regex": RegexDetectorRegistry(),
+#     "file_type": FileTypeDetectorRegistry(),
+# }
 
 @app.post("/api/v1/text/contents", response_model=ContentsAnalysisResponse)
 def detect_content(request: ContentAnalysisHttpRequest):
     detections = []
     for content in request.contents:
         message_detections = []
-        for detector_kind, detector_registry in registry.items():
+        for detector_kind, detector_registry in app.get_all_detectors().items():
+            if not isinstance(detector_registry, BaseDetectorRegistry):
+                raise TypeError(f"Detector {detector_kind} is not a valid BaseDetectorRegistry")
             if detector_kind in request.detector_params:
                 try:
                     message_detections += detector_registry.handle_request(content, request.detector_params)
@@ -37,7 +48,9 @@ def detect_content(request: ContentAnalysisHttpRequest):
 @app.get("/registry")
 def get_registry():
     result = {}
-    for detector_type, detector_registry in registry.items():
+    for detector_type, detector_registry in app.get_all_detectors().items():
+        if not isinstance(detector_registry, BaseDetectorRegistry):
+            raise TypeError(f"Detector {detector_type} is not a valid BaseDetectorRegistry")
         result[detector_type] = {}
         for detector_name, detector_fn in detector_registry.get_registry().items():
             result[detector_type][detector_name] = detector_fn.__doc__

detectors/common/app.py

@@ -34,6 +34,7 @@
 class DetectorBaseAPI(FastAPI):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.state.detectors = {}
         self.add_exception_handler(
             RequestValidationError, self.validation_exception_handler
         )
@@ -94,17 +95,21 @@ async def http_exception_handler(self, request, exc):
             content={"code": exc.status_code, "message": exc.detail},
         )
 
-    def set_detector(self, detector) -> None:
+    def set_detector(self, detector, detector_name="default") -> None:
         """Store detector in app.state"""
-        self.state.detector = detector
+        self.state.detectors[detector_name] = detector
 
-    def get_detector(self):
+    def get_detector(self, detector_name="default"):
         """Retrieve detector from app.state"""
-        return getattr(self.state, 'detector', None)
+        return self.state.detectors.get(detector_name)
+
+    def get_all_detectors(self) -> dict:
+        """Retrieve all detectors from app.state"""
+        return self.state.detectors
 
     def cleanup_detector(self) -> None:
         """Clean up detector resources"""
-        self.state.detector = None
+        self.state.detectors.clear()
 
 async def health():
     return "ok"

detectors/llm_judge/app.py

@@ -1,7 +1,7 @@
 from contextlib import asynccontextmanager
 from typing import Annotated, Dict
 
-from fastapi import Header
+from fastapi import Header, HTTPException
 from prometheus_fastapi_instrumentator import Instrumentator
 
 from detectors.common.app import DetectorBaseAPI as FastAPI
@@ -48,6 +48,8 @@ async def detector_unary_handler(
 ):
     """Analyze content using LLM-as-Judge evaluation."""
     detector: LLMJudgeDetector = app.get_detector()
+    if not detector:
+        raise HTTPException(status_code=503, detail="Detector not found")
     return ContentsAnalysisResponse(root=await detector.run(request))
 
 
@@ -63,7 +65,7 @@ async def list_metrics():
     """List all available evaluation metrics."""
     detector: LLMJudgeDetector = app.get_detector()
     if not detector:
-        return {"metrics": [], "total": 0}
+        raise HTTPException(status_code=503, detail="Detector not found")
 
     metrics = detector.list_available_metrics()
     return MetricsListResponse(metrics=metrics, total=len(metrics))

tests/detectors/builtIn/test_filetype.py

@@ -7,6 +7,10 @@ class TestFileTypeDetectors:
     @pytest.fixture
     def client(self):
         from detectors.built_in.app import app
+        from detectors.built_in.file_type_detectors import FileTypeDetectorRegistry
+
+        app.set_detector(FileTypeDetectorRegistry(), "file_type")
+
         return TestClient(app)
 
     @pytest.fixture
@@ -280,4 +284,79 @@ def test_multiple_filetype_valid_and_invalid(self, client: TestClient):
         assert not any(d["detection"] == "invalid_yaml" for d in detections[5])
 
         # 7: invalid yaml
-        assert any(d["detection"] == "invalid_yaml" for d in detections[6])
+        assert any(d["detection"] == "invalid_yaml" for d in detections[6])
+    
+    
+    # === ERROR HANDLING & INVALID DETECTOR TYPES =================================================
+    def test_unregistered_detector_kind_ignored(self, client: TestClient):
+        """Test that requesting an unregistered detector kind is silently ignored"""
+        payload = {
+            "contents": ['{"a": 1}'],
+            "detector_params": {"nonexistent_detector": ["some_value"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        # Should return empty list since nonexistent_detector is not registered
+        assert resp.json()[0] == []
+
+    def test_mixed_valid_invalid_detector_kinds(self, client: TestClient):
+        """Test mixing valid and invalid detector kinds"""
+        payload = {
+            "contents": ['{a: 1, b: 2}'],
+            "detector_params": {
+                "file_type": ["json"],  # valid detector kind
+                "nonexistent_detector": ["some_value"]  # invalid detector kind
+            }
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        detections = resp.json()[0]
+        # Should only process the valid detector kind
+        assert detections[0]["detection"] == "invalid_json"
+    
+    def test_empty_detector_params(self, client: TestClient):
+        """Test with empty detector_params"""
+        payload = {
+            "contents": ['{"a": 1}'],
+            "detector_params": {}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        # Should return empty list since no detectors are specified
+        assert resp.json()[0] == []
+    
+    def test_multiple_invalid_file_types(self, client: TestClient):
+        """Test multiple invalid file types to ensure all errors are handled"""
+        payload = {
+            "contents": ['test content'],
+            "detector_params": {"file_type": ["invalid_type_1", "invalid_type_2"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 400
+        data = resp.json()
+        assert "message" in data
+        assert "Unrecognized file type" in data["message"]
+
+    def test_mixed_valid_invalid_file_types(self, client: TestClient):
+        """Test mixing valid and invalid file types"""
+        payload = {
+            "contents": ['{a: 1, b: 2}'],
+            "detector_params": {"file_type": ["json", "invalid_type"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 400
+        data = resp.json()
+        assert "message" in data
+        assert "Unrecognized file type" in data["message"]
+    
+    def test_case_sensitivity_file_types(self, client: TestClient):
+        """Test case sensitivity of file types"""
+        payload = {
+            "contents": ['{"a": 1}'],
+            "detector_params": {"file_type": ["JSON"]}  # uppercase
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 400
+        data = resp.json()
+        assert "message" in data
+        assert "Unrecognized file type" in data["message"]

tests/detectors/builtIn/test_regex.py

@@ -5,6 +5,10 @@ class TestRegexDetectors:
     @pytest.fixture
     def client(self):
         from detectors.built_in.app import app
+        from detectors.built_in.regex_detectors import RegexDetectorRegistry
+
+        app.set_detector(RegexDetectorRegistry(), "regex")
+
         return TestClient(app)
 
     @pytest.mark.parametrize(
@@ -194,4 +198,59 @@ def test_single_detector(self, client):
         assert any("[email protected]" in d["text"] for d in results[0])
 
 
+    # === ERROR HANDLING & INVALID DETECTOR TYPES =================================================    
+    def test_unregistered_detector_kind_ignored(self, client):
+        """Test that requesting an unregistered detector kind is silently ignored"""
+        payload = {
+            "contents": ["[email protected]"],
+            "detector_params": {"nonexistent_detector": ["some_value"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        # Should return empty list since nonexistent_detector is not registered
+        assert resp.json()[0] == []
+
+    def test_mixed_valid_invalid_detector_kinds(self, client):
+        """Test mixing valid and invalid detector kinds"""
+        payload = {
+            "contents": ["Contact me at [email protected]"],
+            "detector_params": {
+                "regex": ["email"],  # valid detector kind
+                "nonexistent_detector": ["some_value"]  # invalid detector kind
+            }
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        detections = resp.json()[0]
+        # Should only process the valid detector kind
+        assert detections[0]["text"] == "[email protected]"
+
+    def test_empty_detector_params(self, client):
+        """Test with empty detector_params"""
+        payload = {
+            "contents": ["[email protected]"],
+            "detector_params": {}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 200
+        # Should return empty list since no detectors are specified
+        assert resp.json()[0] == []
+
+    def test_null_regex_pattern(self, client):
+        """Test with null regex pattern"""
+        payload = {
+            "contents": ["[email protected]"],
+            "detector_params": {"regex": [None]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 500  # Should cause an error when processing None
+
+    def test_malformed_regex_groups(self, client):
+        """Test malformed regex with unmatched groups"""
+        payload = {
+            "contents": ["test content"],
+            "detector_params": {"regex": ["(unclosed group"]}
+        }
+        resp = client.post("/api/v1/text/contents", json=payload)
+        assert resp.status_code == 500  # Should cause regex compilation error
 

tox.ini

@@ -10,6 +10,7 @@ deps =
     -r{toxinidir}/detectors/huggingface/requirements.txt
     -r{toxinidir}/detectors/common/requirements.txt
     -r{toxinidir}/detectors/llm_judge/requirements.txt
+    -r{toxinidir}/detectors/built_in/requirements.txt
 setenv =
     PYTHONPATH = {toxinidir}/detectors/huggingface:{toxinidir}/detectors/llm_judge:{toxinidir}/detectors:{toxinidir}
 commands =