Add '/api/v1/text/generation' FMS Detector API support

Author: saichandrapandraju

detectors/llm_judge/app.py

@@ -11,6 +11,8 @@
     ContentsAnalysisResponse,
     MetricsListResponse,
     Error,
+    GenerationAnalysisHttpRequest,
+    GenerationAnalysisResponse,
 )
 
 
@@ -35,21 +37,39 @@ async def lifespan(app: FastAPI):
     "/api/v1/text/contents",
     response_model=ContentsAnalysisResponse,
     description="""LLM-as-Judge detector that evaluates content using various metrics like safety, toxicity, accuracy, helpfulness, etc. \
-                    The metric parameter allows you to specify which evaluation criteria to use. \
+                    The metric detector_params parameter allows you to specify which evaluation criteria to use. \
                     Supports all built-in vllm_judge metrics including safety, accuracy, helpfulness, clarity, and many more.""",
     responses={
         404: {"model": Error, "description": "Resource Not Found"},
         422: {"model": Error, "description": "Validation Error"},
     },
 )
-async def detector_unary_handler(
+async def detector_content_analysis_handler(
     request: ContentAnalysisHttpRequest,
     detector_id: Annotated[str, Header(example="llm_judge_safety")],
 ):
     """Analyze content using LLM-as-Judge evaluation."""
     detector: LLMJudgeDetector = app.get_detector()
-    return ContentsAnalysisResponse(root=await detector.run(request))
+    return ContentsAnalysisResponse(root=await detector.analyze_content(request))
 
+@app.post(
+    "/api/v1/text/generation",
+    response_model=GenerationAnalysisResponse,
+    description="""Analyze a single generation using the specified metric. \
+                    The metric detector_params parameter allows you to specify which evaluation criteria to use. \
+                    Supports all built-in vllm_judge metrics including safety, accuracy, helpfulness, clarity, and many more.""",
+    responses={
+        404: {"model": Error, "description": "Resource Not Found"},
+        422: {"model": Error, "description": "Validation Error"},
+    },
+)
+async def detector_generation_analysis_handler(
+    request: GenerationAnalysisHttpRequest,
+    detector_id: Annotated[str, Header(example="llm_judge_safety")],
+):
+    """Analyze a single generation using LLM-as-Judge evaluation."""
+    detector: LLMJudgeDetector = app.get_detector()
+    return await detector.analyze_generation(request)
 
 @app.get(
     "/api/v1/metrics",

detectors/llm_judge/detector.py

@@ -8,6 +8,8 @@
     ContentAnalysisHttpRequest,
     ContentAnalysisResponse,
     ContentsAnalysisResponse,
+    GenerationAnalysisHttpRequest,
+    GenerationAnalysisResponse,
 )
 
 
@@ -40,33 +42,48 @@ def _initialize_judge(self) -> None:
             logger.error(f"Failed to detect model: {e}")
             raise
 
-    async def evaluate_single_content(self, content: str, params: Dict[str, Any]) -> ContentAnalysisResponse:
+    def _validate_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
         """
-        Evaluate a single piece of content using the specified metric.
-        
-        Args:
-            content: Text content to evaluate
-            params: vLLM Judge parameters for the evaluation
-            
-        Returns:
-            ContentAnalysisResponse with evaluation results
+        Make sure the params have valid metric/criteria and scale.
         """
         if "metric" not in params:
             if "criteria" not in params:
                 params["metric"] = "safety" # Default to safety
             elif "scale" not in params:
                 params["scale"] = (0, 1) # Default to 0-1 scale
-        
-        if "metric" in params:
+        else:
             if params["metric"] not in self.available_metrics:
                 raise MetricNotFoundError(
                     f"Metric '{params['metric']}' not found. Available metrics: {', '.join(sorted(self.available_metrics))}"
                 )
             judge_metric = BUILTIN_METRICS[params["metric"]]
             if judge_metric.scale is None:
                 params["scale"] = (0, 1) # Default to 0-1 scale
+        
+        return params
+    
+    def _get_score(self, result: EvaluationResult) -> float:
+        """
+        Get the score from the evaluation result.
+        """
+        if isinstance(result.decision, (int, float)) or result.score is not None:
+            return float(result.score if result.score is not None else result.decision)
+        return 0.0 # FIXME: default to 0 because of non-optional field in schema
+
+    async def evaluate_single_content(self, content: str, params: Dict[str, Any]) -> ContentAnalysisResponse:
+        """
+        Evaluate a single piece of content using the specified metric.
+        
+        Args:
+            content: Text content to evaluate
+            params: vLLM Judge parameters for the evaluation
+            
+        Returns:
+            ContentAnalysisResponse with evaluation results
+        """
+        params: Dict[str, Any] = self._validate_params(params)
 
-        evaluation_params = {
+        evaluation_params: Dict[str, Any] = {
             "content": content,
             **params
         }
@@ -76,11 +93,8 @@ async def evaluate_single_content(self, content: str, params: Dict[str, Any]) ->
             **evaluation_params
         )
 
-        # Convert to response format
-        score = None
-        if isinstance(result.decision, (int, float)) or result.score is not None:
-            # Numeric result
-            score = float(result.score if result.score is not None else result.decision)
+        # Convert to response format. 
+        score: float = self._get_score(result)
 
         return ContentAnalysisResponse(
             start=0,
@@ -93,12 +107,12 @@ async def evaluate_single_content(self, content: str, params: Dict[str, Any]) ->
             metadata={"reasoning": result.reasoning}
         )
 
-    async def run(self, request: ContentAnalysisHttpRequest) -> ContentsAnalysisResponse:
+    async def analyze_content(self, request: ContentAnalysisHttpRequest) -> ContentsAnalysisResponse:
         """
         Run content analysis for each input text.
         
         Args:
-            request: Input request containing texts and metric to analyze
+            request: Input request containing texts and optional metric to analyze
             
         Returns:
             ContentsAnalysisResponse: The aggregated response for all input texts
@@ -111,7 +125,53 @@ async def run(self, request: ContentAnalysisHttpRequest) -> ContentsAnalysisResp
             contents_analyses.append([analysis])  # Wrap in list to match schema
 
         return contents_analyses
+
+    async def evaluate_single_generation(self, prompt: str, generated_text: str, params: Dict[str, Any]) -> GenerationAnalysisResponse:
+        """
+        Evaluate a single generation based on the prompt and generated text.
+
+        Args:
+            prompt: Prompt to the LLM
+            generated_text: Generated text from the LLM
+            params: vLLM Judge parameters for the evaluation
+            
+        Returns:
+            GenerationAnalysisResponse: The response for the generation analysis
+        """
+        params: Dict[str, Any] = self._validate_params(params)
+        evaluation_params: Dict[str, Any] = {
+            "input": prompt,
+            "content": generated_text,
+            **params
+        }
+
+        result: EvaluationResult = await self.judge.evaluate(
+            **evaluation_params
+        )
+        
+        score: float = self._get_score(result)
+        
+        return GenerationAnalysisResponse(
+            detection=str(result.decision),
+            detection_type="llm_judge",
+            score=score,
+            evidences=[],
+            metadata={"reasoning": result.reasoning}
+        )
+
+    async def analyze_generation(self, request: GenerationAnalysisHttpRequest) -> GenerationAnalysisResponse:
+        """
+        Analyze a single generation based on the prompt and generated text.
+
+        Args:
+            request: Input request containing prompt, generated text and optional metric to analyze
             
+        Returns:
+            GenerationAnalysisResponse: The response for the generation analysis
+        """
+        return await self.evaluate_single_generation(prompt=request.prompt,
+                                                     generated_text=request.generated_text,
+                                                     params=request.detector_params)
 
     async def close(self):
         """Close the judge client."""

detectors/llm_judge/scheme.py

@@ -71,4 +71,23 @@ class Error(BaseModel):
 class MetricsListResponse(BaseModel):
     """Response for listing available metrics."""
     metrics: List[str] = Field(description="List of available metric names")
-    total: int = Field(description="Total number of available metrics")
+    total: int = Field(description="Total number of available metrics")
+
+class GenerationAnalysisHttpRequest(BaseModel):
+    prompt: str = Field(description="Prompt is the user input to the LLM", example="What do you think about the future of AI?")
+    generated_text: str = Field(description="Generated response from the LLM", example="The future of AI is bright but we need to be careful about the risks.")
+    detector_params: Optional[Dict[str, Any]] = Field(
+        default_factory=dict, 
+        description="Detector parameters for evaluation (e.g., metric, criteria, etc.)",
+        example={"metric": "safety"}
+    )
+
+class GenerationAnalysisResponse(BaseModel):
+    detection: str = Field(example="safe")
+    detection_type: str = Field(example="llm_judge")
+    score: float = Field(example=0.8)
+    evidences: Optional[List[EvidenceObj]] = Field(
+        description="Optional field providing evidences for the provided detection",
+        default=[],
+    )
+    metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional metadata from evaluation")