fix: prevent evaluation loop from causing stuck workflows

- Make evaluation informational-only when run_assessment=False
- Add 15s LLM call timeout via request_timeout on ChatLiteLLM
- Default evaluation parsing to ACCEPT when ambiguous
- Derive max_total_iterations from max_validation_attempts + 1
- Add per-node timing to all workflow nodes
- Switch eval model default to openai/gpt-oss-120b on groq
- Lower recursion_limit from 100 to 50
- Update default max_validation_attempts from 5 to 3

Closes #119

Author: neuromechanist

.env.example

@@ -67,11 +67,10 @@ OPENROUTER_API_KEY=your-openrouter-api-key-here
 ANNOTATION_MODEL=mistralai/mistral-small-3.2-24b-instruct
 ANNOTATION_PROVIDER=mistral
 
-# Evaluation/Assessment Model (consistent quality checks: Qwen3-235B via DeepInfra)
+# Evaluation/Assessment Model (fast quality checks: GPT-OSS-120B via Groq)
 # Used for evaluation, assessment, and feedback agents
-# Leave EVALUATION_PROVIDER empty to let OpenRouter auto-route
-EVALUATION_MODEL=qwen/qwen3-235b-a22b-2507
-EVALUATION_PROVIDER=deepinfra/fp8
+EVALUATION_MODEL=openai/gpt-oss-120b
+EVALUATION_PROVIDER=groq
 
 # Vision Model (image description: Qwen3-VL via deepinfra)
 VISION_MODEL=qwen/qwen3-vl-30b-a3b-instruct
@@ -142,8 +141,8 @@ API_WORKERS=4
 # ============================================================================
 # Workflow Configuration
 # ============================================================================
-MAX_VALIDATION_ATTEMPTS=5
-MAX_TOTAL_ITERATIONS=10
+MAX_VALIDATION_ATTEMPTS=3
+MAX_TOTAL_ITERATIONS=4
 
 # ============================================================================
 # Logging

src/agents/evaluation_agent.py

@@ -201,19 +201,13 @@ def _parse_decision(self, feedback: str) -> bool:
             result = faithful_match.group(1)
             return result in ["yes", "partial"]  # Accept partial as good enough!
 
-        # Fallback: look for positive indicators
-        positive_indicators = ["accept", "good", "sufficient", "adequate", "captures well"]
-        negative_indicators = ["refine", "missing", "incorrect", "inaccurate", "lacks"]
+        # Fallback: look for explicit refine indicators only
+        refine_indicators = ["refine", "incorrect", "inaccurate", "wrong"]
+        if any(indicator in feedback_lower for indicator in refine_indicators):
+            return False
 
-        positive_score = sum(1 for indicator in positive_indicators if indicator in feedback_lower)
-        negative_score = sum(1 for indicator in negative_indicators if indicator in feedback_lower)
-
-        # If more positive than negative, accept
-        if positive_score > negative_score:
-            return True
-
-        # Default to refine if ambiguous (conservative)
-        return False
+        # Default to accept if ambiguous -- avoid unnecessary refinement loops
+        return True
 
     def _check_tags_and_suggest(self, annotation: str) -> str:
         """Check annotation for invalid tags and suggest alternatives.

src/agents/workflow.py

@@ -5,6 +5,7 @@
 """
 
 import logging
+import time
 from pathlib import Path
 
 from langchain_core.language_models import BaseChatModel
@@ -221,9 +222,13 @@ async def _annotate_node(self, state: HedAnnotationState) -> dict:
         print(
             f"[WORKFLOW] Entering annotate node (validation attempt {state['validation_attempts']}, total iteration {total_iters})"
         )
+        t0 = time.monotonic()
         result = await self.annotation_agent.annotate(state)
+        elapsed = time.monotonic() - t0
         result["total_iterations"] = total_iters  # Increment counter
-        print(f"[WORKFLOW] Annotation generated: {result.get('current_annotation', '')[:100]}...")
+        print(
+            f"[WORKFLOW] Annotation generated in {elapsed:.1f}s: {result.get('current_annotation', '')[:100]}..."
+        )
         return result
 
     async def _validate_node(self, state: HedAnnotationState) -> dict:
@@ -236,9 +241,11 @@ async def _validate_node(self, state: HedAnnotationState) -> dict:
             State update
         """
         print("[WORKFLOW] Entering validate node")
+        t0 = time.monotonic()
         result = await self.validation_agent.validate(state)
+        elapsed = time.monotonic() - t0
         print(
-            f"[WORKFLOW] Validation result: {result.get('validation_status')}, is_valid: {result.get('is_valid')}"
+            f"[WORKFLOW] Validation result in {elapsed:.1f}s: {result.get('validation_status')}, is_valid: {result.get('is_valid')}"
         )
         if not result.get("is_valid"):
             print(f"[WORKFLOW] Validation errors: {result.get('validation_errors', [])}")
@@ -254,8 +261,12 @@ async def _evaluate_node(self, state: HedAnnotationState) -> dict:
             State update
         """
         print("[WORKFLOW] Entering evaluate node")
+        t0 = time.monotonic()
         result = await self.evaluation_agent.evaluate(state)
-        print(f"[WORKFLOW] Evaluation result: is_faithful={result.get('is_faithful')}")
+        elapsed = time.monotonic() - t0
+        print(
+            f"[WORKFLOW] Evaluation result in {elapsed:.1f}s: is_faithful={result.get('is_faithful')}"
+        )
 
         # Set default assessment values if assessment will be skipped
         run_assessment = state.get("run_assessment", False)
@@ -281,7 +292,11 @@ async def _assess_node(self, state: HedAnnotationState) -> dict:
         Returns:
             State update
         """
-        return await self.assessment_agent.assess(state)
+        t0 = time.monotonic()
+        result = await self.assessment_agent.assess(state)
+        elapsed = time.monotonic() - t0
+        print(f"[WORKFLOW] Assessment completed in {elapsed:.1f}s")
+        return result
 
     async def _summarize_feedback_node(self, state: HedAnnotationState) -> dict:
         """Summarize feedback node: Condense errors and feedback.
@@ -293,9 +308,11 @@ async def _summarize_feedback_node(self, state: HedAnnotationState) -> dict:
             State update with summarized feedback
         """
         print("[WORKFLOW] Entering summarize_feedback node")
+        t0 = time.monotonic()
         result = await self.feedback_summarizer.summarize(state)
+        elapsed = time.monotonic() - t0
         print(
-            f"[WORKFLOW] Feedback summarized: {result.get('validation_errors_augmented', [''])[0][:100] if result.get('validation_errors_augmented') else 'No feedback'}..."
+            f"[WORKFLOW] Feedback summarized in {elapsed:.1f}s: {result.get('validation_errors_augmented', [''])[0][:100] if result.get('validation_errors_augmented') else 'No feedback'}..."
         )
         return result
 
@@ -327,52 +344,39 @@ def _route_after_evaluation(
         self,
         state: HedAnnotationState,
     ) -> str:
-        """Route after evaluation based on faithfulness.
+        """Route after evaluation based on faithfulness and assessment mode.
+
+        When run_assessment=False (default), evaluation is informational only;
+        the result is reported but never triggers refinement loops.
+        When run_assessment=True, evaluation can trigger refinement and the
+        assessment node runs at the end.
 
         Args:
             state: Current workflow state
 
         Returns:
             Next node name
         """
-        # Check if max total iterations reached
-        total_iters = state.get("total_iterations", 0)
-        max_iters = state.get("max_total_iterations", 10)
         run_assessment = state.get("run_assessment", False)
 
+        # When assessment is off, evaluation is informational -- always end
+        if not run_assessment:
+            print(
+                f"[WORKFLOW] Evaluation complete (informational, is_faithful={state['is_faithful']}) - routing to END"
+            )
+            return "end"
+
+        # Assessment mode: allow refinement loops with iteration cap
+        total_iters = state.get("total_iterations", 0)
+        max_iters = state.get("max_total_iterations", 4)
+
         if total_iters >= max_iters:
-            # Only run assessment at max iterations if explicitly requested
-            if run_assessment:
-                print(f"[WORKFLOW] Routing to assess (max total iterations {max_iters} reached)")
-                return "assess"
-            else:
-                print(
-                    "[WORKFLOW] Skipping assessment (max iterations reached, assessment not requested) - routing to END"
-                )
-                return "end"
+            print(f"[WORKFLOW] Routing to assess (max total iterations {max_iters} reached)")
+            return "assess"
 
         if state["is_faithful"]:
-            # Only run assessment if explicitly requested
-            if state.get("is_valid") and run_assessment:
-                print(
-                    "[WORKFLOW] Routing to assess (annotation is valid and faithful, assessment requested)"
-                )
-                return "assess"
-            elif state.get("is_valid"):
-                print(
-                    "[WORKFLOW] Skipping assessment (annotation is valid and faithful, assessment not requested) - routing to END"
-                )
-                return "end"
-            elif run_assessment:
-                print(
-                    "[WORKFLOW] Routing to assess (annotation is faithful but has validation issues)"
-                )
-                return "assess"
-            else:
-                print(
-                    "[WORKFLOW] Skipping assessment (has validation issues, assessment not requested) - routing to END"
-                )
-                return "end"
+            print("[WORKFLOW] Routing to assess (annotation is faithful)")
+            return "assess"
         else:
             print(
                 f"[WORKFLOW] Routing to summarize_feedback (annotation needs refinement, iteration {total_iters}/{max_iters})"
@@ -383,8 +387,8 @@ async def run(
         self,
         input_description: str,
         schema_version: str = "8.4.0",
-        max_validation_attempts: int = 5,
-        max_total_iterations: int = 10,
+        max_validation_attempts: int = 3,
+        max_total_iterations: int = 4,
         run_assessment: bool = False,
         no_extend: bool = False,
         config: dict | None = None,

src/api/main.py

@@ -98,8 +98,8 @@ def create_openrouter_workflow(
         api_key: OpenRouter API key
         annotation_model: Model for annotation (default: ANNOTATION_MODEL env or Claude Haiku 4.5)
         annotation_provider: Provider for annotation model (default: ANNOTATION_PROVIDER env or "anthropic")
-        eval_model: Model for eval/assessment/feedback (default: EVALUATION_MODEL env or Qwen3-235B)
-        eval_provider: Provider for eval models (default: EVALUATION_PROVIDER env or auto-routed)
+        eval_model: Model for eval/assessment/feedback (default: EVALUATION_MODEL env or GPT-OSS-120B)
+        eval_provider: Provider for eval models (default: EVALUATION_PROVIDER env or "groq")
         temperature: LLM temperature (default: 0.1)
         user_id: User ID for cache optimization (derived from API key if not provided)
         schema_dir: Path to HED schemas (None = fetch from GitHub)
@@ -112,8 +112,8 @@ def create_openrouter_workflow(
     # Apply defaults from environment
     default_annotation_model = os.getenv("ANNOTATION_MODEL", "anthropic/claude-haiku-4.5")
     default_annotation_provider = os.getenv("ANNOTATION_PROVIDER", "anthropic")
-    default_eval_model = os.getenv("EVALUATION_MODEL", "qwen/qwen3-235b-a22b-2507")
-    default_eval_provider = os.getenv("EVALUATION_PROVIDER", "")
+    default_eval_model = os.getenv("EVALUATION_MODEL", "openai/gpt-oss-120b")
+    default_eval_provider = os.getenv("EVALUATION_PROVIDER", "groq")
 
     # Resolve final values: parameter > env var > default
     actual_annotation_model = get_model_name(annotation_model or default_annotation_model)
@@ -640,15 +640,17 @@ async def annotate(
         active_workflow = workflow
 
     try:
-        # Run annotation workflow with increased recursion limit for long descriptions
-        # LangGraph default is 25, increase to 100 for complex workflows
-        config = {"recursion_limit": 100}
+        config = {"recursion_limit": 50}
+
+        # Derive total iteration cap from validation attempts (+1 for evaluation refinement)
+        max_total_iterations = request.max_validation_attempts + 1
 
         start_time = time.time()
         final_state = await active_workflow.run(
             input_description=request.description,
             schema_version=request.schema_version,
             max_validation_attempts=request.max_validation_attempts,
+            max_total_iterations=max_total_iterations,
             run_assessment=request.run_assessment,
             config=config,
         )
@@ -839,12 +841,14 @@ async def annotate_from_image(
         image_metadata = vision_result["metadata"]
 
         # Step 2: Pass description through HED annotation workflow
-        config = {"recursion_limit": 100}
+        config = {"recursion_limit": 50}
+        img_max_total_iters = request.max_validation_attempts + 1
 
         final_state = await active_workflow.run(
             input_description=image_description,
             schema_version=request.schema_version,
             max_validation_attempts=request.max_validation_attempts,
+            max_total_iterations=img_max_total_iters,
             run_assessment=request.run_assessment,
             config=config,
         )
@@ -992,12 +996,13 @@ async def annotate_stream(
             raise HTTPException(status_code=503, detail="Workflow not initialized")
         active_workflow = workflow
 
-    # Create initial state
+    # Create initial state with iteration cap derived from validation attempts
+    max_total_iterations = request.max_validation_attempts + 1
     initial_state = create_initial_state(
         request.description,
         request.schema_version,
         request.max_validation_attempts,
-        10,  # max_total_iterations
+        max_total_iterations,
         request.run_assessment,
     )
 
@@ -1031,7 +1036,7 @@ def send_event(event_type: str, data: dict) -> str:
             validation_attempt = 0
 
             # Use LangGraph's astream_events for real-time streaming
-            config = {"recursion_limit": 100}
+            config = {"recursion_limit": 50}
             async for event in active_workflow.graph.astream_events(
                 initial_state, config=config, version="v2"
             ):
@@ -1287,11 +1292,12 @@ def send_event(event_type: str, data: dict) -> str:
             )
 
             # Step 2: Create initial state for annotation workflow
+            img_max_total_iterations = request.max_validation_attempts + 1
             initial_state = create_initial_state(
                 image_description,
                 request.schema_version,
                 request.max_validation_attempts,
-                10,  # max_total_iterations
+                img_max_total_iterations,
                 request.run_assessment,
             )
 
@@ -1301,7 +1307,7 @@ def send_event(event_type: str, data: dict) -> str:
             validation_attempt = 0
 
             # Use LangGraph's astream_events for real-time streaming
-            config = {"recursion_limit": 100}
+            config = {"recursion_limit": 50}
             async for event in active_workflow.graph.astream_events(
                 initial_state, config=config, version="v2"
             ):

src/api/models.py

@@ -28,8 +28,8 @@ class AnnotationRequest(BaseModel):
         examples=["8.3.0", "8.4.0"],
     )
     max_validation_attempts: int = Field(
-        default=5,
-        description="Maximum validation retry attempts",
+        default=3,
+        description="Maximum validation retry attempts (total iterations = this + 1)",
         ge=1,
         le=10,
     )
@@ -155,8 +155,8 @@ class ImageAnnotationRequest(BaseModel):
         examples=["8.3.0", "8.4.0"],
     )
     max_validation_attempts: int = Field(
-        default=5,
-        description="Maximum validation retry attempts",
+        default=3,
+        description="Maximum validation retry attempts (total iterations = this + 1)",
         ge=1,
         le=10,
     )

src/utils/openrouter_llm.py

@@ -62,13 +62,14 @@ def create_openrouter_llm(
     if user_id:
         model_kwargs["user"] = user_id
 
-    # Create base LLM
+    # Create base LLM with timeout to prevent hanging on slow providers
     llm = ChatLiteLLM(
         model=litellm_model,
         api_key=api_key or os.getenv("OPENROUTER_API_KEY"),
         temperature=temperature,
         max_tokens=max_tokens,
         model_kwargs=model_kwargs,
+        request_timeout=15,
     )
 
     # Determine if caching should be enabled