feat: Implement research timeout with time-based polling configurable via environment variables and add a CLI command to recover timed-out research runs.

Author: btfranklin

.env.example

@@ -1,3 +1,8 @@
+# AI Configuration
 OPENAI_API_KEY=sk-...
 PROMPT_REFINER_MODEL=gpt-5.2
 DEEP_RESEARCH_MODEL=o3-deep-research
+
+# Research Configuration
+POLLING_INTERVAL_IN_SECONDS=10
+MAX_POLL_TIME_IN_MINUTES=60

README.md

@@ -36,6 +36,8 @@ Create a `.env` file (untracked) with your OpenAI credentials:
 OPENAI_API_KEY=sk-...
 PROMPT_REFINER_MODEL=gpt-5.2
 DEEP_RESEARCH_MODEL=o3-deep-research
+POLLING_INTERVAL_IN_SECONDS=10
+MAX_POLL_TIME_IN_MINUTES=60
 ```
 
 Deep research requires an OpenAI account with the browsing tooling enabled. Document any environment keys for additional tooling in the repo as you add them.
@@ -68,6 +70,17 @@ pdm run compendium render my-topic.xml --format html
 - `--format FORMAT` — Output format(s) to generate (`md`, `xml`, `html`, `pdf`).
 - `--output PATH` — Base path/filename for the output.
 
+### 5. Recover from a timeout
+
+If a research task times out (exceeding `MAX_POLL_TIME_IN_MINUTES`), recovery information is saved to `timed_out_research.json`. You can resume checking for its completion without starting over:
+
+```bash
+pdm run compendium recover
+```
+
+**Options:**
+- `--input PATH` — Path to the recovery JSON file (defaults to `timed_out_research.json`).
+
 ---
 
 ## Library Usage
@@ -78,7 +91,11 @@ from compendiumscribe import build_compendium, ResearchConfig, DeepResearchError
 try:
     compendium = build_compendium(
         "Emerging pathogen surveillance",
-        config=ResearchConfig(background=False, max_tool_calls=30),
+        config=ResearchConfig(
+            background=False, 
+            max_tool_calls=30,
+            max_poll_time_minutes=15,
+        ),
     )
 except DeepResearchError as exc:
     # Handle or log deep research failures

src/compendiumscribe/cli.py

@@ -1,9 +1,8 @@
-from __future__ import annotations
-
+import json
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
 
 import click
 
@@ -15,6 +14,7 @@
     DeepResearchError,
     ResearchConfig,
     ResearchProgress,
+    ResearchTimeoutError,
     build_compendium,
 )
 
@@ -83,6 +83,13 @@ def handle_progress(update: ResearchProgress) -> None:
         meta = update.metadata or {}
         if "poll_attempt" in meta:
             suffix = f" (poll #{meta['poll_attempt']})"
+        
+        if "elapsed_seconds" in meta:
+            seconds = meta["elapsed_seconds"]
+            mins, secs = divmod(seconds, 60)
+            time_str = f"{mins}m {secs}s" if mins > 0 else f"{secs}s"
+            suffix += f" [Time elapsed: {time_str}]"
+
         stream_kwargs = {"err": update.status == "error"}
         click.echo(
             f"[{timestamp}] {phase_label}: {update.message}{suffix}",
@@ -98,6 +105,19 @@ def handle_progress(update: ResearchProgress) -> None:
     try:
         client = create_openai_client(timeout=config.request_timeout_seconds)
         compendium = build_compendium(topic, client=client, config=config)
+    except ResearchTimeoutError as exc:
+        timeout_data = {
+            "research_id": exc.research_id,
+            "topic": topic,
+            "no_background": no_background,
+            "formats": list(formats),
+            "max_tool_calls": max_tool_calls,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        }
+        Path("timed_out_research.json").write_text(json.dumps(timeout_data, indent=2))
+        click.echo(f"\n[!] Deep research timed out (ID: {exc.research_id}).", err=True)
+        click.echo(f"Stored recovery information in timed_out_research.json", err=True)
+        raise SystemExit(1) from exc
     except MissingAPIKeyError as exc:
         click.echo(f"Configuration error: {exc}", err=True)
         raise SystemExit(1) from exc
@@ -167,6 +187,72 @@ def render(
     _write_outputs(compendium, base_path, formats)
 
 
+@cli.command()
+@click.option(
+    "--input",
+    "input_file",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=Path("timed_out_research.json"),
+    show_default=True,
+    help="Path to the recovery JSON file.",
+)
+def recover(input_file: Path):
+    """Recover a timed-out deep research run."""
+    if not input_file.exists():
+        click.echo(f"Error: Recovery file {input_file} not found.", err=True)
+        raise SystemExit(1)
+
+    try:
+        data = json.loads(input_file.read_text(encoding="utf-8"))
+        research_id = data["research_id"]
+        topic = data["topic"]
+        formats = tuple(data["formats"])
+        max_tool_calls = data.get("max_tool_calls")
+        no_background = data.get("no_background", False)
+    except (json.JSONDecodeError, KeyError) as exc:
+        click.echo(f"Error: Failed to parse recovery file: {exc}", err=True)
+        raise SystemExit(1)
+
+    click.echo(f"Checking status for research ID: {research_id} ('{topic}')...")
+
+    config = ResearchConfig(
+        background=not no_background,
+        max_tool_calls=max_tool_calls,
+    )
+
+    try:
+        client = create_openai_client(timeout=config.request_timeout_seconds)
+        response = client.responses.retrieve(research_id)
+        from .research.utils import coerce_optional_string, get_field
+        status = coerce_optional_string(get_field(response, "status"))
+
+        if status != "completed":
+            click.echo(f"Research is not yet completed (current status: {status}).")
+            click.echo("Please try again later.")
+            return
+
+        click.echo("Research completed! Decoding payload and writing outputs.")
+        from .research.parsing import parse_deep_research_response
+        from .compendium import Compendium
+        
+        payload = parse_deep_research_response(response)
+        compendium = Compendium.from_payload(
+            topic=topic,
+            payload=payload,
+            generated_at=datetime.now(timezone.utc),
+        )
+
+        slug = _generate_slug(topic)
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        base_path = Path(f"{slug}_{timestamp}")
+
+        _write_outputs(compendium, base_path, formats)
+        
+    except Exception as exc:
+        click.echo(f"Error during recovery: {exc}", err=True)
+        raise SystemExit(1)
+
+
 def _write_outputs(
     compendium: "Compendium", base_path: Path, formats: tuple[str, ...]
 ) -> None:

src/compendiumscribe/research/__init__.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
 from .config import ResearchConfig
-from .errors import DeepResearchError
+from .errors import (
+    DeepResearchError,
+    ResearchTimeoutError,
+)
 from .execution import (
     await_completion,
     execute_deep_research,
@@ -37,6 +40,7 @@
 
 __all__ = [
     "DeepResearchError",
+    "ResearchTimeoutError",
     "ResearchConfig",
     "ProgressPhase",
     "ProgressStatus",

src/compendiumscribe/research/config.py

@@ -24,8 +24,16 @@ class ResearchConfig:
     )
     use_prompt_refinement: bool = True
     background: bool = True
-    poll_interval_seconds: float = 5.0
-    max_poll_attempts: int = 240
+    polling_interval_seconds: float = field(
+        default_factory=lambda: float(
+            os.getenv("POLLING_INTERVAL_IN_SECONDS", "10.0")
+        )
+    )
+    max_poll_time_minutes: float = field(
+        default_factory=lambda: float(
+            os.getenv("MAX_POLL_TIME_IN_MINUTES", "60.0")
+        )
+    )
     enable_code_interpreter: bool = True
     use_web_search: bool = True
     max_tool_calls: int | None = None

src/compendiumscribe/research/errors.py

@@ -9,4 +9,16 @@ class MissingConfigurationError(RuntimeError):
     """Raised when required configuration is missing."""
 
 
-__all__ = ["DeepResearchError", "MissingConfigurationError"]
+class ResearchTimeoutError(DeepResearchError):
+    """Raised when deep research exceeds the configured time limit."""
+
+    def __init__(self, message: str, research_id: str):
+        super().__init__(message)
+        self.research_id = research_id
+
+
+__all__ = [
+    "DeepResearchError",
+    "MissingConfigurationError",
+    "ResearchTimeoutError",
+]

src/compendiumscribe/research/execution/polling.py

@@ -4,7 +4,7 @@
 import time
 
 from ..config import ResearchConfig
-from ..errors import DeepResearchError
+from ..errors import DeepResearchError, ResearchTimeoutError
 from ..progress import emit_progress
 from ..utils import coerce_optional_string, get_field
 
@@ -27,9 +27,20 @@ def await_completion(
     )
 
     current = response
-    while attempts < config.max_poll_attempts:
-        time.sleep(config.poll_interval_seconds)
+    start_time = time.monotonic()
+    max_seconds = config.max_poll_time_minutes * 60
+
+    while True:
+        elapsed_seconds = int(time.monotonic() - start_time)
+        if elapsed_seconds > max_seconds:
+            raise ResearchTimeoutError(
+                f"Deep research did not complete within the {config.max_poll_time_minutes} minute limit.",
+                research_id=response.id,
+            )
+
+        time.sleep(config.polling_interval_seconds)
         attempts += 1
+        elapsed_seconds = int(time.monotonic() - start_time)
 
         current = client.responses.retrieve(response.id)
         status = coerce_optional_string(get_field(current, "status"))
@@ -40,7 +51,10 @@ def await_completion(
                 phase="deep_research",
                 status="completed",
                 message="Deep research run finished; decoding payload.",
-                metadata={"status": status},
+                metadata={
+                    "status": status,
+                    "elapsed_seconds": elapsed_seconds,
+                },
             )
             break
 
@@ -57,11 +71,8 @@ def await_completion(
             metadata={
                 "status": status,
                 "poll_attempt": attempts,
+                "elapsed_seconds": elapsed_seconds,
             },
         )
-    else:
-        raise DeepResearchError(
-            "Deep research did not complete within the polling limit."
-        )
 
     return current

tests/research/execution/test_core.py

@@ -104,8 +104,8 @@ def retrieve(self, response_id: str):
 
     config = ResearchConfig(
         background=True,
-        poll_interval_seconds=0,
-        max_poll_attempts=3,
+        polling_interval_seconds=0,
+        max_poll_time_minutes=1,
         progress_callback=callback,
     )
 
@@ -123,3 +123,34 @@ def retrieve(self, response_id: str):
         "completed",
         "Deep research run finished; decoding payload.",
     ) in progress_updates
+
+
+def test_execute_deep_research_raises_timeout_error():
+    pending = SimpleNamespace(
+        id="resp_poll",
+        status="in_progress",
+        output=[],
+    )
+
+    class FastPollingResponses:
+        def create(self, **kwargs):
+            return pending
+
+        def retrieve(self, response_id: str):
+            return pending
+
+    responses = FastPollingResponses()
+    client = SimpleNamespace(responses=responses)
+
+    # Set a very short timeout and interval
+    config = ResearchConfig(
+        background=True,
+        polling_interval_seconds=0.01,
+        max_poll_time_minutes=0.0001,  # Fraction of a second
+    )
+
+    from compendiumscribe.research.errors import ResearchTimeoutError
+    with pytest.raises(ResearchTimeoutError) as excinfo:
+        execute_deep_research(client, "prompt", config)
+    
+    assert excinfo.value.research_id == "resp_poll"