Tanvir/add integrations for vercel and fai (#4831)

Author: tsbhangu

servers/oculus/.env.example

@@ -1,4 +1,18 @@
+# Required for all integrations
 ANTHROPIC_API_KEY=your-api-key-here
 OPENAI_API_KEY=your-api-key-here
 TURBOPUFFER_API_KEY=your-api-key-here
-FERN_TOKEN=your_fern_token
+FERN_TOKEN=your_fern_token
+
+# Integration selection (optional, defaults to fai-local)
+# Options: fai-local, fai-http, vercel-http
+OCULUS_INTEGRATION=fai-local
+
+# FAI HTTP integration (required if using fai-http)
+FAI_URL=https://fai.buildwithfern.com
+
+# Vercel HTTP integration (required if using vercel-http)
+# Production: https://buildwithfern.com
+# Staging: https://fern.docs.staging.buildwithfern.com
+# Preview: https://preview-xyz.vercel.app
+VERCEL_URL=https://buildwithfern.com

servers/oculus/README.md

@@ -3,9 +3,55 @@
 The `oculus` subrepository is used to generate and run evaluations on Ask Fern. There are three main components:
 
 - `generators` create an evaluation set (can either be programmatic or stochastic).
-- `integrations` invoke various forms of the `/chat` endpoint (API, Slack, Discord, etc.).
+- `integrations` invoke various forms of the answer generation system to get responses.
 - `evaluators` run on output responses to measure performance on various metrics.
 
+## Integration Types
+
+Oculus supports multiple integration types for generating answers:
+
+### `fai-local` (Default)
+- Calls FAI functions directly in-process (Python imports)
+- **Fastest** - No network overhead
+- **Best for:** Development, rapid iteration, large-scale experiments
+- **Tests:** FAI code logic only (not production deployment)
+
+### `fai-http`
+- Calls FAI `/chat/{domain}` HTTP endpoint
+- **Medium speed** - Some network overhead
+- **Best for:** Testing FAI service specifically, validating FAI API
+- **Tests:** FAI service deployment (but not full production stack)
+
+### `vercel-http`
+- Calls Vercel `/api/fern-docs/search/v2/chat` endpoint (configured via `VERCEL_URL`)
+- **Slowest** - Full production stack with all middleware
+- **Best for:** Production validation, pre-release testing, staging tests
+- **Tests:** Exact production experience (auth, tool calling, etc.)
+- **Requires:** `VERCEL_URL` environment variable
+
+### Selecting Integration
+
+**Via environment variable:**
+```bash
+export OCULUS_INTEGRATION=vercel-http
+oculus answer --suite simple
+```
+
+**Via CLI argument:**
+```bash
+oculus answer --suite simple --integration fai-http
+oculus run --suite simple --integration vercel-http
+```
+
+**Comparison workflow:**
+```bash
+# Run same eval with all three integrations
+for integration in fai-local fai-http vercel-http; do
+  oculus answer --suite simple --run-id ${integration}_test --integration $integration
+  oculus evaluate --suite simple --run-id ${integration}_test
+done
+```
+
 ## Commands
 
 ### Generate questions for a suite
@@ -14,12 +60,17 @@ The `oculus` subrepository is used to generate and run evaluations on Ask Fern.
 
 ### Run full evaluation pipeline (generate answers + evaluate)
 
-    oculus run --suite {suite_name} [--run-id {id}] [--model {model}]
+    oculus run --suite {suite_name} [--run-id {id}] [--integration {type}] [--model {model}]
 
 ### Generate answers only
 
-    oculus answer --suite {suite_name} [--run-id {id}] [--model {model}]
+    oculus answer --suite {suite_name} [--run-id {id}] [--integration {type}] [--model {model}]
 
 ### Evaluate existing answers
 
     oculus evaluate --suite {suite_name} --run-id {id} [--judge-model {model}]
+
+**Options:**
+- `--integration`: One of `fai-local`, `fai-http`, `vercel-http` (defaults to `OCULUS_INTEGRATION` env var or `fai-local`)
+- `--model`: One of `claude-4-sonnet-20250514`, `command-a-03-2025` (default: `claude-4-sonnet-20250514`)
+- `--judge-model`: Claude model for evaluation judging (default: `claude-opus-4-20250514`)

servers/oculus/pyproject.toml

@@ -32,6 +32,12 @@ oculus = "oculus.__main__:main"
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+
 [tool.mypy]
 python_version = "3.11"
 strict = true

servers/oculus/src/oculus/__main__.py

@@ -14,7 +14,8 @@
     SuiteConfig,
 )
 from oculus.framework.runner import EvaluationRunner
-from oculus.integrations.fai_integration import create_fai_answer_function
+from oculus.integrations.base import create_answer_function
+from oculus.integrations.factory import create_integration, get_default_integration_type
 from oculus.utils.file_utils import (
     load_json,
     save_json,
@@ -99,8 +100,12 @@ def generate_answers_command(args: argparse.Namespace) -> int:
         return 1
 
     try:
-        print(f"Initializing FAI integration for domain: {suite_config.domain}")
-        answer_fn = create_fai_answer_function(domain=suite_config.domain, model=args.model)
+        integration_type = args.integration if hasattr(args, "integration") else get_default_integration_type()
+        print(f"Initializing {integration_type} integration for domain: {suite_config.domain}")
+        integration = create_integration(
+            integration_type=integration_type, domain=suite_config.domain, model=args.model
+        )
+        answer_fn = create_answer_function(integration)
 
         runner = EvaluationRunner(
             suite_name=args.suite,
@@ -304,8 +309,12 @@ def run_evaluation(args: argparse.Namespace) -> int:
         return 1
 
     try:
-        print(f"Initializing FAI integration for domain: {suite_config.domain}")
-        answer_fn = create_fai_answer_function(domain=suite_config.domain, model=args.model)
+        integration_type = args.integration if hasattr(args, "integration") else get_default_integration_type()
+        print(f"Initializing {integration_type} integration for domain: {suite_config.domain}")
+        integration = create_integration(
+            integration_type=integration_type, domain=suite_config.domain, model=args.model
+        )
+        answer_fn = create_answer_function(integration)
 
         runner = EvaluationRunner(
             suite_name=args.suite,
@@ -385,6 +394,13 @@ def main() -> int:
     answer_parser.add_argument("--suite", type=str, required=True, help="Name of the evaluation suite")
     answer_parser.add_argument("--suite-path", type=Path, default=None, help="Base path to suites directory")
     answer_parser.add_argument("--run-id", type=str, default=None, help="Unique run identifier")
+    answer_parser.add_argument(
+        "--integration",
+        type=str,
+        default=None,
+        choices=["fai-local", "fai-http", "vercel-http"],
+        help="Integration type (defaults to OCULUS_INTEGRATION env var or fai-local)",
+    )
     answer_parser.add_argument(
         "--model",
         type=str,
@@ -431,6 +447,13 @@ def main() -> int:
     run_parser.add_argument("--suite", type=str, required=True, help="Name of the evaluation suite")
     run_parser.add_argument("--suite-path", type=Path, default=None, help="Base path to suites directory")
     run_parser.add_argument("--run-id", type=str, default=None, help="Unique run identifier")
+    run_parser.add_argument(
+        "--integration",
+        type=str,
+        default=None,
+        choices=["fai-local", "fai-http", "vercel-http"],
+        help="Integration type (defaults to OCULUS_INTEGRATION env var or fai-local)",
+    )
     run_parser.add_argument(
         "--model",
         type=str,

servers/oculus/src/oculus/generators/endpoints.py

@@ -202,7 +202,10 @@ def generate_openapi_questions(
 
                 completed += 1
                 if num_questions is not None:
-                    print(f"Progress: {len(questions)}/{num_questions} - Generated questions for {method.upper()} {path}")
+                    print(
+                        f"Progress: {len(questions)}/{num_questions} - "
+                        f"Generated questions for {method.upper()} {path}"
+                    )
                 else:
                     print(f"Progress: {completed}/{total_endpoints} - Generated questions for {method.upper()} {path}")
 

servers/oculus/src/oculus/integrations/base.py

@@ -0,0 +1,79 @@
+import json
+from collections.abc import Callable
+from typing import Protocol, TypedDict
+
+
+class SourceMetadata(TypedDict, total=False):
+    """Normalized source metadata across integrations."""
+
+    title: str | None
+    url: str | None
+    slug: str | None
+
+
+class AnswerMetadata(TypedDict, total=False):
+    """
+    Common metadata structure for all integrations.
+
+    Required fields:
+    - integration_type: The type of integration used
+    - model: The model used to generate the answer
+    - sources: List of source documents used
+
+    Optional fields:
+    - fai_local_retrieved_documents: Full JSON with retrieval scores (FAI local only)
+    - fai_http_citations: List of citation URLs (FAI HTTP only)
+    - vercel_query_id: Query tracking ID (Vercel only)
+    - vercel_tool_calls: Number of tool calls made (Vercel only)
+    - response_time_ms: Time taken to generate response
+    """
+
+    # Required
+    integration_type: str
+    model: str
+    sources: list[SourceMetadata]
+
+    # Optional - Integration-specific
+    fai_local_retrieved_documents: str
+    fai_http_citations: list[str]
+    vercel_query_id: str
+    vercel_tool_calls: int
+
+    # Optional - Timing
+    response_time_ms: float
+
+
+class AnswerIntegration(Protocol):
+    """Protocol for generating answers from different sources."""
+
+    def generate_answer(self, question: str) -> tuple[str, AnswerMetadata]:
+        """
+        Generate an answer for the given question.
+
+        Args:
+            question: The question to answer
+
+        Returns:
+            A tuple of (answer_text, metadata)
+            - answer_text: The generated answer
+            - metadata: Metadata about the answer following AnswerMetadata schema
+        """
+        ...
+
+
+def create_answer_function(
+    integration: AnswerIntegration,
+) -> Callable[[str], tuple[str, dict[str, str]]]:
+    """Helper to create a simple callable from an integration."""
+
+    def answer_fn(question: str) -> tuple[str, dict[str, str]]:
+        answer, metadata = integration.generate_answer(question)
+        result = {}
+        for k, v in metadata.items():
+            if isinstance(v, list | dict):
+                result[k] = json.dumps(v)
+            else:
+                result[k] = str(v)
+        return answer, result
+
+    return answer_fn

servers/oculus/src/oculus/integrations/factory.py

@@ -0,0 +1,80 @@
+import os
+
+from oculus.integrations.base import AnswerIntegration
+from oculus.integrations.fai_http import FAIHTTPIntegration
+from oculus.integrations.fai_local import FAILocalIntegration
+from oculus.integrations.vercel_http import VercelHTTPIntegration
+
+INTEGRATION_TYPES = ["fai-local", "fai-http", "vercel-http"]
+
+
+def get_default_integration_type() -> str:
+    return os.environ.get("OCULUS_INTEGRATION", "fai-local")
+
+
+def create_integration(
+    integration_type: str | None = None,
+    domain: str = "",
+    model: str = "claude-4-sonnet-20250514",
+    system_prompt: str | None = None,
+    **kwargs: object,
+) -> AnswerIntegration:
+    """
+    Create an answer integration based on type.
+
+    Args:
+        integration_type: Type of integration ("fai-local", "fai-http", "vercel-http")
+                         If None, uses OCULUS_INTEGRATION env var or defaults to "fai-local"
+        domain: Documentation domain
+        model: Model to use for generation
+        system_prompt: Optional system prompt override
+        **kwargs: Additional integration-specific arguments
+
+    Returns:
+        An AnswerIntegration instance
+
+    Raises:
+        ValueError: If integration_type is not supported
+
+    Environment Variables:
+        OCULUS_INTEGRATION: Default integration type (if integration_type not specified)
+        FAI_URL: Base URL for FAI service (for fai-http)
+        FERN_TOKEN: Auth token for FAI (for fai-http)
+        VERCEL_URL: Base URL for Vercel docs site (for vercel-http, required)
+    """
+    # Use default if not specified
+    if integration_type is None:
+        integration_type = get_default_integration_type()
+
+    integration_type = integration_type.lower()
+
+    if integration_type == "fai-local":
+        return FAILocalIntegration(domain=domain, model=model, system_prompt=system_prompt)
+
+    elif integration_type == "fai-http":
+        fai_url_kwarg = kwargs.get("fai_url")
+        fai_url = (fai_url_kwarg if isinstance(fai_url_kwarg, str) else None) or os.environ.get("FAI_URL")
+        fern_token_kwarg = kwargs.get("fern_token")
+        fern_token = (fern_token_kwarg if isinstance(fern_token_kwarg, str) else None) or os.environ.get("FERN_TOKEN")
+        return FAIHTTPIntegration(
+            domain=domain,
+            model=model,
+            system_prompt=system_prompt,
+            fai_url=fai_url,
+            fern_token=fern_token,
+        )
+
+    elif integration_type == "vercel-http":
+        vercel_url_kwarg = kwargs.get("vercel_url")
+        vercel_url = (vercel_url_kwarg if isinstance(vercel_url_kwarg, str) else None) or os.environ.get("VERCEL_URL")
+        return VercelHTTPIntegration(
+            domain=domain,
+            model=model,
+            system_prompt=system_prompt,
+            vercel_url=vercel_url,
+        )
+
+    else:
+        raise ValueError(
+            f"Unsupported integration type: {integration_type}. " f"Supported types: {', '.join(INTEGRATION_TYPES)}"
+        )