feat(oculus): Implement initial oculus evals framework. (#4715)

Author: eyw520

servers/oculus/.env.example

@@ -0,0 +1,3 @@
+ANTHROPIC_API_KEY=your-api-key-here
+OPENAI_API_KEY=your-api-key-here
+TURBOPUFFER_API_KEY=your-api-key-here

servers/oculus/.gitignore

@@ -0,0 +1,61 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Poetry
+poetry.lock
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Evaluation outputs
+suites/*/answers/
+suites/*/evals/
+suites/*/results_*.json
+
+# Environment variables
+.env
+.env.local
+!.env.example
+
+# OS
+.DS_Store
+Thumbs.db

servers/oculus/logging.conf

@@ -0,0 +1,21 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=console
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[handler_console]
+class=StreamHandler
+level=WARNING
+formatter=simple
+args=(sys.stdout,)
+
+[formatter_simple]
+format=%(levelname)s: %(message)s

servers/oculus/makefile

@@ -0,0 +1,14 @@
+install:
+	poetry install
+
+test:
+	poetry run pytest -sv
+
+typecheck:
+	poetry run mypy src/
+
+lint:
+	poetry run ruff check src/
+
+format:
+	poetry run ruff format src/

servers/oculus/poetry.lock

@@ -0,0 +1,46 @@
+[tool.poetry]
+name = "oculus"
+version = "0.1.0"
+description = "Ask Fern Evaluation Pipeline"
+authors = []
+packages = [{include = "oculus", from = "src"}]
+
+[[tool.poetry.source]]
+name = "fern"
+url = "https://pypi.buildwithfern.com"
+priority = "supplemental"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+anthropic = "^0.57.1"
+pydantic = "^2.8.0"
+pyyaml = "^6.0.1"
+python-dotenv = "^1.0.0"
+fern-ai = {path = "../fai", develop = true}
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.0.0"
+mypy = "^1.8.0"
+ruff = "^0.3.0"
+
+[tool.poetry.scripts]
+oculus = "oculus.__main__:main"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+mypy_path = "src"
+explicit_package_bases = true

servers/oculus/pyproject.toml

@@ -0,0 +1,110 @@
+import argparse
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from oculus.framework.runner import EvaluationRunner
+from oculus.integrations.fai_integration import create_fai_answer_function
+
+load_dotenv()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Run Ask Fern evaluations using LLM-as-a-judge",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  oculus --suite retrieval_quality --domain buildwithfern.com
+  oculus --suite answer_quality --domain docs.cohere.com --run-id experiment_1
+  oculus --suite test --domain example.com --model command-a-03-2025
+  oculus --suite test --domain example.com --no-skip-existing
+        """,
+    )
+
+    parser.add_argument("--suite", type=str, required=True, help="Name of the evaluation suite")
+    parser.add_argument("--domain", type=str, required=True, help="Documentation domain to query")
+    parser.add_argument("--suite-path", type=Path, default=None, help="Base path to suites directory")
+    parser.add_argument("--run-id", type=str, default=None, help="Unique run identifier")
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="claude-4-sonnet-20250514",
+        choices=["claude-4-sonnet-20250514", "command-a-03-2025"],
+        help="Model to use for answer generation",
+    )
+    parser.add_argument("--judge-model", type=str, default="claude-opus-4-20250514", help="Claude model for judging")
+    parser.add_argument("--max-workers", type=int, default=16, help="Number of parallel workers")
+    parser.add_argument("--no-skip-existing", action="store_true", help="Re-generate existing answers/evaluations")
+    parser.add_argument("--output-dir", type=Path, default=None, help="Directory to save results")
+
+    args = parser.parse_args()
+
+    if args.suite_path:
+        suite_base = args.suite_path
+    else:
+        suite_base = Path.cwd() / "suites"
+
+    suite_path = suite_base / args.suite
+
+    if not suite_path.exists():
+        print(f"Error: Suite directory not found: {suite_path}", file=sys.stderr)
+        print(f"\nExpected structure:", file=sys.stderr)
+        print(f"  {suite_path}/", file=sys.stderr)
+        print(f"    questions/", file=sys.stderr)
+        print(f"      question_0.json", file=sys.stderr)
+        return 1
+
+    questions_dir = suite_path / "questions"
+    if not questions_dir.exists() or not any(questions_dir.glob("*.json")):
+        print(f"Error: No questions found in {questions_dir}", file=sys.stderr)
+        return 1
+
+    try:
+        print(f"Initializing FAI integration for domain: {args.domain}")
+        answer_fn = create_fai_answer_function(domain=args.domain, model=args.model)
+
+        runner = EvaluationRunner(
+            suite_name=args.suite,
+            suite_path=suite_path,
+            run_id=args.run_id,
+            max_workers=args.max_workers,
+        )
+
+        result = runner.run(
+            answer_fn=answer_fn,
+            model_name=args.model,
+            judge_model=args.judge_model,
+            skip_existing=not args.no_skip_existing,
+        )
+
+        if args.output_dir:
+            from oculus.utils.file_utils import save_json
+
+            args.output_dir.mkdir(parents=True, exist_ok=True)
+            output_path = args.output_dir / f"results_{result.run_id}.json"
+            save_json(output_path, result.model_dump())
+            print(f"\nAdditional output saved to: {output_path}")
+
+        return 0
+
+    except ImportError as e:
+        print(f"\nError: Failed to import required modules", file=sys.stderr)
+        print(f"{e}", file=sys.stderr)
+        print(f"\nMake sure:", file=sys.stderr)
+        print(f"  1. FAI dependencies are installed (poetry install in servers/fai)", file=sys.stderr)
+        print(f"  2. PYTHONPATH includes the FAI source directory", file=sys.stderr)
+        return 1
+
+    except Exception as e:
+        print(f"\nError: Evaluation failed", file=sys.stderr)
+        print(f"{e}", file=sys.stderr)
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

servers/oculus/src/oculus/__main__.py

@@ -0,0 +1,69 @@
+import time
+from typing import Any, Optional, Type, TypeVar, cast
+
+from anthropic import Anthropic
+from pydantic import BaseModel
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def generate_with_claude(
+    response_type: Type[T],
+    prompt_template: str,
+    model: str = "claude-opus-4-20250514",
+    max_tokens: int = 1000,
+    max_retries: int = 3,
+    **kwargs: str,
+) -> Optional[T]:
+    anthropic_client = Anthropic()
+    formatted_prompt = prompt_template.format(**kwargs)
+
+    tools = [
+        {
+            "name": "build_response_result",
+            "description": "Build the structured response object.",
+            "input_schema": response_type.model_json_schema(),
+        }
+    ]
+
+    tries = 0
+    while tries < max_retries:
+        try:
+            response = anthropic_client.messages.create(
+                model=model,
+                max_tokens=max_tokens,
+                messages=[{"role": "user", "content": formatted_prompt}],
+                tools=cast(Any, tools),
+                tool_choice=cast(Any, {"type": "tool", "name": "build_response_result"}),
+            )
+
+            function_call = response.content[0].input  # type: ignore[union-attr]
+            parsed_response = response_type(**cast(dict[str, Any], function_call))
+            return parsed_response
+
+        except Exception as e:
+            tries += 1
+            if tries >= max_retries:
+                print(f"Failed after {max_retries} attempts: {e}")
+                return None
+            time.sleep(0.5 * tries)
+
+    return None
+
+
+def evaluate_answer(
+    question: str,
+    answer: str,
+    ground_truth: str,
+    model: str = "claude-opus-4-20250514",
+) -> Optional[Any]:
+    from oculus.framework.models import EVALUATION_PROMPT_TEMPLATE, EvaluationResponse
+
+    return generate_with_claude(
+        response_type=EvaluationResponse,
+        prompt_template=EVALUATION_PROMPT_TEMPLATE,
+        model=model,
+        question=question,
+        answer=answer,
+        ground_truth=ground_truth,
+    )

servers/oculus/src/oculus/framework/judge.py

@@ -0,0 +1,62 @@
+from pydantic import BaseModel, Field
+
+
+class EvaluationResponse(BaseModel):
+    is_correct: bool
+    reason: str
+
+
+EVALUATION_PROMPT_TEMPLATE = """You are evaluating the correctness of an AI assistant's answer to a technical question about API documentation.
+
+Question: {question}
+
+Ground Truth / Expected Information:
+{ground_truth}
+
+AI Assistant's Answer:
+{answer}
+
+Evaluate whether the AI assistant's answer is correct and complete based on the ground truth. The answer should:
+1. Accurately represent the information in the ground truth
+2. Not include significant hallucinations or incorrect information
+3. Address the core question being asked
+
+If the answer is mostly correct with minor issues, mark it as correct. Only mark as incorrect if there are significant errors or omissions.
+
+Provide your evaluation with a brief reason."""
+
+
+class Question(BaseModel):
+    question: str
+    ground_truth: str
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+
+class Answer(BaseModel):
+    question: str
+    answer: str
+    model: str
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+
+class Evaluation(BaseModel):
+    question: str
+    answer: str
+    ground_truth: str
+    is_correct: bool
+    reason: str
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+
+class EvaluationRun(BaseModel):
+    run_id: str
+    timestamp: str
+    suite: str
+    results: list[Evaluation]
+    metrics: "EvaluationMetrics"
+
+
+class EvaluationMetrics(BaseModel):
+    total_questions: int
+    total_correct: int
+    accuracy: float