feat(oculus): Implement initial end-to-end pipeline for oculus. (#4740)

Author: eyw520

biome.json

@@ -322,6 +322,7 @@
         "packages/fdr-lambda-sdk/**",
         "fern/apis/fai/openapi.json",
         "**/.local-sdk-output/**",
+        "servers/oculus/**",
         "servers/fai/**",
         "servers/fai-discord/**",
         "servers/fai-lambda/**",

servers/oculus/.env.example

@@ -1,3 +1,4 @@
 ANTHROPIC_API_KEY=your-api-key-here
 OPENAI_API_KEY=your-api-key-here
-TURBOPUFFER_API_KEY=your-api-key-here
+TURBOPUFFER_API_KEY=your-api-key-here
+FERN_TOKEN=your_fern_token

servers/oculus/README.md

@@ -0,0 +1,25 @@
+## Oculus
+
+The `oculus` subrepository is used to generate and run evaluations on Ask Fern. There are three main components:
+
+- `generators` create an evaluation set (can either be programmatic or stochastic).
+- `integrations` invoke various forms of the `/chat` endpoint (API, Slack, Discord, etc.).
+- `evaluators` run on output responses to measure performance on various metrics.
+
+## Commands
+
+### Generate questions for a suite
+
+    oculus generate --suite {suite_name}
+
+### Run full evaluation pipeline (generate answers + evaluate)
+
+    oculus run --suite {suite_name} [--run-id {id}] [--model {model}]
+
+### Generate answers only
+
+    oculus answer --suite {suite_name} [--run-id {id}] [--model {model}]
+
+### Evaluate existing answers
+
+    oculus evaluate --suite {suite_name} --run-id {id} [--judge-model {model}]

servers/oculus/pyproject.toml

@@ -55,7 +55,7 @@ select = [
 ignore = []
 
 [tool.ruff.lint.isort]
-known-first-party = ["fai", "tests", "utils"]
+known-first-party = ["oculus"]
 
 [tool.ruff.format]
 quote-style = "double"
@@ -71,7 +71,7 @@ force_grid_wrap = 2
 line_length = 120
 profile = "black"
 use_parentheses = true
-known_first_party = ["fai", "tests", "utils"]
+known_first_party = ["oculus"]
 src_paths = ["src"]
 
 [tool.autoflake]