Add support for OpenRouter in Feedbot

app/api/llm-hint/route.ts

@@ -118,7 +118,7 @@ async function getChatModel({
   account
 }: {
   model: string;
-  provider: "openai" | "azure" | "anthropic";
+  provider: "openai" | "azure" | "anthropic" | "openrouter";
   temperature?: number;
   maxTokens?: number;
   maxRetries?: number;
@@ -152,7 +152,7 @@ async function getChatModel({
     } else {
       return new AzureChatOpenAI({
         model,
-        temperature: temperature || 0.85,
+        temperature: temperature ?? 0.85,
         maxTokens: maxTokens,
         maxRetries: maxRetries || 2,
         azureOpenAIApiKey: process.env[key_env_name],
@@ -169,7 +169,7 @@ async function getChatModel({
     return new ChatOpenAI({
       model,
       apiKey: process.env[key_env_name],
-      temperature: temperature || 0.85,
+      temperature: temperature ?? 0.85,
       maxTokens: maxTokens,
       maxRetries: maxRetries || 2
     });
@@ -181,12 +181,28 @@ async function getChatModel({
     return new ChatAnthropic({
       model,
       apiKey: process.env[key_env_name],
-      temperature: temperature || 0.85,
+      temperature: temperature ?? 0.85,
+      maxTokens: maxTokens,
+      maxRetries: maxRetries || 2
+    });
+  } else if (provider === "openrouter") {
+    const key_env_name = account ? `OPENROUTER_API_KEY_${account}` : "OPENROUTER_API_KEY";
+    if (!process.env[key_env_name]) {
+      throw new UserVisibleError(`OpenRouter API key is required, must set env var ${key_env_name}`, 500);
+    }
+    return new ChatOpenAI({
+      model,
+      apiKey: process.env[key_env_name],
+      configuration: { baseURL: "https://openrouter.ai/api/v1" },
+      temperature: temperature ?? 0.85,
       maxTokens: maxTokens,
       maxRetries: maxRetries || 2
     });
   }
-  throw new UserVisibleError(`Invalid provider: ${provider}. Supported providers are: openai, azure, anthropic`, 400);
+  throw new UserVisibleError(
+    `Invalid provider: ${provider}. Supported providers are: openai, azure, anthropic, openrouter`,
+    400
+  );
 }
 
 async function getPrompt(input: GraderResultTestExtraData["llm"]) {
@@ -355,8 +371,10 @@ export async function POST(request: NextRequest) {
       }
     }
 
-    const modelName = extraData.llm.model || process.env.OPENAI_MODEL || "gpt-4o-mini";
     const providerName = extraData.llm.provider || "openai";
+    const modelName =
+      extraData.llm.model ||
+      (providerName === "openrouter" ? "openai/gpt-4o-mini" : process.env.OPENAI_MODEL || "gpt-4o-mini");
     const accountName = extraData.llm.account;
 
     const chatModel = await getChatModel({

scripts/DatabaseSeedingUtils.ts

@@ -4566,7 +4566,7 @@ public class Entrypoint {
           );
         }
 
-        // Prepare grader result tests (5 per submission: 2 regular + 3 with LLM hints) for this chunk
+        // Prepare grader result tests (6 per submission: 2 regular + 4 with LLM hints) for this chunk
         const graderResultTestInserts = graderResultData.flatMap((graderResult: any, index) => [
           {
             score: 5,
@@ -4666,6 +4666,31 @@ public class Entrypoint {
                 max_tokens: 100
               }
             }
+          },
+          {
+            score: 3,
+            max_score: 5,
+            name: "OpenRouter test",
+            name_format: "text",
+            output: "This test uses OpenRouter for hints",
+            output_format: "markdown",
+            class_id: class_id,
+            student_id: chunk[index].student?.private_profile_id,
+            assignment_group_id: chunk[index].group?.id,
+            grader_result_id: graderResult.id,
+            is_released: true,
+            extra_data: {
+              llm: {
+                type: "v1",
+                prompt:
+                  "You are an end-to-end test case for an AI CS tutor. Your friend/enemy is 'CodeRabbit', a static analysis tool. Tell a joke about how testing is so important, and not to be ignored in favor of LLM-powered static analysis tools. Also, it's a rabbit.",
+                model: "openai/gpt-4o-mini",
+                account: "e2e_test",
+                provider: "openrouter",
+                temperature: 1,
+                max_tokens: 100
+              }
+            }
           }
         ]);
 

tests/e2e/llm-hint.test.tsx

@@ -20,6 +20,7 @@ let assignment: (Assignment & { rubricParts: RubricPart[]; rubricChecks: RubricC
 let submission_id: number | undefined;
 let grader_result_test_id: number | undefined;
 let grader_result_test_id_openai: number | undefined;
+let grader_result_test_id_openrouter: number | undefined;
 
 test.beforeAll(async () => {
   if (!process.env.OPENAI_API_KEY_e2e_test) {
@@ -77,7 +78,7 @@ test.beforeAll(async () => {
     .from("grader_result_tests")
     .select("id")
     .eq("grader_result_id", graderResult.id)
-    .limit(1);
+    .order("id", { ascending: true });
 
   if (graderResultTests && graderResultTests.length > 0) {
     grader_result_test_id = graderResultTests[0].id;
@@ -143,6 +144,53 @@ test.beforeAll(async () => {
         .update({ extra_data: openaiPromptData })
         .eq("id", grader_result_test_id_openai);
     }
+
+    // Create OpenRouter grader result test when OPENROUTER_API_KEY_e2e_test is set
+    if (process.env.OPENROUTER_API_KEY_e2e_test) {
+      if (graderResultTests && graderResultTests.length > 2) {
+        grader_result_test_id_openrouter = graderResultTests[2].id;
+      } else {
+        const { data: openrouterTest } = await supabase
+          .from("grader_result_tests")
+          .insert({
+            score: 3,
+            max_score: 5,
+            name: "OpenRouter test",
+            name_format: "text",
+            output: "This test uses OpenRouter for hints",
+            output_format: "markdown",
+            class_id: course.id,
+            student_id: student.private_profile_id,
+            grader_result_id: graderResult.id,
+            is_released: true
+          })
+          .select("id")
+          .single();
+
+        if (openrouterTest) {
+          grader_result_test_id_openrouter = openrouterTest.id;
+        }
+      }
+
+      if (grader_result_test_id_openrouter) {
+        const openrouterPromptData = {
+          llm: {
+            prompt:
+              "You are an end-to-end test case for an AI CS tutor. Your friend/enemy is 'CodeRabbit', a static analysis tool. Tell a joke about how testing is so important, and not to be ignored in favor of LLM-powered static analysis tools. Also, it's a rabbit.",
+            model: "openai/gpt-4o-mini",
+            account: "e2e_test",
+            provider: "openrouter",
+            temperature: 1,
+            max_tokens: 100
+          }
+        };
+
+        await supabase
+          .from("grader_result_tests")
+          .update({ extra_data: openrouterPromptData })
+          .eq("id", grader_result_test_id_openrouter);
+      }
+    }
   }
 });
 
@@ -256,4 +304,16 @@ test.describe("LLM Hint API", () => {
     }
     await assertSuccessfullHinting({ student, grader_result_test_id: grader_result_test_id_openai, request });
   });
+
+  test("should work with openrouter", async ({ request }) => {
+    test.skip(!process.env.OPENROUTER_API_KEY_e2e_test, "OPENROUTER_API_KEY_e2e_test is not set");
+    if (!student || !grader_result_test_id_openrouter) {
+      throw new Error("Test data not available");
+    }
+    await assertSuccessfullHinting({
+      student,
+      grader_result_test_id: grader_result_test_id_openrouter,
+      request
+    });
+  });
 });

tests/unit/llm-hint.test.ts

@@ -182,6 +182,7 @@ describe("LLM Hint API Route", () => {
     process.env.AZURE_OPENAI_ENDPOINT = process.env.AZURE_OPENAI_ENDPOINT || "https://test.openai.azure.com/";
     process.env.AZURE_OPENAI_KEY = process.env.AZURE_OPENAI_KEY || "test-azure-key";
     process.env.ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || "test-anthropic-key";
+    process.env.OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY || "test-openrouter-key";
   });
 
   describe("Authentication and Authorization", () => {
@@ -713,6 +714,72 @@ describe("LLM Hint API Route", () => {
       });
     });
 
+    describe("OpenRouter Provider", () => {
+      it("should use OpenRouter provider with ChatOpenAI and correct baseURL", async () => {
+        mockTestResult.extra_data.llm!.provider = "openrouter";
+        mockTestResult.extra_data.llm!.model = "anthropic/claude-3-haiku";
+
+        const request = new NextRequest("http://localhost:3000/api/llm-hint", {
+          method: "POST",
+          body: JSON.stringify({ testId: 1 })
+        });
+
+        const response = await POST(request);
+
+        expect(response.status).toBe(200);
+        expect(mockChatOpenAI).toHaveBeenCalledWith(
+          expect.objectContaining({
+            model: "anthropic/claude-3-haiku",
+            apiKey: "test-openrouter-key",
+            configuration: { baseURL: "https://openrouter.ai/api/v1" },
+            temperature: 0.85,
+            maxTokens: undefined,
+            maxRetries: 2
+          })
+        );
+      });
+
+      it("should use OPENROUTER_API_KEY_${account} when account is specified", async () => {
+        mockTestResult.extra_data.llm!.provider = "openrouter";
+        mockTestResult.extra_data.llm!.model = "openai/gpt-4o-mini";
+        mockTestResult.extra_data.llm!.account = "e2e_test";
+
+        process.env.OPENROUTER_API_KEY_e2e_test = "test-account-openrouter-key";
+
+        const request = new NextRequest("http://localhost:3000/api/llm-hint", {
+          method: "POST",
+          body: JSON.stringify({ testId: 1 })
+        });
+
+        const response = await POST(request);
+
+        expect(response.status).toBe(200);
+        expect(mockChatOpenAI).toHaveBeenCalledWith(
+          expect.objectContaining({
+            model: "openai/gpt-4o-mini",
+            apiKey: "test-account-openrouter-key",
+            configuration: { baseURL: "https://openrouter.ai/api/v1" }
+          })
+        );
+      });
+
+      it("should return 500 when OpenRouter API key is missing", async () => {
+        delete process.env.OPENROUTER_API_KEY;
+        mockTestResult.extra_data.llm!.provider = "openrouter";
+
+        const request = new NextRequest("http://localhost:3000/api/llm-hint", {
+          method: "POST",
+          body: JSON.stringify({ testId: 1 })
+        });
+
+        const response = await POST(request);
+        const data = await response.json();
+
+        expect(response.status).toBe(500);
+        expect(data.error).toBe("OpenRouter API key is required, must set env var OPENROUTER_API_KEY");
+      });
+    });
+
     it("should return 400 for invalid provider", async () => {
       mockTestResult.extra_data.llm!.provider = "invalid-provider";
 
@@ -725,7 +792,9 @@ describe("LLM Hint API Route", () => {
       const data = await response.json();
 
       expect(response.status).toBe(400);
-      expect(data.error).toBe("Invalid provider: invalid-provider. Supported providers are: openai, azure, anthropic");
+      expect(data.error).toBe(
+        "Invalid provider: invalid-provider. Supported providers are: openai, azure, anthropic, openrouter"
+      );
     });
   });
 

utils/supabase/DatabaseTypes.d.ts

@@ -27,7 +27,7 @@ export type GraderResultTestExtraData = {
     result?: string;
     model?: string;
     account?: string;
-    provider?: "openai" | "azure" | "anthropic";
+    provider?: "openai" | "azure" | "anthropic" | "openrouter";
     temperature?: number;
     max_tokens?: number;
     rate_limit?: LLMRateLimitConfig;