feat(chat): instrument LLM usage

pan93412 · pan93412 · commit d4ef043280f0 · 2025-10-06T20:49:30.000+08:00
diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts
@@ -1,10 +1,12 @@
 import { graphql } from "@/gql";
 import { getClient } from "@/lib/apollo.rsc";
-import { checkAuthorizedStatus } from "@/lib/auth.rsc";
+import { getAuthorizedUserInfo } from "@/lib/auth.rsc";
 import { anthropic, type AnthropicProviderOptions } from "@ai-sdk/anthropic";
 import { convertToModelMessages, stepCountIs, streamText, tool, type UIMessage } from "ai";
+import { withTracing } from "@posthog/ai"
 import { NextResponse } from "next/server";
 import { z } from "zod";
+import { createPostHogClient } from "@/lib/posthog.rsc";
 
 export const maxDuration = 30;
 
@@ -72,8 +74,8 @@ interface ChatRouteRequest {
 }
 
 export async function POST(req: Request) {
-  const authorized = await checkAuthorizedStatus(["*", "ai"]);
-  if (!authorized) {
+  const userInfo = await getAuthorizedUserInfo(["*", "ai"]);
+  if (!userInfo) {
     return new NextResponse("Unauthorized", { status: 401 });
   }
 
@@ -98,104 +100,115 @@ export async function POST(req: Request) {
     .replace("{{QUESTION_DIFFICULTY}}", data.question.difficulty)
     .replace("{{QUESTION_CATEGORY}}", data.question.category);
 
-  const result = streamText({
-    model: anthropic("claude-sonnet-4-20250514"),
-    providerOptions: {
-      anthropic: {
-        thinking: { type: "enabled", budgetTokens: 12000 },
-      } satisfies AnthropicProviderOptions,
-    },
-    messages: convertToModelMessages(messages),
-    system: preparedPrompt,
-    stopWhen: stepCountIs(10),
-    tools: {
-      getMyAnswer: tool({
-        description:
-          "取得使用者最後提交的答案結果，包括查詢結果、錯誤訊息和狀態。如果使用者問關於他們的答案，使用這個工具。",
-        inputSchema: z.object({}),
-        execute: async () => {
-          const { data, error } = await apollo.query({
-            query: USER_ANSWER_RESULT,
-            variables: { id: questionId },
-            errorPolicy: "all",
-          });
-          if (!data?.question) {
-            return { error: "無法取得題目資料", details: error?.message };
-          }
-
-          const { lastSubmission } = data.question;
-          if (!lastSubmission) {
-            return { error: "使用者尚未提交答案" };
-          }
-
-          return {
-            status: lastSubmission.status,
-            submittedCode: lastSubmission.submittedCode,
-            queryResult: lastSubmission.queryResult
-              ? {
-                columns: lastSubmission.queryResult.columns,
-                rows: lastSubmission.queryResult.rows,
-              }
-              : null,
-            error: lastSubmission.error,
-          };
-        },
-      }),
-      getCorrectAnswer: tool({
-        description: "取得題目的正確答案結果，你可以對照和使用者的答案差異。",
-        inputSchema: z.object({}),
-        execute: async () => {
-          const { data, error } = await apollo.query({
-            query: CORRECT_ANSWER_RESULT,
-            variables: { id: questionId },
-            errorPolicy: "all",
-          });
-          if (!data?.question) {
-            return { error: "無法取得題目資料", details: error?.message };
-          }
-
-          return {
-            queryResult: data.question.referenceAnswerResult
-              ? {
-                columns: data.question.referenceAnswerResult.columns,
-                rows: data.question.referenceAnswerResult.rows,
-              }
-              : null,
-          };
-        },
-      }),
-      getQuestionSchema: tool({
-        description: "取得題目的資料庫結構，你可以用這個數據輔助了解 SQL 結構。",
-        inputSchema: z.object({}),
-        execute: async () => {
-          const { data, error } = await apollo.query({
-            query: QUESTION_SCHEMA,
-            variables: { id: questionId },
-            errorPolicy: "all",
-          });
-          if (!data?.question) {
-            return { error: "無法取得題目資料", details: error?.message };
-          }
-
-          return {
-            schema: data.question.database.structure
-              ? {
-                tables: data.question.database.structure.tables.map((table) => ({
-                  name: table.name,
-                  columns: table.columns,
-                })),
-              }
-              : null,
-          };
-        },
-      }),
-      webSearch: anthropic.tools.webSearch_20250305({
-        maxUses: 5,
-      }),
-    },
-  });
-
-  return result.toUIMessageStreamResponse();
+  const model = anthropic("claude-sonnet-4-20250514");
+  const posthogClient = await createPostHogClient();
+
+  try {
+    const tracedModel = withTracing(model, posthogClient, {
+      posthogDistinctId: userInfo.sub,
+    });
+
+    const result = streamText({
+      model: tracedModel,
+      providerOptions: {
+        anthropic: {
+          thinking: { type: "enabled", budgetTokens: 12000 },
+        } satisfies AnthropicProviderOptions,
+      },
+      messages: convertToModelMessages(messages),
+      system: preparedPrompt,
+      stopWhen: stepCountIs(10),
+      tools: {
+        getMyAnswer: tool({
+          description:
+            "取得使用者最後提交的答案結果，包括查詢結果、錯誤訊息和狀態。如果使用者問關於他們的答案，使用這個工具。",
+          inputSchema: z.object({}),
+          execute: async () => {
+            const { data, error } = await apollo.query({
+              query: USER_ANSWER_RESULT,
+              variables: { id: questionId },
+              errorPolicy: "all",
+            });
+            if (!data?.question) {
+              return { error: "無法取得題目資料", details: error?.message };
+            }
+
+            const { lastSubmission } = data.question;
+            if (!lastSubmission) {
+              return { error: "使用者尚未提交答案" };
+            }
+
+            return {
+              status: lastSubmission.status,
+              submittedCode: lastSubmission.submittedCode,
+              queryResult: lastSubmission.queryResult
+                ? {
+                  columns: lastSubmission.queryResult.columns,
+                  rows: lastSubmission.queryResult.rows,
+                }
+                : null,
+              error: lastSubmission.error,
+            };
+          },
+        }),
+        getCorrectAnswer: tool({
+          description: "取得題目的正確答案結果，你可以對照和使用者的答案差異。",
+          inputSchema: z.object({}),
+          execute: async () => {
+            const { data, error } = await apollo.query({
+              query: CORRECT_ANSWER_RESULT,
+              variables: { id: questionId },
+              errorPolicy: "all",
+            });
+            if (!data?.question) {
+              return { error: "無法取得題目資料", details: error?.message };
+            }
+
+            return {
+              queryResult: data.question.referenceAnswerResult
+                ? {
+                  columns: data.question.referenceAnswerResult.columns,
+                  rows: data.question.referenceAnswerResult.rows,
+                }
+                : null,
+            };
+          },
+        }),
+        getQuestionSchema: tool({
+          description: "取得題目的資料庫結構，你可以用這個數據輔助了解 SQL 結構。",
+          inputSchema: z.object({}),
+          execute: async () => {
+            const { data, error } = await apollo.query({
+              query: QUESTION_SCHEMA,
+              variables: { id: questionId },
+              errorPolicy: "all",
+            });
+            if (!data?.question) {
+              return { error: "無法取得題目資料", details: error?.message };
+            }
+
+            return {
+              schema: data.question.database.structure
+                ? {
+                  tables: data.question.database.structure.tables.map((table) => ({
+                    name: table.name,
+                    columns: table.columns,
+                  })),
+                }
+                : null,
+            };
+          },
+        }),
+        webSearch: anthropic.tools.webSearch_20250305({
+          maxUses: 5,
+        }),
+      },
+    });
+
+    return result.toUIMessageStreamResponse();
+  } finally {
+    await posthogClient.shutdown();
+  }
 }
 
 export const prompt =
diff --git a/lib/auth.rsc.ts b/lib/auth.rsc.ts
@@ -17,28 +17,28 @@ export async function redirectIfAuthenticated(): Promise<void> {
   redirect("/");
 }
 
-export async function checkAuthorizedStatus(requiredScopes?: string[]): Promise<boolean> {
+export async function getAuthorizedUserInfo(requiredScopes?: string[]) {
   const token = await getAuthToken();
   if (!token) {
-    return false;
+    return null;
   }
 
   const authStatus = await getAuthStatus(token);
 
   if (!authStatus.loggedIn || !authStatus.introspectResult?.active) {
-    return false;
+    return null;
   }
 
   // check if the token has the required scope
   if (requiredScopes) {
     for (const scope of requiredScopes) {
       if (authStatus.introspectResult?.scope.includes(scope)) {
-        return true;
+        return authStatus.introspectResult;
       }
     }
 
-    return false;
+    return null;
   }
 
-  return true;
+  return authStatus.introspectResult;
 }
diff --git a/lib/posthog.rsc.ts b/lib/posthog.rsc.ts
@@ -0,0 +1,13 @@
+"use server";
+
+import { PostHog } from 'posthog-node'
+
+export async function createPostHogClient() {
+  const posthogClient = new PostHog(process.env.NEXT_PUBLIC_POSTHOG_KEY!, {
+    host: process.env.NEXT_PUBLIC_POSTHOG_HOST,
+    flushAt: 1,
+    flushInterval: 0
+  })
+
+  return posthogClient
+}

Original file line number	Diff line number	Diff line change
`@@ -17,28 +17,28 @@ export async function redirectIfAuthenticated(): Promise<void> {`
`17`	`17`	`redirect("/");`
`18`	`18`	`}`
`19`	`19`
`20`		`-export async function checkAuthorizedStatus(requiredScopes?: string[]): Promise<boolean> {`
	`20`	`+export async function getAuthorizedUserInfo(requiredScopes?: string[]) {`
`21`	`21`	`const token = await getAuthToken();`
`22`	`22`	`if (!token) {`
`23`		`- return false;`
	`23`	`+ return null;`
`24`	`24`	`}`
`25`	`25`
`26`	`26`	`const authStatus = await getAuthStatus(token);`
`27`	`27`
`28`	`28`	`if (!authStatus.loggedIn \|\| !authStatus.introspectResult?.active) {`
`29`		`- return false;`
	`29`	`+ return null;`
`30`	`30`	`}`
`31`	`31`
`32`	`32`	`// check if the token has the required scope`
`33`	`33`	`if (requiredScopes) {`
`34`	`34`	`for (const scope of requiredScopes) {`
`35`	`35`	`if (authStatus.introspectResult?.scope.includes(scope)) {`
`36`		`- return true;`
	`36`	`+ return authStatus.introspectResult;`
`37`	`37`	`}`
`38`	`38`	`}`
`39`	`39`
`40`		`- return false;`
	`40`	`+ return null;`
`41`	`41`	`}`
`42`	`42`
`43`		`- return true;`
	`43`	`+ return authStatus.introspectResult;`
`44`	`44`	`}`