feat: add AI task extraction for notetaker recordings

lucasheriques · lucasheriques · commit ab840fc72ac1 · 2025-10-27T14:55:46.000-03:00
- Add task extraction IPC handler using OpenAI GPT-4o-mini
- Create useExtractTasks hook with TanStack Query mutation
- Add Extract Tasks button in LiveTranscriptView
- Display extracted tasks with title and description
- Show loading, error, and empty states
- Require OpenAI API key in settings

This allows users to extract actionable tasks from meeting transcripts
with a single click, perfect for the demo today!
diff --git a/src/main/preload.ts b/src/main/preload.ts
@@ -202,6 +202,11 @@ contextBridge.exposeInMainWorld("electronAPI", {
     ipcRenderer.invoke("notetaker:get-recording", recordingId),
   notetakerDeleteRecording: (recordingId: string): Promise<void> =>
     ipcRenderer.invoke("notetaker:delete-recording", recordingId),
+  notetakerExtractTasks: (
+    transcriptText: string,
+    openaiApiKey: string,
+  ): Promise<Array<{ title: string; description: string }>> =>
+    ipcRenderer.invoke("notetaker:extract-tasks", transcriptText, openaiApiKey),
   // Real-time transcript listener
   onTranscriptSegment: (
     listener: (segment: {
diff --git a/src/main/services/recallRecording.ts b/src/main/services/recallRecording.ts
@@ -1,6 +1,10 @@
+import { createOpenAI } from "@ai-sdk/openai";
 import { PostHogAPIClient } from "@api/posthogClient";
 import RecallAiSdk from "@recallai/desktop-sdk";
+import { generateObject } from "ai";
 import { ipcMain } from "electron";
+import { z } from "zod";
+import { TASK_EXTRACTION_PROMPT } from "./transcription-prompts";
 
 interface RecordingSession {
   windowId: string;
@@ -428,6 +432,45 @@ export function getActiveSessions() {
   return Array.from(activeSessions.values());
 }
 
+async function extractTasksFromTranscript(
+  transcriptText: string,
+  openaiApiKey: string,
+): Promise<Array<{ title: string; description: string }>> {
+  try {
+    const openai = createOpenAI({ apiKey: openaiApiKey });
+
+    const schema = z.object({
+      tasks: z.array(
+        z.object({
+          title: z.string().describe("Brief task title"),
+          description: z.string().describe("Detailed description with context"),
+        }),
+      ),
+    });
+
+    const { object } = await generateObject({
+      model: openai("gpt-4o-mini"),
+      schema,
+      messages: [
+        {
+          role: "system",
+          content:
+            "You are a helpful assistant that extracts actionable tasks from conversation transcripts. Be generous in identifying work items - include feature requests, requirements, and any work that needs to be done.",
+        },
+        {
+          role: "user",
+          content: `${TASK_EXTRACTION_PROMPT}\n${transcriptText}`,
+        },
+      ],
+    });
+
+    return object.tasks || [];
+  } catch (error) {
+    console.error("[Task Extraction] Error:", error);
+    throw error;
+  }
+}
+
 export function registerRecallIPCHandlers() {
   ipcMain.handle(
     "recall:initialize",
@@ -483,4 +526,17 @@ export function registerRecallIPCHandlers() {
     }
     return await posthogClient.deleteDesktopRecording(recordingId);
   });
+
+  ipcMain.handle(
+    "notetaker:extract-tasks",
+    async (_event, transcriptText, openaiApiKey) => {
+      console.log("[Task Extraction] Starting task extraction...");
+      const tasks = await extractTasksFromTranscript(
+        transcriptText,
+        openaiApiKey,
+      );
+      console.log(`[Task Extraction] Extracted ${tasks.length} tasks`);
+      return tasks;
+    },
+  );
 }
diff --git a/src/renderer/features/notetaker/components/LiveTranscriptView.tsx b/src/renderer/features/notetaker/components/LiveTranscriptView.tsx
@@ -1,5 +1,18 @@
-import { Badge, Box, Card, Flex, ScrollArea, Text } from "@radix-ui/themes";
+import { ListChecksIcon, SparkleIcon } from "@phosphor-icons/react";
+import {
+  Badge,
+  Box,
+  Button,
+  Card,
+  Flex,
+  Heading,
+  ScrollArea,
+  Separator,
+  Text,
+} from "@radix-ui/themes";
 import { useEffect, useRef, useState } from "react";
+import { useAuthStore } from "../../../stores/authStore";
+import { useExtractTasks } from "../hooks/useExtractTasks";
 import { useLiveTranscript } from "../hooks/useTranscript";
 
 interface LiveTranscriptViewProps {
@@ -11,10 +24,21 @@ export function LiveTranscriptView({
 }: LiveTranscriptViewProps) {
   const scrollRef = useRef<HTMLDivElement>(null);
   const [autoScroll, setAutoScroll] = useState(true);
+  const openaiApiKey = useAuthStore((state) => state.openaiApiKey);
 
   const { segments, addSegment, forceUpload, pendingCount } =
     useLiveTranscript(posthogRecordingId);
 
+  const extractTasksMutation = useExtractTasks();
+
+  const handleExtractTasks = () => {
+    const fullText = segments.map((s) => s.text).join(" ");
+    extractTasksMutation.mutate({
+      recordingId: posthogRecordingId,
+      transcriptText: fullText,
+    });
+  };
+
   // Auto-scroll to bottom when new segments arrive
   useEffect(() => {
     if (autoScroll && scrollRef.current && segments.length > 0) {
@@ -177,6 +201,77 @@ export function LiveTranscriptView({
           </Text>
         </Box>
       )}
+
+      {/* Extract Tasks Section */}
+      {segments.length > 0 && (
+        <>
+          <Separator size="4" my="4" />
+          <Flex direction="column" gap="3">
+            <Flex justify="between" align="center">
+              <Heading size="4">
+                <Flex align="center" gap="2">
+                  <ListChecksIcon size={20} />
+                  Extracted tasks
+                </Flex>
+              </Heading>
+              <Button
+                size="2"
+                onClick={handleExtractTasks}
+                disabled={extractTasksMutation.isPending || !openaiApiKey}
+              >
+                <SparkleIcon />
+                {extractTasksMutation.isPending
+                  ? "Extracting..."
+                  : "Extract tasks"}
+              </Button>
+            </Flex>
+
+            {!openaiApiKey && (
+              <Card>
+                <Text size="2" color="gray">
+                  Add OpenAI API key in settings to extract tasks
+                </Text>
+              </Card>
+            )}
+
+            {extractTasksMutation.isSuccess && extractTasksMutation.data && (
+              <Flex direction="column" gap="2">
+                {extractTasksMutation.data.tasks.length === 0 ? (
+                  <Card>
+                    <Text size="2" color="gray">
+                      No tasks found in transcript
+                    </Text>
+                  </Card>
+                ) : (
+                  extractTasksMutation.data.tasks.map((task, idx) => (
+                    <Card key={`${task.title}-${idx}`}>
+                      <Flex direction="column" gap="1">
+                        <Text size="2" weight="medium">
+                          {task.title}
+                        </Text>
+                        <Text size="1" color="gray">
+                          {task.description}
+                        </Text>
+                      </Flex>
+                    </Card>
+                  ))
+                )}
+              </Flex>
+            )}
+
+            {extractTasksMutation.isError && (
+              <Card>
+                <Text size="2" color="red">
+                  Failed to extract tasks:{" "}
+                  {extractTasksMutation.error instanceof Error
+                    ? extractTasksMutation.error.message
+                    : "Unknown error"}
+                </Text>
+              </Card>
+            )}
+          </Flex>
+        </>
+      )}
     </Box>
   );
 }
diff --git a/src/renderer/features/notetaker/hooks/useExtractTasks.ts b/src/renderer/features/notetaker/hooks/useExtractTasks.ts
@@ -0,0 +1,42 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { useAuthStore } from "../../../stores/authStore";
+
+export interface ExtractedTask {
+  title: string;
+  description: string;
+}
+
+export function useExtractTasks() {
+  const openaiApiKey = useAuthStore((state) => state.openaiApiKey);
+  const queryClient = useQueryClient();
+
+  return useMutation({
+    mutationFn: async ({
+      recordingId,
+      transcriptText,
+    }: {
+      recordingId: string;
+      transcriptText: string;
+    }) => {
+      if (!openaiApiKey) {
+        throw new Error("OpenAI API key not configured");
+      }
+
+      const tasks = await window.electronAPI.notetakerExtractTasks(
+        transcriptText,
+        openaiApiKey,
+      );
+
+      return { tasks, recordingId };
+    },
+    onSuccess: (data) => {
+      // Invalidate recording query to refetch with new tasks
+      queryClient.invalidateQueries({
+        queryKey: ["notetaker-recording", data.recordingId],
+      });
+      queryClient.invalidateQueries({
+        queryKey: ["notetaker-transcript", data.recordingId],
+      });
+    },
+  });
+}
diff --git a/src/renderer/types/electron.d.ts b/src/renderer/types/electron.d.ts
@@ -148,6 +148,10 @@ export interface IElectronAPI {
     recall_recording_id: string | null;
   }>;
   notetakerDeleteRecording: (recordingId: string) => Promise<void>;
+  notetakerExtractTasks: (
+    transcriptText: string,
+    openaiApiKey: string,
+  ) => Promise<Array<{ title: string; description: string }>>;
   // Real-time transcript listener
   onTranscriptSegment: (
     listener: (segment: {
