feat: ElevenLabs voice agent KB sync, system prompt injection, and EN/DE auto-language

amethystani · claude · amethystani · commit 860c0452fc88 · 2026-03-17T03:00:39.000+05:30
- New Netlify function /api/elevenlabs-kb-sync: syncs all document chunks to ElevenLabs agent KB (full replace strategy)
- New utility buildElevenLabsSystemPrompt: builds voice-optimized system prompt from KnowledgeBaseConfig (persona, categories, priority rules, custom instructions, guidelines + EN/DE auto-detect)
- ragService: fires syncKbToElevenLabs after processDocument() and deleteDocument() succeed (non-fatal, fire-and-forget)
- UserPhoneInterface: injects KB system prompt and greeting as ElevenLabs session overrides on call start

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/netlify/functions/elevenlabs-kb-sync.mts b/netlify/functions/elevenlabs-kb-sync.mts
@@ -0,0 +1,228 @@
+import type { Context, Config } from "@netlify/functions";
+
+const EL_BASE = "https://api.elevenlabs.io/v1";
+
+const CORS_HEADERS = {
+  "Access-Control-Allow-Origin": "*",
+  "Access-Control-Allow-Methods": "POST, OPTIONS",
+  "Access-Control-Allow-Headers": "Content-Type",
+  "Content-Type": "application/json",
+};
+
+export default async (req: Request, _context: Context) => {
+  // Handle CORS preflight
+  if (req.method === "OPTIONS") {
+    return new Response(null, { status: 204, headers: CORS_HEADERS });
+  }
+
+  if (req.method !== "POST") {
+    return new Response(JSON.stringify({ error: "Method not allowed" }), {
+      status: 405,
+      headers: CORS_HEADERS,
+    });
+  }
+
+  // Read env vars
+  const apiKey =
+    Netlify.env.get("ELEVENLABS_API_KEY") ||
+    Netlify.env.get("VITE_ELEVENLABS_API_KEY") ||
+    process.env.ELEVENLABS_API_KEY ||
+    process.env.VITE_ELEVENLABS_API_KEY;
+
+  if (!apiKey) {
+    return new Response(
+      JSON.stringify({ error: "ELEVENLABS_API_KEY not configured" }),
+      { status: 500, headers: CORS_HEADERS }
+    );
+  }
+
+  const agentId =
+    Netlify.env.get("ELEVENLABS_AGENT_ID") ||
+    Netlify.env.get("VITE_ELEVENLABS_AGENT_ID") ||
+    process.env.ELEVENLABS_AGENT_ID ||
+    process.env.VITE_ELEVENLABS_AGENT_ID;
+
+  if (!agentId) {
+    return new Response(
+      JSON.stringify({
+        error:
+          "ELEVENLABS_AGENT_ID not configured. Set it as an env var.",
+      }),
+      { status: 500, headers: CORS_HEADERS }
+    );
+  }
+
+  const supabaseUrl =
+    Netlify.env.get("VITE_SUPABASE_URL") ||
+    Netlify.env.get("SUPABASE_URL") ||
+    process.env.VITE_SUPABASE_URL ||
+    process.env.SUPABASE_URL;
+
+  if (!supabaseUrl) {
+    return new Response(
+      JSON.stringify({ error: "SUPABASE_URL not configured" }),
+      { status: 500, headers: CORS_HEADERS }
+    );
+  }
+
+  // Never use VITE_ prefix for service role key — Vite would inline it into the client bundle
+  const supabaseServiceKey =
+    Netlify.env.get("SUPABASE_SERVICE_ROLE_KEY") ||
+    process.env.SUPABASE_SERVICE_ROLE_KEY;
+
+  if (!supabaseServiceKey) {
+    return new Response(
+      JSON.stringify({ error: "SUPABASE_SERVICE_ROLE_KEY not configured" }),
+      { status: 500, headers: CORS_HEADERS }
+    );
+  }
+
+  // Parse request body
+  let kbId: string | undefined;
+  try {
+    const body = await req.json();
+    kbId = body.kbId;
+  } catch {
+    return new Response(
+      JSON.stringify({ error: "Invalid or missing JSON body" }),
+      { status: 400, headers: CORS_HEADERS }
+    );
+  }
+
+  if (!kbId) {
+    return new Response(
+      JSON.stringify({ error: "Missing required field: kbId" }),
+      { status: 400, headers: CORS_HEADERS }
+    );
+  }
+
+  try {
+    // Step 1: Fetch all chunks for this kbId from Supabase
+    const supabaseRes = await fetch(
+      `${supabaseUrl}/rest/v1/kb_documents?kb_id=eq.${encodeURIComponent(kbId)}&select=content,chunk_index&order=chunk_index.asc`,
+      {
+        headers: {
+          apikey: supabaseServiceKey,
+          Authorization: `Bearer ${supabaseServiceKey}`,
+          "Content-Type": "application/json",
+        },
+      }
+    );
+
+    if (!supabaseRes.ok) {
+      const errorText = await supabaseRes.text();
+      throw new Error(
+        `Supabase fetch failed (HTTP ${supabaseRes.status}): ${errorText}`
+      );
+    }
+
+    const chunks: { content: string; chunk_index: number }[] =
+      await supabaseRes.json();
+
+    // Step 2: Fetch current agent config to find existing KB item IDs
+    const agentRes = await fetch(`${EL_BASE}/convai/agents/${agentId}`, {
+      headers: { "xi-api-key": apiKey },
+    });
+
+    if (!agentRes.ok) {
+      const errorText = await agentRes.text();
+      throw new Error(
+        `Failed to fetch ElevenLabs agent config (HTTP ${agentRes.status}): ${errorText}`
+      );
+    }
+
+    const agentData = await agentRes.json();
+    const existingKbItems: { id: string }[] =
+      agentData?.conversation_config?.agent?.prompt?.knowledge_base ?? [];
+
+    // Step 3: Delete all existing KB items from the agent
+    for (const item of existingKbItems) {
+      const delRes = await fetch(
+        `${EL_BASE}/convai/agents/${agentId}/knowledge-base/${item.id}`,
+        {
+          method: "DELETE",
+          headers: { "xi-api-key": apiKey },
+        }
+      );
+      if (!delRes.ok) {
+        const errorText = await delRes.text();
+        throw new Error(
+          `Failed to delete KB item ${item.id} (HTTP ${delRes.status}): ${errorText}`
+        );
+      }
+    }
+
+    // Step 4: If no chunks, return early
+    if (chunks.length === 0) {
+      return new Response(
+        JSON.stringify({ ok: true, chunks_synced: 0 }),
+        { status: 200, headers: CORS_HEADERS }
+      );
+    }
+
+    // Step 5: Combine all chunks into one text blob
+    const combinedText = chunks.map((c) => c.content).join("\n\n");
+
+    // Step 6: Upload combined text as a new KB document to ElevenLabs
+    const textBlob = new Blob([combinedText], { type: "text/plain" });
+    const formData = new FormData();
+    formData.append("file", textBlob, `kb-${kbId}.txt`);
+    formData.append("name", `kb-${kbId}`);
+
+    const uploadRes = await fetch(`${EL_BASE}/convai/knowledge-base`, {
+      method: "POST",
+      headers: { "xi-api-key": apiKey },
+      body: formData,
+    });
+
+    if (!uploadRes.ok) {
+      const errorText = await uploadRes.text();
+      throw new Error(
+        `Failed to upload KB document to ElevenLabs (HTTP ${uploadRes.status}): ${errorText}`
+      );
+    }
+
+    const uploadData = await uploadRes.json();
+    const kbDocId: string = uploadData.id;
+
+    // Step 7: Attach the new KB doc to the agent
+    const patchRes = await fetch(`${EL_BASE}/convai/agents/${agentId}`, {
+      method: "PATCH",
+      headers: {
+        "xi-api-key": apiKey,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        conversation_config: {
+          agent: {
+            prompt: {
+              knowledge_base: [{ type: "file", id: kbDocId }],
+            },
+          },
+        },
+      }),
+    });
+
+    if (!patchRes.ok) {
+      const errorText = await patchRes.text();
+      throw new Error(
+        `Failed to attach KB document to ElevenLabs agent (HTTP ${patchRes.status}): ${errorText}`
+      );
+    }
+
+    return new Response(
+      JSON.stringify({ ok: true, chunks_synced: chunks.length, kb_doc_id: kbDocId }),
+      { status: 200, headers: CORS_HEADERS }
+    );
+  } catch (error) {
+    console.error("elevenlabs-kb-sync error:", error);
+    return new Response(
+      JSON.stringify({ error: "Internal server error", details: String(error) }),
+      { status: 500, headers: CORS_HEADERS }
+    );
+  }
+};
+
+export const config: Config = {
+  path: "/api/elevenlabs-kb-sync",
+};
diff --git a/src/components/DemoCall/UserPhoneInterface.tsx b/src/components/DemoCall/UserPhoneInterface.tsx
@@ -5,6 +5,7 @@ import { useConversation } from '@elevenlabs/react';
 import { cn } from '../../utils/cn';
 import { useDemoCall } from '../../contexts/DemoCallContext';
 import { fetchElevenLabsSignedUrl } from '../../services/elevenlabsSignedUrl';
+import { buildElevenLabsSystemPrompt } from '../../services/elevenLabsSystemPrompt';
 
 interface UserPhoneInterfaceProps {
     isDark?: boolean;
@@ -27,6 +28,7 @@ export default function UserPhoneInterface({
         startCall,
         endCall,
         addMessage,
+        knowledgeBase,
     } = useDemoCall();
 
     const [callDuration, setCallDuration] = useState(0);
@@ -130,17 +132,28 @@ export default function UserPhoneInterface({
             // Get signed URL from our server (keeps API key safe)
             const signedUrl = await getSignedUrl();
 
-            // Start ElevenLabs Conversational AI session
-            // This single WebSocket connection handles STT + LLM + TTS
+            // Build the system prompt from the current knowledge base config.
+            // This mirrors what Groq gets, keeping voice and text fully in sync.
+            const systemPrompt = buildElevenLabsSystemPrompt(knowledgeBase);
+
             await conversation.startSession({
                 signedUrl,
+                overrides: {
+                    agent: {
+                        prompt: {
+                            prompt: systemPrompt,
+                        },
+                        // Use the KB greeting as the agent's opening line
+                        first_message: knowledgeBase.greeting || undefined,
+                    },
+                },
             });
         } catch (error) {
             console.error('📞 Failed to start ElevenLabs session:', error);
             setConnectionError(error instanceof Error ? error.message : 'Failed to connect');
             setAgentStatus('idle');
         }
-    }, [startCall, getSignedUrl, conversation]);
+    }, [startCall, getSignedUrl, conversation, knowledgeBase]);
 
     const handleEndCall = useCallback(async () => {
         console.log('🔴 End call button pressed');
diff --git a/src/services/ragService.ts b/src/services/ragService.ts
@@ -476,10 +476,10 @@ class RAGService {
 
             if (error) throw error;
 
-      // Sync updated KB to ElevenLabs (fire-and-forget, non-blocking)
-      if (doc?.kb_id) {
-        this.syncKbToElevenLabs((doc as any).kb_id).catch(() => {/* already logged inside */});
-      }
+            // Sync updated KB to ElevenLabs (fire-and-forget, non-blocking)
+            if ((doc as any)?.kb_id) {
+                this.syncKbToElevenLabs((doc as any).kb_id).catch(() => {/* already logged inside */});
+            }
 
             return true;
         } catch (error) {
@@ -488,26 +488,26 @@ class RAGService {
         }
     }
 
-  // ─── Sync documents to ElevenLabs agent KB ───────────────────────
-  async syncKbToElevenLabs(kbId: string): Promise<void> {
-    try {
-      const res = await fetch('/api/elevenlabs-kb-sync', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ kbId }),
-      });
-      if (!res.ok) {
-        const err = await res.text();
-        console.warn('[ragService] ElevenLabs KB sync failed (non-fatal):', err);
-      } else {
-        const data = await res.json();
-        console.log(`[ragService] ElevenLabs KB synced: ${data.chunks_synced ?? 0} chunks`);
-      }
-    } catch (e) {
-      // Non-fatal — voice KB sync failure should never break the document upload UX
-      console.warn('[ragService] ElevenLabs KB sync error (non-fatal):', e);
+    // ─── Sync documents to ElevenLabs agent KB ───────────────────────
+    async syncKbToElevenLabs(kbId: string): Promise<void> {
+        try {
+            const res = await fetch('/api/elevenlabs-kb-sync', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ kbId }),
+            });
+            if (!res.ok) {
+                const err = await res.text();
+                console.warn('[ragService] ElevenLabs KB sync failed (non-fatal):', err);
+            } else {
+                const data = await res.json();
+                console.log(`[ragService] ElevenLabs KB synced: ${data.chunks_synced ?? 0} chunks`);
+            }
+        } catch (e) {
+            // Non-fatal — voice KB sync failure should never break the document upload UX
+            console.warn('[ragService] ElevenLabs KB sync error (non-fatal):', e);
+        }
     }
-  }
 }
 
 export const ragService = new RAGService();
