Merge pull request #339 from open-edge-platform/update-branch

feat: implement wake word detection service (#887)

Author: gooishin

usecases/ai/edge-ai-demo-studio/.gitignore

@@ -20,6 +20,7 @@ __pycache__
 /workers/lipsync/build
 
 /workers/embedding/data
+/workers/wake-word-detection/data
 
 /models
 

usecases/ai/edge-ai-demo-studio/README.md

@@ -19,6 +19,7 @@ Edge AI Demo Studio is a modern toolkit for deploying, managing, and serving AI
   - Lipsync
   - Image Generation
   - MCP Manager
+  - Wake Word Detection
 - **Samples:** Samples use cases that implements the ai services
   - Digital Avatar
   - RAG Chat

usecases/ai/edge-ai-demo-studio/docs/Architecture.png

@@ -9,6 +9,7 @@ import {
   SPEECH_TO_TEXT_PORT,
   TEXT_TO_SPEECH_PORT,
   IMAGE_GENERATION_PORT,
+  WAKE_WORD_DETECTION_PORT,
 } from '@/lib/constants'
 
 const nextConfig: NextConfig = {
@@ -61,6 +62,10 @@ const nextConfig: NextConfig = {
         source: '/api/images/v1/:slug*',
         destination: `http://localhost:${IMAGE_GENERATION_PORT}/v3/images/:slug*`,
       },
+      {
+        source: '/api/wake-word-detection/:slug*',
+        destination: `http://localhost:${WAKE_WORD_DETECTION_PORT}/:slug*`,
+      },
     ]
   },
 }

usecases/ai/edge-ai-demo-studio/frontend/next.config.ts

@@ -41,7 +41,7 @@
     "graphql": "^16.11.0",
     "hast-util-to-jsx-runtime": "^2.3.6",
     "lucide-react": "^0.552.0",
-    "next": "^15.5.7",
+    "next": "^15.5.9",
     "next-themes": "^0.4.6",
     "openai": "^6.7.0",
     "openvino-node": "^2025.1.0",

usecases/ai/edge-ai-demo-studio/frontend/package-lock.json

@@ -36,6 +36,8 @@ export default function DigitalAvatarPage() {
 
   const [sessionId, setSessionId] = useState('')
   const [connectionStatus, setConnectionStatus] = useState('disconnected')
+  const [useSTT, setUseSTT] = useState(false)
+  const [useDenoise, setUseDenoise] = useState(true)
   const [useEmbedding, setUseEmbedding] = useState(false)
   const [selectedKnowledgeBase, setSelectedKnowledgeBase] =
     useState<KnowledgeBase | null>(null)
@@ -45,20 +47,25 @@ export default function DigitalAvatarPage() {
   const prerequisiteServices = useMemo(() => {
     const ps = ['text-generation', 'text-to-speech', 'lipsync']
 
+    if (useSTT) ps.push('speech-to-text')
     if (useEmbedding) ps.push('embedding')
 
     return ps
-  }, [useEmbedding])
+  }, [useEmbedding, useSTT])
 
   const handleSessionIdChange = (newSessionId: string) => {
     setSessionId(newSessionId)
   }
 
   const handleSettingsUpdate = (settings: {
+    useSTT: boolean
+    useDenoise: boolean
     useEmbedding: boolean
     selectedKnowledgeBase: KnowledgeBase | null
     useMcpTools: boolean
   }) => {
+    setUseSTT(settings.useSTT)
+    setUseDenoise(settings.useDenoise)
     setUseEmbedding(settings.useEmbedding)
     setSelectedKnowledgeBase(settings.selectedKnowledgeBase)
     setUseMcpTools(settings.useMcpTools)
@@ -217,6 +224,8 @@ export default function DigitalAvatarPage() {
                 inactivePrerequisites.length > 0 ||
                 (preparingPrerequisites && preparingPrerequisites.length > 0)
               }
+              isSTTEnabled={useSTT}
+              isDenoiseEnabled={useDenoise}
               sessionId={sessionId}
               connectionStatus={connectionStatus}
               knowledgeBaseId={selectedKnowledgeBase?.id || undefined}
@@ -232,6 +241,8 @@ export default function DigitalAvatarPage() {
         <DigitalAvatarSettings
           isOpen={isSettingsOpen}
           onClose={() => setIsSettingsOpen(false)}
+          useSTT={useSTT}
+          useDenoise={useDenoise}
           useEmbedding={useEmbedding}
           selectedKnowledgeBase={selectedKnowledgeBase}
           useMcpTools={useMcpTools}

usecases/ai/edge-ai-demo-studio/frontend/package.json

@@ -0,0 +1,153 @@
+// Copyright (C) 2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+'use client'
+
+import WorkloadComponent from '@/components/workloads/workload'
+import {
+  useCreateWorkload,
+  useGetWorkloadByType,
+  useUpdateWorkload,
+} from '@/hooks/use-workload'
+
+import useDisclosure from '@/hooks/use-disclosure'
+import {
+  Model,
+  SettingsModal,
+} from '@/components/workloads/wake-word-detection/settings'
+import { Button } from '@/components/ui/button'
+import { Settings } from 'lucide-react'
+import {
+  WAKE_WORD_DETECTION_WORKLOAD,
+  WAKE_WORD_DETECTION_TYPE,
+  WAKE_WORD_DETECTION_DESCRIPTION,
+  WAKE_WORD_DETECTION_MODELS,
+} from '@/lib/workloads/wake-word-detection'
+import WakeWordDetectionDemo from '@/components/workloads/wake-word-detection/demo'
+import Logs from '@/components/workloads/log'
+import {
+  DocumentationProps,
+  DocumentationTemplate,
+} from '@/components/workloads/documentation'
+import WakeWordDetectionDocumentation from '@/components/workloads/wake-word-detection/documentation'
+import { WAKE_WORD_DETECTION_PORT } from '@/lib/constants'
+import Endpoint from '@/components/workloads/endpoint'
+import { wakeWordDetectionEndpoints } from '@/components/workloads/wake-word-detection/api'
+
+const TYPE = WAKE_WORD_DETECTION_TYPE
+const DESCRIPTION = WAKE_WORD_DETECTION_DESCRIPTION
+
+export default function TextGenerationPage() {
+  const { data: workload, isLoading } = useGetWorkloadByType(
+    'wake-word-detection',
+  )
+  const { isOpen, onClose, onOpen } = useDisclosure()
+
+  const updateWorkload = useUpdateWorkload()
+  const createWorkload = useCreateWorkload()
+
+  const data: DocumentationProps = {
+    overview: (
+      <WakeWordDetectionDocumentation port={WAKE_WORD_DETECTION_PORT} />
+    ),
+    endpoints: (
+      <Endpoint
+        apis={wakeWordDetectionEndpoints}
+        port={WAKE_WORD_DETECTION_PORT}
+      />
+    ),
+  }
+
+  const updateSettings = (models: Model[], vadThreshold: number) => {
+    return new Promise((resolve, reject) => {
+      const modelString = models.map((model) => model.value).join(' ')
+
+      if (!workload) {
+        createWorkload.mutate(
+          {
+            ...WAKE_WORD_DETECTION_WORKLOAD,
+            model: modelString,
+            metadata: {
+              vadThreshold,
+            },
+            status: 'inactive',
+          },
+          {
+            onSuccess: () => resolve(true),
+            onError: (error) => {
+              console.error('Failed to create workload:', error)
+              reject(error)
+            },
+          },
+        )
+      } else if (workload && !isLoading) {
+        updateWorkload.mutateAsync(
+          {
+            id: workload?.id || 0,
+            data: {
+              model: modelString,
+              metadata: {
+                vadThreshold,
+              },
+            },
+          },
+          {
+            onSuccess: () => resolve(true),
+            onError: (error) => {
+              console.error('Failed to update workload:', error)
+              reject(error)
+            },
+          },
+        )
+      } else {
+        resolve(true)
+      }
+    })
+  }
+
+  const SettingsButton = () => {
+    return (
+      <Button
+        variant="secondary"
+        size="icon"
+        className="size-8"
+        onClick={onOpen}
+      >
+        <Settings />
+      </Button>
+    )
+  }
+
+  return (
+    <>
+      <SettingsModal
+        task="Wake Word Detection"
+        isOpen={isOpen}
+        onClose={onClose}
+        predefinedModels={WAKE_WORD_DETECTION_MODELS}
+        updateSettings={updateSettings}
+        selectedModel={workload?.model || WAKE_WORD_DETECTION_WORKLOAD.model}
+        vadThreshold={
+          workload?.metadata?.vadThreshold ??
+          WAKE_WORD_DETECTION_WORKLOAD.metadata.vadThreshold
+        }
+        workloadStatus={workload?.status}
+      />
+      <WorkloadComponent
+        title="Wake Word Detection"
+        settingsButton={<SettingsButton />}
+        workload={workload}
+        description={DESCRIPTION}
+        workloadType={TYPE}
+        demoElement={
+          <WakeWordDetectionDemo
+            disabled={!workload || workload.status !== 'active'}
+          />
+        }
+        docsElement={<DocumentationTemplate data={data} />}
+        logsElement={<Logs name={`${workload?.name}_${workload?.id}`} />}
+        isLoading={isLoading}
+      />
+    </>
+  )
+}

usecases/ai/edge-ai-demo-studio/frontend/src/app/(frontend)/(samples)/digital-avatar/page.tsx

@@ -183,13 +183,21 @@ class SentenceHandler {
     // const wordCount = countWords(sentence)
     // console.log(`Sentence (${wordCount} words):`, sentence)
 
+    // Remove special characters and emojis
+    const processedSentence = sentence
+      .replace(
+        /[*#]|[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F700}-\u{1F77F}\u{1F780}-\u{1F7FF}\u{1F800}-\u{1F8FF}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA6F}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu,
+        '',
+      )
+      .trim()
+
     fetch(`http://localhost:${LIPSYNC_PORT}/v1/lipsync/chat`, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
       body: JSON.stringify({
-        text: sentence,
+        text: processedSentence,
         session_id: sessionId,
         chat_type: 'echo',
         voice: voice,

usecases/ai/edge-ai-demo-studio/frontend/src/app/(frontend)/(services)/wake-word-detection/page.tsx

@@ -0,0 +1,94 @@
+// Copyright (C) 2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+import { NextRequest } from 'next/server'
+
+const sseClients = new Set<ReadableStreamDefaultController>()
+
+function broadcastEvent(data: {
+  type: 'detection' | 'clear' | 'connected'
+  event?: { timestamp: string; model: string; score: number }
+}) {
+  const message = `data: ${JSON.stringify(data)}\n\n`
+  sseClients.forEach((controller) => {
+    try {
+      controller.enqueue(new TextEncoder().encode(message))
+    } catch (error) {
+      console.error('Failed to send SSE message:', error)
+      sseClients.delete(controller)
+    }
+  })
+}
+
+export async function GET() {
+  const stream = new ReadableStream({
+    start(controller) {
+      sseClients.add(controller)
+
+      const message = `data: ${JSON.stringify({ type: 'connected' })}\n\n`
+      controller.enqueue(new TextEncoder().encode(message))
+
+      // Keep-alive ping every 30 seconds
+      const keepAliveInterval = setInterval(() => {
+        try {
+          controller.enqueue(new TextEncoder().encode(': keep-alive\n\n'))
+        } catch (error) {
+          console.error('Failed to send keep-alive ping:', error)
+          clearInterval(keepAliveInterval)
+          sseClients.delete(controller)
+        }
+      }, 30000)
+
+      return () => {
+        clearInterval(keepAliveInterval)
+        sseClients.delete(controller)
+      }
+    },
+  })
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      Connection: 'keep-alive',
+    },
+  })
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const body = await request.json()
+    console.log('Received detection:', JSON.stringify(body))
+
+    // Create detection event
+    const event = {
+      timestamp: new Date().toISOString(),
+      model: body.model || 'Unknown',
+      score: body.score || 0,
+    }
+
+    // Broadcast to all connected SSE clients
+    broadcastEvent({ type: 'detection', event })
+
+    return Response.json({
+      success: true,
+      message: 'Wake word detected successfully',
+      event,
+    })
+  } catch (error) {
+    let errorMessage = 'Failed to process detection'
+    if (error instanceof Error) {
+      errorMessage = error.message
+    }
+    console.error(errorMessage, error)
+
+    return Response.json(
+      {
+        success: false,
+        message: errorMessage,
+        data: null,
+      },
+      { status: 500 },
+    )
+  }
+}