Merge pull request #86 from intel/update-branch

gooishin · web-flow · commit 05ed3ae01b28 · 2024-12-18T15:46:13.000+08:00
feat: fix digital avatar demo & enhance microservices dockerfile to production standard (#265)
diff --git a/usecases/ai/digital-avatar/.env.template b/usecases/ai/digital-avatar/.env.template
diff --git a/usecases/ai/digital-avatar/.gitignore b/usecases/ai/digital-avatar/.gitignore
@@ -8,5 +8,7 @@ backend/musetalk/models
 backend/musetalk/data/avatars
 backend/wav2lip/wav2lip/results
 backend/wav2lip/wav2lip/temp
+assets/*
 weights/*
-backend/liveportrait/templates
+backend/liveportrait/templates
+data/ollama
diff --git a/usecases/ai/digital-avatar/README.md b/usecases/ai/digital-avatar/README.md
@@ -85,8 +85,7 @@ You can offload model inference to specific device by modifying the environment
 | Workload             | Environment Variable |Supported Device         | 
 |----------------------|----------------------|-------------------------|
 | LLM                  |            -         |        GPU              |
-| STT - Encoded Device | STT_ENCODED_DEVICE   | CPU,GPU,NPU             | 
-| STT - Decided Device | STT_DECODED_DEVICE   | CPU,GPU                 |
+| STT                  | STT_DEVICE           | CPU,GPU,NPU             | 
 | TTS                  | TTS_DEVICE           | CPU                     |
 | Lipsync (Wav2lip)    | DEVICE               | CPU, GPU                |
 
@@ -95,14 +94,18 @@ Example Configuration:
 * To offload the STT encoded workload to `NPU`, you can use the following configuration.
 
 ```
-wav2lip:
+stt_service:
   ...
   environment:
     ...
-    DEVICE=CPU
+    STT_DEVICE=CPU
     ...
 ```
 
+## Limitations
+### 1. Automatic Speech Recognition Compatibility
+Automatic speech recognition functionality is not supported in Firefox. Please use Chrome for validated performance.
+
 ## FAQ
 ### 1. Update Render Group ID
 1. Ensure the [Intel GPU driver](#prerequisite) is installed.
@@ -116,4 +119,3 @@ wav2lip:
     ```
 4. The group ID is the number in the third field (e.g., `110` in the example above).
 5. Ensure the `RENDER_GROUP_ID` in the [docker-compose.yml](./docker-compose.yml) file matches the render group ID.
-
diff --git a/usecases/ai/digital-avatar/data/.gitkeep b/usecases/ai/digital-avatar/data/.gitkeep
diff --git a/usecases/ai/digital-avatar/docker-compose.yml b/usecases/ai/digital-avatar/docker-compose.yml
@@ -120,7 +120,7 @@ services:
     ports:
       - 8014:5996
     volumes:
-      - ./data:/usr/src/app/data:rw
+      - stt_volume:/usr/src/app/data:rw
     devices:
       - /dev:/dev:rw
       - /lib/modules:/lib/modules:rw
@@ -151,13 +151,22 @@ services:
       - /dev:/dev:rw
       - /lib/modules:/lib/modules:rw
     volumes:
-      - ../microservices/ollama:/root/.ollama
+      - ./data/ollama:/root/.ollama
+      - ./scripts/run.sh:/opt/intel/llm-app/run.sh
+      - ./scripts/healthcheck.sh:/opt/intel/llm-app/healthcheck.sh
+    healthcheck:
+      test: ["CMD", "bash", "/opt/intel/llm-app/healthcheck.sh"]
+      interval: 1m30s
+      timeout: 30s
+      retries: 5
+      start_period: 30s
     command: "./run.sh"
 
 networks:
   app-network:
 
 volumes:
+  stt_volume:
   tts_volume:
   data_volume:
   ollama:
diff --git a/usecases/ai/digital-avatar/scripts/healthcheck.sh b/usecases/ai/digital-avatar/scripts/healthcheck.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 # Ensure OLLAMA_HOST is set
 if [ -z "$OLLAMA_HOST" ]; then
   echo "Error: OLLAMA_HOST environment variable is not set."
@@ -13,13 +12,11 @@ if [ -z "$LLM_MODEL" ]; then
 fi
 
 PORT=$(echo "$OLLAMA_HOST" | cut -d':' -f2)
-
 # Capture the HTTP response code and response body separately
 http_code=$(curl -s -o response.txt -w "%{http_code}" -X POST "http://localhost:$PORT/api/show" -d "{\"model\": \"$LLM_MODEL\"}")
 response=$(cat response.txt)
 
 echo "HTTP Response Code: $http_code"
-
 if [ "$http_code" -ne 200 ]; then
   echo "Error: HTTP request failed with status code $http_code"
   rm response.txt
diff --git a/usecases/ai/digital-avatar/scripts/run.sh b/usecases/ai/digital-avatar/scripts/run.sh
@@ -29,4 +29,4 @@ curl -# http://localhost:"$PORT"/api/generate -d "{\"model\": \"$LLM_MODEL\", \"
 
 # To keep the container running, use an indefinite wait
 echo "Ollama is running. Keeping container alive..."
-tail -f /dev/null
+tail -f /dev/null
diff --git a/usecases/ai/microservices/ollama/Dockerfile b/usecases/ai/microservices/ollama/Dockerfile
@@ -1,27 +1,31 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04
+FROM intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04
 
 WORKDIR /opt/intel/llm-app
-RUN apt update \
-    && apt install -y python3.11-venv jq \
-    wget \
+RUN rm -rf /etc/apt/sources.list.d/intel-graphics.list \
+    && apt update \
+    && apt install -y python3.11 \
+        python3.11-venv \
+        wget \
     && wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg \
     && echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | tee /etc/apt/sources.list.d/intel-gpu-jammy.list \
     && apt update \
-    && apt-get install -y libze1 intel-level-zero-gpu intel-opencl-icd clinfo \
+    && apt-get install -y libze1 \
+        intel-level-zero-gpu \
+        intel-opencl-icd \
+        clinfo \
     && rm -rf /var/lib/apt/lists/* \
     && python3.11 -m venv .venv
 ENV PATH="/opt/intel/llm-app/.venv/bin:$PATH"
+ENV LD_LIBRARY_PATH=".:$LD_LIBRARY_PATH"
 RUN python3 -m pip install --upgrade pip \
-    && python3 -m pip install --pre --upgrade 'ipex-llm[cpp]==2.2.0b20240917' \
-    && python3 -m pip install bigdl-core-cpp==2.6.0b20240917 \
+    && python3 -m pip install --pre --upgrade 'ipex-llm[cpp]==2.2.0b20241216' \
     && python3 -m pip install --upgrade accelerate==0.33.0 \
     && init-ollama
 
-COPY ./run.sh .
+HEALTHCHECK --interval=60s --timeout=5m --start-period=5s --retries=5 \
+    CMD curl --fail http://localhost:11434 || exit 1
 
-COPY ./healthcheck.sh .
-HEALTHCHECK --interval=60s --timeout=20m --start-period=5s --retries=20 \
-    CMD ./healthcheck.sh
+CMD ["./ollama", "serve"]
diff --git a/usecases/ai/microservices/speech-to-text/Dockerfile b/usecases/ai/microservices/speech-to-text/Dockerfile
@@ -6,22 +6,25 @@ FROM openvino/ubuntu22_dev:2024.5.0
 USER root
 
 # Install dependencies
-RUN apt-get update && apt-get install -y ffmpeg \
-    wget \
-    gnupg2 \
-    libtbb12 \
-    python3.11 \
-    python3.11-venv 
+RUN apt-get update \
+    && apt-get install -y ffmpeg \
+        wget \
+        gnupg2 \
+        libtbb12 \
+        python3.11 \
+        python3.11-venv 
 
 # Install GPU drivers
-RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg \
-    && echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | tee /etc/apt/sources.list.d/intel-gpu-jammy.list \
-    && apt update \
-    && apt-get install -y libze1 \
-        intel-level-zero-gpu \
-        intel-opencl-icd \
-        clinfo \
-    && rm -rf /var/lib/apt/lists/*
+RUN mkdir /tmp/neo \
+    && cd /tmp/neo \
+    && wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb \
+    && wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb \
+    && wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu-dbgsym_1.6.31294.12_amd64.ddeb \
+    && wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb \
+    && wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd-dbgsym_24.39.31294.12_amd64.ddeb \
+    && wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb \
+    && wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb \
+    && dpkg -i *.deb
 
 # Install NPU drivers
 RUN mkdir /tmp/npu-driver \
@@ -33,7 +36,11 @@ RUN mkdir /tmp/npu-driver \
     && dpkg -i *.deb
 
 RUN mkdir -p /usr/src \
-    && chown -R openvino:openvino /usr/src
+    && chown -R openvino:openvino /usr/src \
+    && mkdir -p /usr/src/app/data \
+    && chown -R openvino:openvino /usr/src/app/data \
+    && mkdir -p /usr/src/app/tmp_audio \
+    && chown -R openvino:openvino /usr/src/app/tmp_audio
 
 USER openvino
 WORKDIR /usr/src/app
@@ -44,8 +51,10 @@ ENV PATH="/usr/src/.venv/bin:$PATH"
 COPY --chown=openvino:openvino requirements.txt .
 RUN python3 -m pip install -r requirements.txt
 
-COPY --chown=openvino:openvino server.py .
+COPY --chown=openvino:openvino main.py .
 COPY --chown=openvino:openvino utils.py .
 
 HEALTHCHECK --interval=60s --timeout=180s --start-period=5s --retries=3 \
-    CMD wget --no-verbose --no-proxy -O /dev/null --tries=1 http://stt_service:5996/healthcheck || exit 1
+    CMD wget --no-verbose -O /dev/null --tries=1 http://stt_service:5996/healthcheck || exit 1
+
+CMD ["python3", "-m", "uvicorn", "main:app", "--host", "stt_service", "--port", "5996"]
diff --git a/usecases/ai/microservices/speech-to-text/main.py b/usecases/ai/microservices/speech-to-text/main.py
diff --git a/usecases/ai/microservices/speech-to-text/requirements.txt b/usecases/ai/microservices/speech-to-text/requirements.txt
@@ -4,5 +4,6 @@ uvicorn==0.32.1
 soundfile==0.12.1
 pydub==0.25.1
 
+openvino==2024.5.0
 openvino_genai==2024.5.0
 optimum-intel[openvino,nncf]==1.21.0
diff --git a/usecases/ai/microservices/text-to-speech/Dockerfile b/usecases/ai/microservices/text-to-speech/Dockerfile
@@ -23,10 +23,11 @@ RUN mkdir -p thirdparty && \
 WORKDIR /usr/src/app
 COPY requirements.txt .
 RUN python3 -m pip install -r requirements.txt
-COPY nltk_download.py .
-RUN python3 nltk_download.py
-
 COPY . .
 
+ENV NLTK_DATA="/usr/src/app/data/nltk_data"
+
 HEALTHCHECK --interval=30s --timeout=180s --start-period=50s --retries=5 \
-    CMD wget --no-verbose --no-proxy -O /dev/null --tries=1 http://tts_service:5995/healthcheck || exit 1
+    CMD wget --no-verbose --no-proxy -O /dev/null --tries=1 http://tts_service:5995/healthcheck || exit 1
+
+CMD ["python3", "-m", "uvicorn", "main:app", "--host", "tts_service", "--port", "5995"]
diff --git a/usecases/ai/microservices/text-to-speech/main.py b/usecases/ai/microservices/text-to-speech/main.py
@@ -6,21 +6,19 @@
 
 import os
 import uuid
-# import torch
 import time
-# import intel_extension_for_pytorch as ipex
-# import ipex_llm
-from scipy.io.wavfile import write as write_wav
+import json
+from typing import Optional
 
-from contextlib import asynccontextmanager
 from pydantic import BaseModel
-from typing import Optional
-from fastapi.responses import FileResponse
 from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse
+from contextlib import asynccontextmanager
 from starlette.background import BackgroundTasks
 from fastapi.middleware.cors import CORSMiddleware
+
+import nltk
 from melo.api import TTS
-import json
 
 TTS_PROCESSOR = None
 TTS_SPEAKER_IDS = None
@@ -42,11 +40,22 @@ def warmup(model, speaker_ids):
     os.remove(output_path)
     print("Model warmup completed ...")
 
+def download_nltk_packages():
+    print("Downloading NLTK packages ...")
+    if not os.path.exists("./data/nltk_data/corpora/cmudict"):
+        nltk.download('cmudict')
+        
+    if not os.path.exists("./data/nltk_data/taggers/averaged_perceptron_tagger"):
+        nltk.download('averaged_perceptron_tagger')
+    
+    if not os.path.exists("./data/nltk_data/taggers/averaged_perceptron_tagger_eng"):
+        nltk.download('averaged_perceptron_tagger_eng')
 
 def startup():
     global TTS_MODEL, TTS_SPEAKER_IDS, TTS_DEVICE
     if not os.path.exists("/tmp_audio"):
         os.makedirs("/tmp_audio")
+    download_nltk_packages()
     TTS_DEVICE = os.getenv('TTS_DEVICE').lower()
     print("Starting up model ...")
     TTS_MODEL, TTS_SPEAKER_IDS = load_bark_model(language='EN', device=TTS_DEVICE)
@@ -88,8 +97,6 @@ class TTSModel(BaseModel):
 @app.post("/v1/audio/load_speech_model")
 async def load_model(data: TTSModel):
     global TTS_MODEL, TTS_SPEAKER_IDS
-    # device = "xpu" if torch.xpu.is_available() else "cpu"
-    # device = "xpu"
     TTS_MODEL, TTS_SPEAKER_IDS = load_bark_model(
         language=data.language, device=data.device)
 
diff --git a/usecases/ai/microservices/text-to-speech/nltk_download.py b/usecases/ai/microservices/text-to-speech/nltk_download.py
diff --git a/usecases/ai/openwebui-ollama/.gitignore b/usecases/ai/openwebui-ollama/.gitignore
@@ -1,2 +1,4 @@
 */__pycache__
-data/models
+data/models
+data/huggingface
+data/nltk_data
diff --git a/usecases/ai/openwebui-ollama/README.md b/usecases/ai/openwebui-ollama/README.md
@@ -116,3 +116,7 @@ Example:
 * STT Pipeline: Click on the `Record Voice` icon to start recording, and click again to stop. The generated text will appear in the input field.
 
     ![Alt Test](./assets/stt-example.gif)
+
+## Limitations
+### 1. Automatic Speech Recognition Compatibility
+Automatic speech recognition functionality is not supported in Firefox. Please use Chrome for validated performance.
diff --git a/usecases/ai/openwebui-ollama/docker-compose.yml b/usecases/ai/openwebui-ollama/docker-compose.yml
@@ -18,7 +18,9 @@ services:
     devices:
       - /dev:/dev:rw
       - /lib/modules:/lib/modules:rw
-    command: "python3 -m uvicorn main:app --host tts_service --port 5995"
+    volumes:
+      - ./data/huggingface:/root/.cache/huggingface
+      - ./data/nltk_data:/usr/src/app/data/nltk_data
 
   stt_service:
     build:
@@ -44,7 +46,6 @@ services:
     devices:
       - /dev:/dev
       - /lib/modules:/lib/modules
-    command: "python3 -m uvicorn server:app --host stt_service --port 5996"
 
   open-webui:
     hostname: open-webui
@@ -58,12 +59,20 @@ services:
         condition: service_healthy
       tts_service:
         condition: service_healthy
+      ollama:
+        condition: service_healthy
     networks:
       - app-network
     environment:
       - OLLAMA_BASE_URL=http://ollama:11434
+      - AUDIO_STT_ENGINE=openai
       - AUDIO_STT_OPENAI_API_BASE_URL=http://stt_service:5996/v1
       - AUDIO_STT_OPENAI_API_KEY=-
+      - AUDIO_TTS_ENGINE=openai
+      - AUDIO_TTS_OPENAI_API_BASE_URL=http://tts_service:5995/v1
+      - AUDIO_TTS_OPENAI_API_KEY=-
+      - AUDIO_TTS_MODEL=
+      - AUDIO_TTS_VOICE=
     volumes:
       - open-webui:/app/backend/data
 
@@ -84,14 +93,11 @@ services:
       - ZES_ENABLE_SYSMAN=1
       - SYCL_CACHE_PERSISTENT=1
       - SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
-      - ONEAPI_DEVICE_SELECTOR=level_zero:0
     devices:
       - /dev:/dev:rw
       - /lib/modules:/lib/modules:rw
     volumes:
-      - ./serving/ollama/setup.sh:/setup.sh
       - ollama:/root/.ollama
-    command: "./ollama serve"
 
 networks:
   app-network:
diff --git a/usecases/ai/rag-toolkit/edge-ui/package-lock.json b/usecases/ai/rag-toolkit/edge-ui/package-lock.json
diff --git a/usecases/ai/rag-toolkit/edge-ui/package.json b/usecases/ai/rag-toolkit/edge-ui/package.json
