413 user story vision language model microservice with pixtral 12b (#416)

Author: gooishin

usecases/ai/microservices/text-to-image/stable-diffusion-v3.5/backend/client.py

@@ -89,7 +89,7 @@ def main():
         return
 
     # Step 2: Select the device
-    response = client.make_request("POST", "/pipeline/select-device", {"device": "CPU"})
+    response = client.make_request("POST", "/pipeline/select-device", {"device": "GPU"})
     if response:
         print(response.json())
 

usecases/ai/microservices/vision-language-model/Dockerfile

@@ -0,0 +1,102 @@
+# Use a minimal Debian base image for a smaller container footprint
+FROM debian:12-slim
+
+# Avoid interactive prompts during package installation
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Define the default allowed models and the default model
+ARG ALLOWED_MODELS="pixtral-12b"
+ARG DEFAULT_MODEL="pixtral-12b"
+
+# Set default model as a build argument and runtime environment variable
+ENV DEFAULT_MODEL=${DEFAULT_MODEL}
+
+# Ensure the container is running as root for package installations
+USER root
+
+# Set the working directory early for clarity and organization
+WORKDIR /usr/src/app
+
+# Install system dependencies, including Python 3.11 and venv
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ffmpeg \
+        wget \
+        git \
+        gnupg2 \
+        libtbb12 \
+        python3.11 \
+        python3.11-venv \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install Intel GPU drivers with error handling and version pinning
+RUN mkdir /tmp/neo \
+    && cd /tmp/neo \
+    && wget -q https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb \
+    && wget -q https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb \
+    && wget -q https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb \
+    && wget -q https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb \
+    && wget -q https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb \
+    && dpkg -i *.deb \
+    && rm -rf /tmp/neo
+
+# Install Intel NPU drivers with error handling and version pinning
+RUN mkdir /tmp/npu-driver \
+    && cd /tmp/npu-driver \
+    && wget -q https://github.com/intel/linux-npu-driver/releases/download/v1.10.0/intel-driver-compiler-npu_1.10.0.20241107-11729849322_ubuntu22.04_amd64.deb \
+    && wget -q https://github.com/intel/linux-npu-driver/releases/download/v1.10.0/intel-fw-npu_1.10.0.20241107-11729849322_ubuntu22.04_amd64.deb \
+    && wget -q https://github.com/intel/linux-npu-driver/releases/download/v1.10.0/intel-level-zero-npu_1.10.0.20241107-11729849322_ubuntu22.04_amd64.deb \
+    && wget -q https://github.com/oneapi-src/level-zero/releases/download/v1.17.6/level-zero_1.17.6+u22.04_amd64.deb \
+    && dpkg -i *.deb \
+    && rm -rf /tmp/npu-driver
+
+# Create a non-root user for OpenVINO and avoid privilege escalation
+RUN groupadd -r openvino && useradd -r -g openvino -G video openvino
+
+# Set up a dedicated home directory for the user
+RUN mkdir -p /home/openvino && \
+    chown -R openvino:openvino /home/openvino
+
+# Copy application files and adjust permissions (excluding the virtual environments)
+COPY . /usr/src/app
+RUN find /usr/src/app -not -path "/usr/src/app/*/.venv*" -exec chown openvino:openvino {} + \
+    && find /usr/src/app -not -path "/usr/src/app/*/.venv*" -exec chmod 755 {} +
+
+# Remove existing virtual environments to ensure a clean install
+RUN rm -rf /usr/src/app/*/.venv
+
+# Create a Python virtual environment for each model and install dependencies
+RUN for model in $ALLOWED_MODELS; do \
+        python3.11 -m venv /usr/src/app/$model/.venv; \
+        /usr/src/app/$model/.venv/bin/python -m pip install --no-cache-dir --upgrade pip; \
+        if [ -f "/usr/src/app/$model/requirements.txt" ]; then \
+            /usr/src/app/$model/.venv/bin/python -m pip install --no-cache-dir -r /usr/src/app/$model/requirements.txt; \
+        fi; \
+        chown -R openvino:openvino /usr/src/app/$model/.venv; \
+    done
+
+# Set the environment variable for the virtual environment based on the selected model
+ENV PATH="/usr/src/app/${SELECTED_MODEL}/.venv/bin:$PATH"
+
+# Switch to the non-root user for security
+USER openvino
+
+# Set the working directory based on the selected model
+WORKDIR /usr/src/app/${SELECTED_MODEL}
+
+# Expose port for the microservice
+EXPOSE 8100
+
+# Allow runtime injection of Hugging Face Token and model selection
+# If HF_TOKEN is not provided, models requiring it will fail securely.
+CMD ["bash", "-c", \
+     "SELECTED_MODEL=${MODEL:-$DEFAULT_MODEL} && \
+      export PATH=/usr/src/app/${SELECTED_MODEL}/.venv/bin:$PATH && \
+      export HF_TOKEN=${HF_TOKEN:-''} && \
+      echo Using model: $SELECTED_MODEL && \
+      echo HF_TOKEN set to: ${HF_TOKEN:0:5}****** && \
+      /usr/src/app/${SELECTED_MODEL}/.venv/bin/python /usr/src/app/${SELECTED_MODEL}/backend/server.py"]
+
+# Add a basic health check
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \
+  CMD exit 0

usecases/ai/microservices/vision-language-model/README.md

@@ -0,0 +1,167 @@
+# Vision Language Model
+
+## Overview
+Vision Language Model microservice is a FastAPI-based API that leverages vision-language models to perform visual question answering and image captioning tasks. 
+It provides endpoints for managing the pipeline, checking the pipeline status, retrieving the generated text responses, and performing a health check. The service enables AI-powered analysis of images with natural language interactions, allowing users to get detailed descriptions of images and ask questions about image content.
+
+### Supported Models
+* Pixtral 12B
+
+### Supported Inference Device
+* CPU
+* GPU
+
+---
+
+## Quick Start
+
+### 1. Install Operating System
+- Install the latest [Ubuntu 24.04 LTS Desktop](https://releases.ubuntu.com/jammy/). Refer to the [Ubuntu Desktop installation tutorial](https://ubuntu.com/tutorials/install-ubuntu-desktop#1-overview) if needed.
+
+### 2. Install GPU Driver (Optional)
+- If you plan to use a GPU for inference, install the appropriate GPU driver:
+  - **Intel® Arc™ A-Series Graphics:** [Installation Guide](https://github.com/intel/edge-developer-kit-reference-scripts/tree/main/gpu/arc/dg2)
+  - **Intel® Data Center GPU Flex Series:** [Installation Guide](https://github.com/intel/edge-developer-kit-reference-scripts/tree/main/gpu/flex/ats)
+
+### 3. Install Docker Engine
+- Follow the official [Docker installation guide](https://docs.docker.com/engine/install/) to set up Docker Engine on your system.
+
+### 4. Build and Run the Docker Container
+- Build the text-to-image generation docker image.
+  ```bash
+  docker build --network=host -t vlm .
+  ```
+- Export the required environment variables and run the container:
+  ```bash
+  # Select your text-to-image model.
+  # Eg. pixtral-12b.
+  export VLM_MODEL=pixtral-12b
+  
+  # Insert huggingface login token
+  export HF_TOKEN=<your_huggingface_token>
+  
+  # Run the container  
+  export RENDER_GROUP_ID=$(getent group render | cut -d: -f3)
+  
+  docker run -it --rm \
+      --name vlm-container \
+      --group-add $RENDER_GROUP_ID \
+      --device /dev/dri:/dev/dri \
+      -p 8100:8100 \
+      -e MODEL=$VLM_MODEL \
+      -e HF_TOKEN=$HF_TOKEN \
+      -v $(pwd)/data:/usr/src/app/data \
+      vlm
+  ```
+
+
+## Development
+
+---
+
+### 1. Setup text-to-image generation server  
+
+- Change the current directory to the selected model. For example:
+  ```bash
+  cd pixtral-12b
+  ```
+
+- Execute the setup script.
+  ```bash
+  ./setup.sh
+  ```
+
+### 2. Verify the server by running the example (optional)
+```bash
+./run.sh
+```
+
+___
+
+## Routes
+
+### 1. **POST /pipeline/select-device**
+   - **Description**: Selects and compiles the device for the pipeline.
+   - **Request Body**: 
+     ```
+     {
+       "device": "<device_name>"
+     }
+     ```
+   - **Response**:
+     - Success:
+       ```
+       {
+         "status": "success",
+         "message": "Pipeline prepared on <device_name>."
+       }
+       ```
+     - Error:
+       ```
+       {
+         "status": "error",
+         "message": "<error_message>"
+       }
+       ```
+
+### 2. **POST /pipeline/run**
+   - **Description**: Starts the pipeline execution asynchronously in the background.
+   - **Request Body**: 
+     ```
+     {
+       "prompt": "<your_prompt>"
+     }
+     ```
+   - **Response**:
+     - Success:
+       ```
+       {
+         "status": "success",
+         "message": "Pipeline execution started in background."
+       }
+       ```
+     - Error:
+       ```
+       {
+         "status": "error",
+         "message": "Pipeline execution is already running or pipeline is not initialized."
+       }
+       ```
+
+### 3. **GET /pipeline/status**
+   - **Description**: Checks the current status of the pipeline.
+   - **Response**:
+     ```
+     {
+       "running": <true or false>,
+       "completed": <true or false>
+     }
+     ```
+
+### 4. **GET /pipeline/answer**
+   - **Description**: Retrieves the generated answer once the pipeline has completed execution. This endpoint is available only when the pipeline has finished processing.
+   - **Response**:
+     - Success:
+     ```
+     {
+       "status": "success",
+       "answer": "<generated_answer>"
+     }
+     ```
+     Error:
+     ```
+     {
+       "status": "error",
+       "message": "Pipeline execution is not yet complete."
+     }
+     ```
+
+### 5. **GET /health**
+   - **Description**: A simple health check endpoint to ensure that the API is up and running.
+   - **Response**:
+     ```
+     {
+       "status": "healthy"
+     }
+     ```
+     

usecases/ai/microservices/vision-language-model/pixtral-12b/.gitignore

@@ -0,0 +1,3 @@
+.venv
+pixtral-12b
+kernel.errors.txt