Merge pull request #145 from intel/update-branch

feat: update Dockerfile and README for OpenVINO versioning; adjust cache space and volume handling (#392)

Author: gooishin

usecases/ai/microservices/text-generation/vllm/Dockerfile

@@ -4,6 +4,7 @@
 FROM debian:12-slim
 ARG DEBIAN_FRONTEND=noninteractive
 ARG VLLM_VERSION=v0.6.6
+ARG OPENVINO_VERSION=2024.6.0
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN apt-get update \
     && apt-get upgrade -y \
@@ -28,7 +29,7 @@ RUN apt-get update \
     && adduser --system --ingroup intel --uid 1000 --home /home/intel intel \
     && echo "intel ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/intel \
     && rm -rf /var/lib/apt/lists/* \
-    && mkdir -p /usr/src \
+    && mkdir -p /usr/src/app/data \
     && chown -R intel:intel /usr/src
 
 USER intel
@@ -46,8 +47,8 @@ RUN git checkout ${VLLM_VERSION} \
         optimum-intel[openvino,nncf]==1.21.0 \
         optimum==1.23.3 \
         transformers==4.46.3 \
-        openvino==2025.0.0 \
-        openvino_genai==2025.0.0
+        openvino==${OPENVINO_VERSION} \
+        openvino_genai==${OPENVINO_VERSION}
 
 WORKDIR /usr/src/app
 RUN opt_in_out --opt_out

usecases/ai/microservices/text-generation/vllm/README.md

@@ -38,10 +38,9 @@ docker run -it --rm \
     -e MAX_MODEL_LEN=2048 \
     -e MAX_NUM_SEQS=1 \
     -e VLLM_OPENVINO_DEVICE=CPU \
-    -e VLLM_OPENVINO_KVCACHE_SPACE=8 \
+    -e VLLM_OPENVINO_KVCACHE_SPACE=4 \
     -e VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8 \
-    -e VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \
-    -v ./data:/usr/src/app/data \
+    -v ov-vllm:/usr/src/app/data \
     ov-vllm
 ```
 
@@ -59,10 +58,9 @@ docker run -it --rm \
     -e MAX_NUM_SEQS=1 \
     -e GPU_MEMORY_UTILIZATION=0.9 \
     -e VLLM_OPENVINO_DEVICE=GPU \
-    -e VLLM_OPENVINO_KVCACHE_SPACE=8 \
+    -e VLLM_OPENVINO_KVCACHE_SPACE=4 \
     -e VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=u8 \
-    -e VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=ON \
-    -v ./data:/usr/src/app/data \
+    -v ov-vllm:/usr/src/app/data \
     ov-vllm
 ```
 
@@ -111,9 +109,9 @@ curl "http://localhost:8000/v1/chat/completions" \
 ```
 
 ### 2. How can I change the default model after it has been run once?
-1. Delete the existing model located in `./data/ov_model`.
+1. Delete the volume for the container.
 ```bash
-rm -rf ./data/ov_model
+docker volume rm ov-vllm
 ```
 2. Rerun the `docker run` command to load and quantize the new model.
 

usecases/ai/microservices/text-generation/vllm/entrypoint.sh

@@ -14,13 +14,11 @@ export MAX_NUM_SEQS=${MAX_NUM_SEQS:-1}
 export VLLM_OPENVINO_DEVICE=${VLLM_OPENVINO_DEVICE:-CPU}
 export VLLM_OPENVINO_KVCACHE_SPACE=${VLLM_OPENVINO_KVCACHE_SPACE:-8}
 export VLLM_OPENVINO_CPU_KV_CACHE_PRECISION=${VLLM_OPENVINO_CPU_KV_CACHE_PRECISION:-u8}
-export VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS=${VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS:-ON}
 
 echo -e "Using the following configuration:"
 echo -e "- VLLM_OPENVINO_DEVICE: ${VLLM_OPENVINO_DEVICE}"
 echo -e "- VLLM_OPENVINO_KVCACHE_SPACE: ${VLLM_OPENVINO_KVCACHE_SPACE}"
 echo -e "- VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: ${VLLM_OPENVINO_CPU_KV_CACHE_PRECISION}"
-echo -e "- VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: ${VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS}"
 echo -e "- DEFAULT_MODEL_ID: ${DEFAULT_MODEL_ID}"
 echo -e "- MODEL_PATH: ${MODEL_PATH}"
 echo -e "- MODEL_PRECISION: ${MODEL_PRECISION}"
@@ -45,9 +43,6 @@ fi
 if [ ! -f "$MODEL_PATH/openvino_model.xml" ]; then
     echo -e "Model file does not exist: $MODEL_PATH/openvino_model.xml. Please export the model first and save to $MODEL_PATH"
     exit 1
-else
-    echo -e "Model file available. Setting VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS to OFF ..."
-    unset VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS
 fi
 
 echo -e "Starting OpenVINO VLLM service ..."