Merge branch 'main' of https://github.com/lyudmil-pelov/oci-data-science-ai-samples

Author: lyudmil-pelov

model-deployment/containers/llama2/Dockerfile.vllm

@@ -1,15 +1,24 @@
-FROM nvidia/cuda:11.8.0-base-ubuntu20.04 as base
+FROM nvcr.io/nvidia/cuda@sha256:3bbed06f530534a5f797a2a09df9b609783796d323663c94bc7ebe082c64a81f as base
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get -y install tzdata && apt-get install -y curl && apt-get install -y git
+# nvidia-container-runtime
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV NVIDIA_REQUIRE_CUDA "cuda>=11.6"
+
+RUN apt-get update && apt-get -y install tzdata && apt-get install -y curl && apt-get install -y git && apt-get clean && apt-get autoremove
 RUN curl -L https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh >> miniconda.sh
 RUN bash ./miniconda.sh -b -p /miniconda; rm ./miniconda.sh;
 ENV PATH="/miniconda/bin:$PATH"
-RUN mkdir -p /opt/vllm
 
+# install oci-cli
+RUN bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
+
+RUN mkdir -p /opt/vllm
 ARG INSTALL_DIR=/opt/vllm
 COPY vllm-env.yaml /opt/vllm/environment.yaml
 RUN conda env create --name vllm -f ${INSTALL_DIR}/environment.yaml
+RUN conda clean -a
 
 ENV TMPDIR=/home/datascience
 WORKDIR /home/datascience
@@ -20,6 +29,11 @@ COPY vllm-log-config.yaml ${INSTALL_DIR}/vllm-log-config.yaml
 ENV UVICORN_LOG-CONFIG=${INSTALL_DIR}/vllm-log-config.yaml
 ENV UVICORN_LOG_CONFIG=${INSTALL_DIR}/vllm-log-config.yaml
 
+# for debugging 
+RUN mkdir -p /aiapps
+COPY runner.sh /aiapps/
+RUN chmod +x /aiapps/runner.sh
+
 # Default location where downloaded models are mapped on model container. No need to override, if using model catalog.
 ENV MODEL /opt/ds/model/deployed_model
 

model-deployment/containers/llama2/Makefile

@@ -1,10 +1,24 @@
+# Initial setup to create the version file if it doesn't exist
+init:
+	@if [ ! -f version.txt ]; then \
+		echo 0 > version.txt; \
+	fi
+
+increment_version:
+	@echo "Reading current version..."
+	$(eval VERSION=$(shell cat version.txt || echo "0"))
+	@echo "Current version is $(VERSION)"
+	$(eval NEW_VERSION=$(shell echo $$(($(VERSION) + 1))))
+	@echo $(NEW_VERSION) > version.txt
+	@echo "Version incremented to $(NEW_VERSION)"
+
 TENANCY:=${TENANCY_NAME}
 CONTAINER_REGISTRY:=${REGION_KEY}.ocir.io
 
 TGI_INFERENCE_IMAGE:=${CONTAINER_REGISTRY}/${TENANCY}/text-generation-interface-odsc:0.9.3
 TGI_CONTAINER_NAME:=tgi-odsc
 
-VLLM_INFERENCE_IMAGE:=${CONTAINER_REGISTRY}/${TENANCY}/vllm-odsc:0.1.4
+VLLM_INFERENCE_IMAGE:=${CONTAINER_REGISTRY}/${TENANCY}/vllm-odsc:0.2.2-v
 VLLM_CONTAINER_NAME:=vllm-odsc
 
 MODEL_DIR:=${PWD}/hfdata
@@ -31,8 +45,8 @@ check-env:
 	@echo "Both TENANCY_NAME and REGION_KEY are set and have values."
 build.tgi:
 	docker build --network host -t ${TGI_INFERENCE_IMAGE} -f Dockerfile.tgi .
-build.vllm: check-env
-	docker build --network host -t ${VLLM_INFERENCE_IMAGE} -f Dockerfile.vllm .
+build.vllm: check-env init increment_version
+	docker build --network host -t ${VLLM_INFERENCE_IMAGE}$(shell cat version.txt) -f Dockerfile.vllm .
 run.tgi.hf:
 	docker run --rm -it --gpus all --shm-size 1g -p ${port}:${port} -e PORT=${port} -e TOKEN_FILE=${target_token} -e PARAMS=${params} -e MODEL=${model} -v ${MODEL_DIR}:${TARGET_DIR} -v ${token}:${target_token} --name ${TGI_CONTAINER_NAME} ${TGI_INFERENCE_IMAGE}
 run.tgi.oci:
@@ -48,6 +62,6 @@ stop.vllm:
 push.tgi:
 	docker push ${TGI_INFERENCE_IMAGE}
 push.vllm:
-	docker push ${VLLM_INFERENCE_IMAGE}
+	docker push ${VLLM_INFERENCE_IMAGE}$(shell cat version.txt)
 app:
-	MODEL=${model} gradio app.py
+	MODEL=${model} gradio app.py

model-deployment/containers/llama2/start-vllm.sh

@@ -1,8 +1,17 @@
 #!/bin/bash
 
-if [ -z "$TOKEN_FILE" ] ; then
-  echo "No authentication token is provided. Weights are assumed to be downloaded from OCI Model Catalog."
+if [[ -z "${MODEL_DEPLOYMENT_OCID}" ]]; then
+  auth_method=instance_principal
 else
+  auth_method=resource_principal
+fi
+
+if [ -n "\$BUCKET" ]; then
+  echo "BUCKET variable are set."
+  #oci os object sync --auth resource_principal --bucket-name genai --dest-dir /home/datascience/llma2/
+  /root/bin/oci os object sync --auth $auth_method --bucket-name $BUCKET --dest-dir /home/datascience/model/
+  MODEL="/home/datascience/model/$MODEL"  
+elif [ -n "\$TOKEN_FILE" ]; then
   export HUGGING_FACE_HUB_TOKEN=$(cat $TOKEN_FILE)
   echo "The md5 of token is $(md5sum $TOKEN_FILE)"
   mkdir -p /home/datascience/.cache/huggingface
@@ -14,17 +23,19 @@ else
   echo $(df -h /home/datascience)
   df -h
   echo "Checking internet connection: "
-  curl -sI -v https://www.wikipedia.org
+  curl -s --connect-timeout 15 http://example.com > /dev/null && echo "Connected" || echo "Not connected"
   echo $(du -sh /home/datascience/*)
+else
+  echo "No bucket or authentication token is provided. Weights are assumed to be downloaded from OCI Model Catalog."
 fi
 
 echo "Starting vllm engine..."
 source activate vllm
-WEB_CONCURRENCY=1 python $VLLM_DIR/vllm-api-server.py --port ${PORT} --host 0.0.0.0 --log-config $VLLM_DIR/vllm-log-config.yaml --model ${MODEL} --tensor-parallel-size ${TENSOR_PARALLELISM} ${PARAMS}
+WEB_CONCURRENCY=1 python $VLLM_DIR/vllm-api-server.py --port ${PORT} --host 0.0.0.0 --log-config $VLLM_DIR/vllm-log-config.yaml --model ${MODEL} --tensor-parallel-size ${TENSOR_PARALLELISM}
 
 
 echo "Exiting vLLM. Here is the disk utilization of /home/datascience - "
 echo $(du -sh /home/datascience)
 echo "server logs: "
 ls -lah /home/datascience
-cat /home/datascience/server.log
+cat /home/datascience/server.log

model-deployment/containers/llama2/vllm-env.yaml

@@ -1,16 +1,15 @@
+channels:
+  - pytorch
+  - nvidia
+  - main
+  - conda-forge
 dependencies:
-  - main::python=3.8
-  - main::pip
-  - nvidia::cuda-cccl=11.8.89
-  - nvidia::cuda-compiler=11.8.0
-  - nvidia::cuda-cuobjdump=11.8.86
-  - nvidia::cuda-cuxxfilt=11.8.86
-  - nvidia::cuda-nvcc=11.8.89
-  - nvidia::cuda-nvdisasm=11.8.86
-  - nvidia::cuda-nvml-dev=11.8.86
+  - python=3.9
+  - pip
   - pip:
-    - vllm
-    - transformers
-    - pandas
-    - flask
-    - GPUtil
+    - https://github.com/vllm-project/vllm/releases/download/v0.2.1.post1/vllm-0.2.1.post1-cp39-cp39-manylinux1_x86_64.whl
+    - transformers >= 4.34.0
+    - pandas==2.1.3
+    - fastapi==0.104.1
+    - GPUtil==1.4.0
+    - oci-cli