Update Dockerfile-cuda-all and cuda-all-entrypoint.sh

Author: alvarobartt

Dockerfile-cuda-all

@@ -115,13 +115,13 @@ ARG DEFAULT_USE_FLASH_ATTENTION=True
 ENV HUGGINGFACE_HUB_CACHE=/data \
     PORT=80 \
     USE_FLASH_ATTENTION=$DEFAULT_USE_FLASH_ATTENTION \
-    LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
+    LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
     libssl-dev \
     curl \
-    cuda-compat-13-1 \
+    cuda-compat-12-9 \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /usr/src/target/release/text-embeddings-router-75 /usr/local/bin/text-embeddings-router-75

cuda-all-entrypoint.sh

@@ -5,6 +5,27 @@ if ! command -v nvidia-smi &>/dev/null; then
     exit 1
 fi
 
+# NOTE: Given that we need to support CUDA versions earlier than CUDA 12.9.1, we
+# need to include the `cuda-compat-12-9` in `LD_LIBRARY_PATH` when the host CUDA
+# version is lower than that; whilst we shouldn't include that when CUDA is 13.0+
+# as otherwise it will fail due to it.
+if [ -d /usr/local/cuda/compat ]; then
+    DRIVER_CUDA=$(nvidia-smi 2>/dev/null | awk '/CUDA Version/ {print $3; exit}')
+
+    IFS='.' read -r MAJ MIN PATCH <<EOF
+${DRIVER_CUDA:-0.0.0}
+EOF
+    : "${MIN:=0}"
+    : "${PATCH:=0}"
+
+    DRIVER_INT=$((10#${MAJ} * 10000 + 10#${MIN} * 100 + 10#${PATCH}))
+    TARGET_INT=$((12 * 10000 + 9 * 100 + 1))
+
+    if [ "$DRIVER_INT" -lt "$TARGET_INT" ]; then
+        export LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH}"
+    fi
+fi
+
 compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv | sed -n '2p' | sed 's/\.//g')
 
 if [ ${compute_cap} -eq 75 ]; then