reverting to use llamacpp and adding cuda toolkit

Author: dougyster

Dockerfile

@@ -103,37 +103,43 @@ USER modelrunner
 
 # Install uv and vLLM as modelrunner user
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
- && ~/.local/bin/uv venv --python /usr/bin/python3 /opt/vllm-env \
- && if [ "$TARGETARCH" = "amd64" ]; then \
-      WHEEL_ARCH="manylinux_2_31_x86_64"; \
-      WHEEL_URL="https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}%2B${VLLM_CUDA_VERSION}-${VLLM_PYTHON_TAG}-${WHEEL_ARCH}.whl"; \
-      ~/.local/bin/uv pip install --python /opt/vllm-env/bin/python "$WHEEL_URL"; \
+    && ~/.local/bin/uv venv --python /usr/bin/python3 /opt/vllm-env \
+    && if [ "$TARGETARCH" = "amd64" ]; then \
+    WHEEL_ARCH="manylinux_2_31_x86_64"; \
+    WHEEL_URL="https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}%2B${VLLM_CUDA_VERSION}-${VLLM_PYTHON_TAG}-${WHEEL_ARCH}.whl"; \
+    ~/.local/bin/uv pip install --python /opt/vllm-env/bin/python "$WHEEL_URL"; \
     else \
-      ~/.local/bin/uv pip install --python /opt/vllm-env/bin/python "vllm==${VLLM_VERSION}"; \
+    ~/.local/bin/uv pip install --python /opt/vllm-env/bin/python "vllm==${VLLM_VERSION}"; \
     fi
 
 RUN /opt/vllm-env/bin/python -c "import vllm; print(vllm.__version__)" > /opt/vllm-env/version
 
 # --- SGLang variant ---
 FROM llamacpp AS sglang
 
-ARG SGLANG_VERSION=0.4.0
+ARG SGLANG_VERSION=0.5.6
 
 USER root
 
-RUN apt update && apt install -y python3 python3-venv python3-dev curl ca-certificates build-essential && rm -rf /var/lib/apt/lists/*
+# Install CUDA toolkit for nvcc (needed for flashinfer JIT compilation)
+# Plus other SGLang dependencies
+RUN apt update && apt install -y \
+    python3 python3-venv python3-dev \
+    curl ca-certificates build-essential \
+    libnuma1 libnuma-dev numactl ninja-build \
+    nvidia-cuda-toolkit \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /opt/sglang-env && chown -R modelrunner:modelrunner /opt/sglang-env
 
 USER modelrunner
 
 # Install uv and SGLang as modelrunner user
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
- && ~/.local/bin/uv venv --python /usr/bin/python3 /opt/sglang-env \
- && ~/.local/bin/uv pip install --python /opt/sglang-env/bin/python "sglang==${SGLANG_VERSION}"
+    && ~/.local/bin/uv venv --python /usr/bin/python3 /opt/sglang-env \
+    && ~/.local/bin/uv pip install --python /opt/sglang-env/bin/python "sglang==${SGLANG_VERSION}"
 
 RUN /opt/sglang-env/bin/python -c "import sglang; print(sglang.__version__)" > /opt/sglang-env/version
-
 FROM llamacpp AS final-llamacpp
 # Copy the built binary from builder
 COPY --from=builder /app/model-runner /app/model-runner

pkg/inference/backends/sglang/sglang.go

@@ -171,7 +171,10 @@ func (s *sglang) Run(ctx context.Context, socket, model string, modelRef string,
 
 	// Add served model name and weight version
 	if model != "" {
-		args = append(args, "--served-model-name", model)
+		// SGLang 0.5.6+ doesn't allow colons in served-model-name (reserved for LoRA syntax)
+		// Replace colons with underscores to sanitize the model name
+		sanitizedModel := strings.ReplaceAll(model, ":", "_")
+		args = append(args, "--served-model-name", sanitizedModel)
 	}
 	if modelRef != "" {
 		args = append(args, "--weight-version", modelRef)