From 5aa0c5a7aee416a7a711ec24bc083c2edf71ee40 Mon Sep 17 00:00:00 2001 From: Tushar Sharma Date: Mon, 3 Nov 2025 12:44:40 -0800 Subject: [PATCH 1/3] initial commit for frontend image deployment Signed-off-by: Tushar Sharma --- container/Dockerfile | 18 ++++++++++++++++++ container/run.sh | 2 +- examples/backends/sglang/deploy/agg.yaml | 10 +++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/container/Dockerfile b/container/Dockerfile index b17c57ca7b..0afa72173d 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -401,3 +401,21 @@ RUN --mount=type=bind,source=./container/launch_message.txt,target=/opt/dynamo/l ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] CMD [] + +FROM nvcr.io/nvidia/distroless/python:3.12-v3.5.0 as frontend + +COPY --from=dev /opt/dynamo/wheelhouse/ /opt/dynamo/wheelhouse/ + +RUN pip install /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl /opt/dynamo/wheelhouse/ai_dynamo*any.whl /opt/dynamo/wheelhouse/nixl/nixl*.whl +RUN if [ "$ENABLE_KVBM" = "true" ]; then \ + pip install /opt/dynamo/wheelhouse/kvbm*.whl; \ + fi + +COPY benchmarks/ /opt/dynamo/benchmarks/ +RUN pip install /opt/dynamo/benchmarks/ + +COPY launch/ /opt/dynamo/launch/ +COPY lib/ /opt/dynamo/lib/ +COPY components/ /opt/dynamo/components/ + +CMD [] \ No newline at end of file diff --git a/container/run.sh b/container/run.sh index 2f16b3e729..2b2e62f0dd 100755 --- a/container/run.sh +++ b/container/run.sh @@ -42,7 +42,7 @@ ENVIRONMENT_VARIABLES= REMAINING_ARGS= INTERACTIVE= USE_NIXL_GDS= -RUNTIME=nvidia +RUNTIME= WORKDIR=/workspace NETWORK=host diff --git a/examples/backends/sglang/deploy/agg.yaml b/examples/backends/sglang/deploy/agg.yaml index c6302906b6..7f14bf937d 100644 --- a/examples/backends/sglang/deploy/agg.yaml +++ b/examples/backends/sglang/deploy/agg.yaml @@ -4,19 +4,19 @@ apiVersion: nvidia.com/v1alpha1 kind: DynamoGraphDeployment metadata: - name: sglang-agg + name: tusharma-sglang-agg spec: services: Frontend: - dynamoNamespace: sglang-agg + dynamoNamespace: tusharma componentType: frontend replicas: 1 extraPodSpec: mainContainer: - image: my-registry/sglang-runtime:my-tag + image: nvcr.io/nvidian/dynamo-dev/tusharma-frontend:0.2.0 decode: envFromSecret: hf-token-secret - dynamoNamespace: sglang-agg + dynamoNamespace: tusharma componentType: worker replicas: 1 resources: @@ -24,7 +24,7 @@ spec: gpu: "1" extraPodSpec: mainContainer: - image: my-registry/sglang-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/sglang-runtime:0.6.0 workingDir: /workspace/examples/backends/sglang command: - python3 From 48ced8fe20af83e4ff44c0ce1d5757b6d6af58f0 Mon Sep 17 00:00:00 2001 From: Tushar Sharma Date: Tue, 4 Nov 2025 10:37:54 -0800 Subject: [PATCH 2/3] fixes Signed-off-by: Tushar Sharma --- container/Dockerfile | 55 +++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/container/Dockerfile b/container/Dockerfile index 0afa72173d..82bbd22fc5 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -7,12 +7,12 @@ ################################## # Base image configuration -ARG BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base" +ARG BASE_IMAGE="nvcr.io/nvidia/base/ubuntu" # TODO OPS-612: NCCL will hang with 25.03, so use 25.01 for now # Please check https://github.com/ai-dynamo/dynamo/pull/1065 # for details and reproducer to manually test if the image # can be updated to later versions. -ARG BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04" +ARG BASE_IMAGE_TAG="noble-20250619" # Build configuration ARG ENABLE_KVBM=false @@ -322,9 +322,9 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \ fi && \ /tmp/use-sccache.sh show-stats "Dynamo" -############################################## +################################################### ########## Dev entrypoint image ############## -############################################## +################################################### FROM base AS dev @@ -333,7 +333,8 @@ ARG ARCH_ALT # Application environment variables ENV DYNAMO_HOME=/opt/dynamo \ - CARGO_TARGET_DIR=/opt/dynamo/target + CARGO_TARGET_DIR=/opt/dynamo/target \ + CARGO_HOME=/usr/local/cargo # NIXL environment variables ENV NIXL_PREFIX=/opt/nvidia/nvda_nixl \ @@ -402,20 +403,42 @@ RUN --mount=type=bind,source=./container/launch_message.txt,target=/opt/dynamo/l ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] CMD [] -FROM nvcr.io/nvidia/distroless/python:3.12-v3.5.0 as frontend +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS frontend -COPY --from=dev /opt/dynamo/wheelhouse/ /opt/dynamo/wheelhouse/ +# Install uv package manager +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -RUN pip install /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl /opt/dynamo/wheelhouse/ai_dynamo*any.whl /opt/dynamo/wheelhouse/nixl/nixl*.whl -RUN if [ "$ENABLE_KVBM" = "true" ]; then \ - pip install /opt/dynamo/wheelhouse/kvbm*.whl; \ - fi +# Create and activate virtual environment +ARG PYTHON_VERSION +RUN mkdir -p /opt/dynamo/venv && \ + uv venv /opt/dynamo/venv --python $PYTHON_VERSION -COPY benchmarks/ /opt/dynamo/benchmarks/ -RUN pip install /opt/dynamo/benchmarks/ +ENV VIRTUAL_ENV=/opt/dynamo/venv \ + PATH="/opt/dynamo/venv/bin:${PATH}" -COPY launch/ /opt/dynamo/launch/ -COPY lib/ /opt/dynamo/lib/ -COPY components/ /opt/dynamo/components/ +COPY --from=dev /opt/dynamo/wheelhouse/ /opt/dynamo/wheelhouse/ +RUN uv pip install \ + /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl \ + /opt/dynamo/wheelhouse/ai_dynamo*any.whl \ + /opt/dynamo/wheelhouse/nixl/nixl*.whl && \ + if [ "$ENABLE_KVBM" = "true" ]; then \ + uv pip install /opt/dynamo/wheelhouse/kvbm*.whl; \ + fi \ + && cd /opt/dynamo/benchmarks \ + && UV_GIT_LFS=1 uv pip install --no-cache . \ + && cd - \ + && rm -rf /opt/dynamo/benchmarks +RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \ + UV_GIT_LFS=1 uv pip install \ + --no-cache \ + --requirement /tmp/requirements.txt + +# Setup launch banner +RUN --mount=type=bind,source=./container/launch_message.txt,target=/opt/dynamo/launch_message.txt \ + sed '/^#\s/d' /opt/dynamo/launch_message.txt > ~/.launch_screen && \ + echo "cat ~/.launch_screen" >> ~/.bashrc && \ + echo "source $VIRTUAL_ENV/bin/activate" >> ~/.bashrc + +ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] CMD [] \ No newline at end of file From 55c140b0d99b1a356bc6fc95b2cb1ab4621ec91e Mon Sep 17 00:00:00 2001 From: Tushar Sharma Date: Wed, 5 Nov 2025 20:29:25 -0800 Subject: [PATCH 3/3] Initial commit for frontend image Signed-off-by: Tushar Sharma --- container/Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/container/Dockerfile b/container/Dockerfile index 82bbd22fc5..4ff237f2ff 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -408,6 +408,11 @@ FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS frontend # Install uv package manager COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +COPY --from=dev /usr/bin/nats-server /usr/bin/nats-server +COPY --from=dev /usr/local/bin/etcd/ /usr/local/bin/etcd/ +# Add ETCD to PATH +ENV PATH=/usr/local/bin/etcd:$PATH + # Create and activate virtual environment ARG PYTHON_VERSION RUN mkdir -p /opt/dynamo/venv && \ @@ -416,6 +421,7 @@ RUN mkdir -p /opt/dynamo/venv && \ ENV VIRTUAL_ENV=/opt/dynamo/venv \ PATH="/opt/dynamo/venv/bin:${PATH}" +COPY benchmarks/ /opt/dynamo/benchmarks/ COPY --from=dev /opt/dynamo/wheelhouse/ /opt/dynamo/wheelhouse/ RUN uv pip install \ /opt/dynamo/wheelhouse/ai_dynamo_runtime*.whl \ @@ -430,9 +436,13 @@ RUN uv pip install \ && rm -rf /opt/dynamo/benchmarks RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \ + --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \ UV_GIT_LFS=1 uv pip install \ --no-cache \ - --requirement /tmp/requirements.txt + --requirement /tmp/requirements.txt \ + --requirement /tmp/requirements.test.txt + +COPY . /workspace/ # Setup launch banner RUN --mount=type=bind,source=./container/launch_message.txt,target=/opt/dynamo/launch_message.txt \