Merge remote-tracking branch 'upstream/main'

Author: dchourasia

examples/hpo-raytune/notebook/raytune-oai-demo-mlmd.ipynb

@@ -54,7 +54,7 @@
     "    min_memory=4,\n",
     "    max_memory=4,\n",
     "    num_gpus=0,\n",
-    "    image=\"quay.io/modh/ray:2.35.0-py39-cu121\",                                           \n",
+    "    image=\"quay.io/modh/ray:2.35.0-py311-cu121\",                                           \n",
     "))"
    ]
   },

examples/hpo-raytune/notebook/raytune-oai-demo.ipynb

@@ -55,7 +55,7 @@
     "    min_memory=4,\n",
     "    max_memory=4,\n",
     "    num_gpus=0,\n",
-    "    image=\"quay.io/modh/ray:2.35.0-py39-cu121\"\n",
+    "    image=\"quay.io/modh/ray:2.35.0-py311-cu121\"\n",
     "))"
    ]
   },

examples/ray-finetune-llm-deepspeed/README.md

@@ -129,7 +129,7 @@ This example has been validated on the following configurations:
         head_memory_limits=96,
         head_extended_resource_requests={'amd.com/gpu':1},
         worker_extended_resource_requests={'amd.com/gpu':1},
-        image="quay.io/rhoai/ray:2.35.0-py39-rocm61-torch24-fa26",
+        image="quay.io/rhoai/ray:2.35.0-py311-rocm61-torch24-fa26",
     )
     ```
 * Ray job:

examples/ray-finetune-llm-deepspeed/ray_finetune_llm_deepspeed.ipynb

@@ -70,11 +70,12 @@
     "    head_memory_requests=128,\n",
     "    head_memory_limits=256,\n",
     "    # Use the following parameters with NVIDIA GPUs\n",
-    "    image=\"quay.io/rhoai/ray:2.35.0-py39-cu121-torch24-fa26\",\n",
+    "    # Ensure the Python version in the notebook image matches the version used in the Ray cluster to avoid compatibility issues\n",
+    "    image=\"quay.io/rhoai/ray:2.35.0-py311-cu121-torch24-fa26\",\n",
     "    head_extended_resource_requests={'nvidia.com/gpu':1},\n",
     "    worker_extended_resource_requests={'nvidia.com/gpu':1},\n",
     "    # Or replace them with these parameters for AMD GPUs\n",
-    "    # image=\"quay.io/rhoai/ray:2.35.0-py39-rocm61-torch24-fa26\",\n",
+    "    # image=\"quay.io/rhoai/ray:2.35.0-py311-rocm61-torch24-fa26\",\n",
     "    # head_extended_resource_requests={'amd.com/gpu':1},\n",
     "    # worker_extended_resource_requests={'amd.com/gpu':1},\n",
     "))"

examples/stable-diffusion-dreambooth/notebook/00 Intro.ipynb

@@ -61,7 +61,7 @@
     "                                       worker_memory_limits=8, \n",
     "                                       head_extended_resource_requests={'nvidia.com/gpu':0},\n",
     "                                       worker_extended_resource_requests={'nvidia.com/gpu':0},\n",
-    "                                       image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n",
+    "                                       image=\"quay.io/modh/ray:2.35.0-py311-cu121\",\n",
     "                                       write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n",
     "                                       local_queue=\"local-queue\"\n",
     "                                       ))"

examples/stable-diffusion-dreambooth/notebook/raycluster.yaml

@@ -20,7 +20,7 @@ spec:
           - env:
             - name: NVIDIA_VISIBLE_DEVICES
               value: "void"
-            image: 'quay.io/project-codeflare/ray:latest-py39-cu118'
+            image: 'quay.io/modh/ray:2.35.0-py311-cu121'
             imagePullPolicy: Always
             lifecycle:
               preStop:
@@ -60,7 +60,7 @@ spec:
       template:        
         spec:
           containers:
-            - image: 'quay.io/project-codeflare/ray:latest-py39-cu118'
+            - image: 'quay.io/modh/ray:2.35.0-py311-cu121'
               lifecycle:
                 preStop:
                   exec:

images/runtime/training/cuda/Dockerfile

@@ -1,9 +1,9 @@
 ## Global Args ######################################################
-ARG BASE_UBI_IMAGE_TAG=latest
-ARG PYTHON_VERSION=3.11
+ARG IMAGE_TAG=1-77.1729776556
+ARG PYTHON_VERSION=311
 
 # use UBI9 latest
-FROM registry.access.redhat.com/ubi9/ubi:${BASE_UBI_IMAGE_TAG} AS base
+FROM registry.access.redhat.com/ubi9/python-${PYTHON_VERSION}:${IMAGE_TAG} AS base
 
 LABEL name="training:py311-cuda121-torch241" \
       summary="CUDA 12.1 Python 3.11 PyTorch 2.4.1 image based on UBI9 for Training" \
@@ -16,24 +16,18 @@ LABEL name="training:py311-cuda121-torch241" \
 COPY LICENSE.md /licenses/cuda-license.md
 
 # Set the working directory in the container
+USER 0
 WORKDIR /app
 
-# remove subscription-manager and install python3.11
-RUN dnf remove -y --disableplugin=subscription-manager \
-        subscription-manager \
-    && dnf install -y python${PYTHON_VERSION} procps g++ python${PYTHON_VERSION}-devel \
-    && python -m ensurepip --upgrade \
-    && python -m pip install --upgrade pip \
-    && python -m pip install --upgrade setuptools \
-    && dnf update -y \
-    && dnf clean all
-
-RUN dnf remove -y python3-requests && \
-    pip install --no-cache-dir requests==2.32.3
+# upgrade requests package
+RUN pip install --no-cache-dir --upgrade requests==2.32.3
 
 ## CUDA Base ###################################################################
 FROM base AS cuda-base
 
+# Install CUDA
+WORKDIR /opt/app-root/bin
+
 # Ref: https://docs.nvidia.com/cuda/archive/12.1.0/cuda-toolkit-release-notes/
 ENV CUDA_VERSION=12.1.0 \
     NV_CUDA_LIB_VERSION=12.1.0-1 \
@@ -106,3 +100,7 @@ RUN pip install --no-cache-dir -U "micropipenv[toml]"
 COPY Pipfile.lock ./
 
 RUN micropipenv install && rm -f ./Pipfile.lock
+
+# Restore user workspace
+USER 1001
+WORKDIR /opt/app-root/src

images/runtime/training/rocm/Dockerfile

@@ -1,9 +1,9 @@
 ## Global Args ######################################################
-ARG BASE_UBI_IMAGE_TAG=latest
-ARG PYTHON_VERSION=3.11
+ARG IMAGE_TAG=1-77.1729776556
+ARG PYTHON_VERSION=311
 
 # use UBI9 latest
-FROM registry.access.redhat.com/ubi9/ubi:${BASE_UBI_IMAGE_TAG} AS base
+FROM registry.access.redhat.com/ubi9/python-${PYTHON_VERSION}:${IMAGE_TAG} AS base
 
 LABEL name="training:py311-rocm61-torch241" \
       summary="ROCm 6.1 Python 3.11 PyTorch 2.4.1 image based on UBI9 for Training" \
@@ -16,23 +16,13 @@ LABEL name="training:py311-rocm61-torch241" \
 COPY LICENSE.md /licenses/rocm-license.md
 
 # Set the working directory in the container
+USER 0
 WORKDIR /app
 
-# remove subscription-manager and install python3.11
-RUN dnf remove -y --disableplugin=subscription-manager \
-        subscription-manager \
-    && dnf install -y python${PYTHON_VERSION} procps g++ python${PYTHON_VERSION}-devel \
-    && python -m ensurepip --upgrade \
-    && python -m pip install --upgrade pip \
-    && python -m pip install --upgrade setuptools \
-    && dnf update -y \
-    && dnf clean all
-
-RUN dnf remove -y python3-requests && \
-    pip install --no-cache-dir requests==2.32.3
+# upgrade requests package
+RUN pip install --no-cache-dir --upgrade requests==2.32.3
 
 # Install ROCm
-USER 0
 WORKDIR /opt/app-root/bin
 
 ARG ROCM_VERSION=6.1.2