[llm-d] Keep working (#909)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Persistent PVC for benchmark results and a helper pod to copy results.
* JSON-based GuideLLM benchmark parsing and load-shape subtitle in
throughput plots.

* **Improvements**
* Configurable PVC size (default 1Gi), and new flags to control running
containers as root/user.
* Configurable gateway name, conditional GPU readiness/preload, and
reduced memory request for an inference service.

* **Documentation**
  * Added docs for storage and security-related parameters.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: kpouget

docs/toolbox.generated/Cluster.preload_image.rst

@@ -54,3 +54,10 @@ Parameters
 
 * Pod toleration to apply to the DaemonSet.
 
+
+``run_as_user``  
+
+* User ID to run the preloader container as (defaults to 10001 if not specified).
+
+* default value: ``10001``
+

docs/toolbox.generated/Llmd.run_guidellm_benchmark.rst

@@ -56,7 +56,19 @@ Parameters
 * default value: ``900``
 
 
+``pvc_size``  
+
+* Size of the PersistentVolumeClaim for storing results
+
+* default value: ``1Gi``
+
+
 ``guidellm_args``  
 
 * List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"])
 
+
+``run_as_root``  
+
+* Run the GuideLLM container as root user
+

docs/toolbox.generated/Storage.download_to_pvc.rst

@@ -80,3 +80,8 @@ Parameters
 
 * default value: ``registry.access.redhat.com/ubi9/ubi``
 
+
+``run_as_root``  
+
+* Run the download container as root user
+

projects/cluster/toolbox/cluster.py

@@ -316,7 +316,8 @@ def destroy_osd(self, cluster_name):
     def preload_image(self,
                       name, image, namespace="default",
                       node_selector_key="", node_selector_value="",
-                      pod_toleration_key="", pod_toleration_effect=""):
+                      pod_toleration_key="", pod_toleration_effect="",
+                      run_as_user="10001"):
         """
         Preload a container image on all the nodes of a cluster.
 
@@ -328,6 +329,7 @@ def preload_image(self,
           node_selector_value: NodeSelector value to apply to the DaemonSet.
           pod_toleration_key: Pod toleration to apply to the DaemonSet.
           pod_toleration_effect: Pod toleration to apply to the DaemonSet.
+          run_as_user: User ID to run the preloader container as (defaults to 10001 if not specified).
         """
 
         toolbox_name_suffix = os.environ.get("ARTIFACT_TOOLBOX_NAME_SUFFIX", "")

projects/cluster/toolbox/cluster_preload_image/defaults/main/config.yml

@@ -26,6 +26,9 @@ cluster_preload_image_pod_toleration_key:
 # Pod toleration to apply to the DaemonSet.
 cluster_preload_image_pod_toleration_effect:
 
+# User ID to run the preloader container as (defaults to 10001 if not specified).
+cluster_preload_image_run_as_user: '10001'
+
 # Default Ansible variables
 # Default value for ansible_os_family to ensure role remains standalone
 ansible_os_family: Linux

projects/cluster/toolbox/cluster_preload_image/tasks/main.yml

@@ -11,16 +11,13 @@
     state: directory
     mode: '0755'
 
-- name: Lookup the namespace user ID range
-  shell:
-    set -o pipefail;
-
-    oc get ns {{ cluster_preload_image_namespace }} -ojsonpath={.metadata.annotations} | jq  -r '.["openshift.io/sa.scc.uid-range"]' | cut -d/ -f1
-  register: namespace_uid_range_cmd
-
-- name: Save the namespace uid as run_as_user
+- name: Set run_as_user from configuration
   set_fact:
-    run_as_user: "{{ namespace_uid_range_cmd.stdout }}"
+    run_as_user: "{{ cluster_preload_image_run_as_user }}"
+
+- name: Log run_as_user value
+  debug:
+    msg: "Using run_as_user: {{ run_as_user }}"
 
 - name: Apply the DaemonSet template
   template:

projects/llm-d/testing/config.yaml

@@ -12,8 +12,43 @@ ci_presets:
 
   dev:
     matbench.enabled: false
+    tests.llmd.inference_service.model: facebook-opt-125m
+
+  opt-125m:
+    tests.llmd.inference_service.model: facebook-opt-125m
+    tests.llmd.inference_service.vllm_args[5]: "--max-model-len=2048"
+
+  psap_h200:
+    tests.capture_prom: false
+    tests.capture_prom_uwm: false
+    tests.llmd.skip_prepare: true
+    prepare.namespace.name: kpouget-dev
+
+  pvc_rwx:
+    prepare.pvc.name: storage-rwx
+    prepare.pvc.access_mode: ReadWriteMany
+
+  azure:
+    security.run_as_root: true
+    prepare.preload.skip: true
+    prepare.operators.skip: true
+    prepare.cluster.skip: true
+    prepare.rhoai.skip: true
+
+    tests.llmd.inference_service.model: llama3-1-8b
+    tests.capture_prom: false
+    tests.capture_prom_uwm: false
+    tests.llmd.skip_prepare: false
+
+  azure_light:
+    extends: [azure, opt-125m]
+    prepare.pvc.storage_class: managed-csi
+
 
   cks:
+    extends: [pvc_rwx]
+
+    tests.llmd.inference_service.model: llama3-3-70b
     tests.capture_prom: false
     tests.capture_prom_uwm: false
     tests.llmd.skip_prepare: true
@@ -60,6 +95,10 @@ ci_presets:
 clusters:
   cleanup_on_exit: false
 
+security:
+  # Run containers as root user (affects both GuideLLM benchmarks and storage download operations)
+  run_as_root: false
+
 secrets:
   dir:
     name: psap-ods-secret
@@ -149,15 +188,21 @@ prepare:
     namespaces:
       - "@prepare.namespace.name"
 
+  gpu:
+    wait_for_readiness: false
+
   preload:
+    skip: false
     extra_images: {}
     node_selector_key: nvidia.com/gpu.present
     node_selector_value: "true"
 
   pvc:
+    enabled: true
     size: 2000Gi
-    name: storage-rwx
-    access_mode: ReadWriteMany
+    name: storage
+    access_mode: ReadWriteOnce
+    storage_class: null
 
   model_downloader:
     image: ghcr.io/opendatahub-io/rhaii-on-xks/kserve-storage-initializer:e6b5db0@sha256:b305264fe2211be2c6063500c4c11da79e8357af4b34dd8567b0d8e8dea7e1d4
@@ -166,21 +211,38 @@ prepare:
     skip: false
 
 models:
-  llama3.1-8b:
-    name: RedHatAI/podllama-3-1-8b-instruct-fp8
+  facebook-opt-125m:
+    name: facebook/opt-125m
+    source: hf://facebook/opt-125m
+    resources:
+      cpu: 2
+      memory: 8Gi
+
+  llama3-1-8b:
+    name: RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic
     uri: oci://registry.redhat.io/rhelai1/modelcar-llama-3-1-8b-instruct-fp8-dynamic:1.5
+    # source: hf://RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic
+    resources: {}
 
-  llama3.3-70b:
+  llama3-3-70b:
     name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic
     source: hf://RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic
+    resources:
+      cpu: 4
+      memory: 64Gi
 
   gpt-oss-120:
     name: openai/gpt-oss-120b
     source: hf://openai/gpt-oss-120b
+    resources:
+      cpu: 4
+      memory: 64Gi
 
   granite4-tiny:
     name: RedHatAI/granite-4.0-h-tiny-FP8-dynamic
     source: hf://RedHatAI/granite-4.0-h-tiny-FP8-dynamic
+    resources: {}
+
 tests:
   llmd:
     skip: false
@@ -194,8 +256,9 @@ tests:
       yaml_file: llama-3-1-8b-instruct-fp8.yaml
       timeout: 900
       do_simple_test: true
-
-      model: llama3.3-70b
+      gateway:
+        name: gateway-external
+      model: llama3-1-8b
 
       # vLLM arguments (always applied)
       vllm_args:
@@ -204,7 +267,7 @@ tests:
         - "--uvicorn-log-level=debug"
         - "--trust-remote-code"
         - "--disable-log-requests"
-        - "--max-model-len=40960"
+        - "--max-model-len=40960" # keep in 5th position or udpdate the presets
         - "--gpu-memory-utilization=0.92"
 
       kueue:
@@ -225,11 +288,12 @@ tests:
         name: guidellm-benchmark
         backend_type: openai_http
         rate_type: concurrent
-        max_seconds: null
+        max_seconds: 120
         max_requests: null
         timeout: 900
         data: prompt_tokens=256,output_tokens=128
         rate: 1
+        sample_requests: 20
 
   capture_prom: true
   capture_prom_uwm: true

projects/llm-d/testing/llmisvcs/llama-3-1-8b-instruct-fp8.yaml

@@ -71,6 +71,8 @@ spec:
             emptyDir: {}
           - name: cachi2-cache
             emptyDir: {}
+        nodeSelector:
+          nvidia.com/gpu.present: "true"
     route: {}
     gateway: {}
   template:
@@ -85,7 +87,7 @@ spec:
             nvidia.com/gpu: "1"
           requests:
             cpu: '4'
-            memory: 64Gi
+            memory: 8Gi
             nvidia.com/gpu: "1"
         livenessProbe:
           httpGet:

projects/llm-d/testing/prepare_llmd.py

@@ -195,10 +195,15 @@ def prepare():
     prepare_gateway()
     scale_up()
 
+    model_ref = config.project.get_config("tests.llmd.inference_service.model")
     with run.Parallel("prepare_node") as parallel:
-        parallel.delayed(download_models_to_pvc)
-        parallel.delayed(wait_for_gpu_readiness)
-        parallel.delayed(preload_llm_model_image)
+        parallel.delayed(download_single_model, model_ref)
+
+        if config.project.get_config("prepare.gpu.wait_for_readiness"):
+            parallel.delayed(wait_for_gpu_readiness)
+
+        if not config.project.get_config("prepare.preload.skip"):
+            parallel.delayed(preload_llm_model_image)
 
 
 def prepare_operators():
@@ -493,14 +498,21 @@ def download_models_to_pvc():
     """
     logging.info("Starting model download process")
 
+    # Check if PVC prefetch is enabled
+    pvc_enabled = config.project.get_config("prepare.pvc.enabled", True)  # Default to True for backward compatibility
+
+    if not pvc_enabled:
+        logging.info("PVC prefetch disabled (prepare.pvc.enabled: false) - skipping model downloads")
+        return
+
     try:
         # Get models configuration
         models_config = config.project.get_config("models")
         if not models_config:
             logging.info("No models configured for download - skipping")
             return
 
-        logging.info(f"Downloading {len(models_config)} model(s) to PVC...")
+        logging.info(f"PVC prefetch enabled - downloading {len(models_config)} model(s) to PVC...")
 
         # Download models in parallel for efficiency
         with run.Parallel("download_models") as parallel:
@@ -522,6 +534,13 @@ def download_single_model(model_key):
         model_key: The key identifying the model in the models configuration
     """
     try:
+        # Check if PVC prefetch is enabled
+        pvc_enabled = config.project.get_config("prepare.pvc.enabled", True)  # Default to True for backward compatibility
+
+        if not pvc_enabled:
+            logging.info(f"PVC prefetch disabled - skipping download for model '{model_key}'")
+            return
+
         logging.info(f"Starting download for model '{model_key}'")
 
         # Get model configuration
@@ -540,6 +559,7 @@ def download_single_model(model_key):
         pvc_name = config.project.get_config("prepare.pvc.name")
         pvc_size = config.project.get_config("prepare.pvc.size")
         pvc_access_mode = config.project.get_config("prepare.pvc.access_mode")
+        pvc_storage_class = config.project.get_config("prepare.pvc.storage_class", None)
         namespace = config.project.get_config("prepare.namespace.name")
         downloader_image = config.project.get_config("prepare.model_downloader.image")
 
@@ -559,15 +579,26 @@ def download_single_model(model_key):
         secret_dir = config.project.get_config("secrets.dir.env_key")
         hf_creds_path = pathlib.Path(os.environ[secret_dir]) / hf_token_secret
 
-        run.run_toolbox("storage", "download_to_pvc",
-                       name=model_key,
-                       source=source,
-                       pvc_name=pvc_name,
-                       namespace=namespace,
-                       pvc_size=pvc_size,
-                       image=downloader_image,
-                       creds=str(hf_creds_path),
-                       clean_first=False)  # Don't clean to allow multiple models in same PVC
+        # Prepare download_to_pvc arguments
+        download_args = {
+            "name": model_key,
+            "source": source,
+            "pvc_name": pvc_name,
+            "namespace": namespace,
+            "pvc_size": pvc_size,
+            "pvc_access_mode": pvc_access_mode,
+            "image": downloader_image,
+            "creds": str(hf_creds_path),
+            "clean_first": False,  # Don't clean to allow multiple models in same PVC
+            "run_as_root": config.project.get_config("security.run_as_root")
+        }
+
+        # Add storage class if configured
+        if pvc_storage_class:
+            download_args["pvc_storage_class_name"] = pvc_storage_class
+            logging.info(f"Using PVC storage class: {pvc_storage_class}")
+
+        run.run_toolbox("storage", "download_to_pvc", **download_args)
 
         logging.info(f"Successfully downloaded model '{model_key}'")