[Standup] Add a "standalone" test for pre-merge CI/CD (Kind) (#525)

* [Standup] Add a "standalone" test for pre-merge CI/CD (Kind)

Fixed an issue when running in "dry-run" mode

Fixed a syntax error in functions.py

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

* Additional fixes for better standup on a kind cluster

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

---------

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

Author: maugustosilva

.github/workflows/ci-pr-benchmark.yaml

@@ -48,9 +48,24 @@ jobs:
         run: pip install ./config_explorer
         shell: bash
 
-      - name: Standup a modelservice using llm-d-inference-sim
+      - name: Standup (standalone) using llm-d-inference-sim
         run: |
-          ./setup/standup.sh -c kind_sim_fb -t modelservice -s 0,1,2,4,7,8
+          ./setup/standup.sh -c kind_sim_fb -t standalone -s 0,1,2,4,5,6,10
+        shell: bash
+
+      - name: Run harness (standalone)
+        run: |
+          ./setup/run.sh -c kind_sim_fb --dry-run
+        shell: bash
+
+      - name: Teardown (standalone)
+        run: |
+          ./setup/teardown.sh -c kind_sim_fb -t standalone
+        shell: bash
+
+      - name: Standup (modelservice) using llm-d-inference-sim
+        run: |
+          ./setup/standup.sh -c kind_sim_fb -t modelservice -s 0,1,2,4,5,7,8
         shell: bash
 
       - name: Run harness (mock)
@@ -60,7 +75,7 @@ jobs:
           ./setup/run.sh -c kind_sim_fb --dry-run
         shell: bash
 
-      - name: Teardown
+      - name: Teardown (modelservice)
         run: |
           ./setup/teardown.sh -c kind_sim_fb
         shell: bash

build/Dockerfile

@@ -51,7 +51,7 @@ RUN cd vllm; \
 
 ARG GUIDELLM_REPO=https://github.com/vllm-project/guidellm.git
 ARG GUIDELLM_BRANCH=main
-ARG GUIDELLM_COMMIT=ba51acf5b0ba377c5edc35109a78cd3ebb402922
+ARG GUIDELLM_COMMIT=f6175cdd8a88f0931bd46822ed7a71787dcd7cee
 RUN git clone --branch ${GUIDELLM_BRANCH} ${GUIDELLM_REPO}
 RUN cd guidellm; \
     pip install torch --index-url https://download.pytorch.org/whl/cpu; \

scenarios/cicd/kind_sim_fb.sh

@@ -1,13 +1,22 @@
 # A scenario to capture running inference-sim on a Kind cluster without requiring GPUs
 export LLMDBENCH_DEPLOY_METHODS=modelservice
 export LLMDBENCH_VLLM_COMMON_REPLICAS=1
+export LLMDBENCH_VLLM_COMMON_ACCELERATOR_NR=0
+export LLMDBENCH_VLLM_COMMON_CPU_NR=0
+export LLMDBENCH_VLLM_COMMON_CPU_MEM=100Mi
+export LLMDBENCH_VLLM_COMMON_SHM_MEM=500Mi
 export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_TENSOR_PARALLELISM=0
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM=0
 export LLMDBENCH_VLLM_COMMON_MAX_MODEL_LEN=1024
 export LLMDBENCH_VLLM_COMMON_AFFINITY=kubernetes.io/os:linux
 export LLMDBENCH_CONTROL_WAIT_TIMEOUT=90
 export LLMDBENCH_LLMD_IMAGE_NAME="llm-d-inference-sim"
 export LLMDBENCH_LLMD_ROUTINGSIDECAR_IMAGE_TAG="v0.2.0@sha256:a623a0752af0a71b7b05ebf95517848b5dbc3d8d235c1897035905632d5b7d80"
+export LLMDBENCH_VLLM_STANDALONE_IMAGE_REGISTRY=ghcr.io
+export LLMDBENCH_VLLM_STANDALONE_IMAGE_REPO=llm-d
+export LLMDBENCH_VLLM_STANDALONE_IMAGE_NAME=llm-d-inference-sim
+export LLMDBENCH_VLLM_STANDALONE_IMAGE_TAG=auto
+export LLMDBENCH_VLLM_STANDALONE_ARGS="/app/llm-d-inference-sim____--model____/model-cache/models/REPLACE_ENV_LLMDBENCH_DEPLOY_CURRENT_MODEL____--port____REPLACE_ENV_LLMDBENCH_VLLM_COMMON_INFERENCE_PORT____--served-model-name____REPLACE_ENV_LLMDBENCH_DEPLOY_CURRENT_MODEL"
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_MODEL_COMMAND=imageDefault
 export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_MODEL_COMMAND=imageDefault
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_EXTRA_ARGS="[]"
@@ -16,7 +25,11 @@ export LLMDBENCH_VLLM_MODELSERVICE_DECODE_CPU_NR=0
 export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_CPU_NR=0
 export LLMDBENCH_VLLM_MODELSERVICE_DECODE_CPU_MEM=100Mi
 export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_CPU_MEM=100Mi
+export LLMDBENCH_VLLM_MODELSERVICE_DECODE_SHM_MEM=500Mi
+export LLMDBENCH_VLLM_MODELSERVICE_PREFILL_SHM_MEM=500Mi
 export LLMDBENCH_VLLM_MODELSERVICE_URI_PROTOCOL="hf"
+export LLMDBENCH_VLLM_COMMON_PVC_ACCESS_MODE="ReadWriteOnce"
 export LLMDBENCH_DEPLOY_MODEL_LIST="facebook/opt-125m"
+export LLMDBENCH_VLLM_COMMON_PVC_MODEL_CACHE_SIZE=2Gi
 export LLMDBENCH_HARNESS_PVC_SIZE=3Gi
 export LLMDBENCH_VLLM_COMMON_ACCELERATOR_MEMORY=24  # To pass capacity planner sanity checking

setup/env.sh

@@ -124,6 +124,7 @@ export LLMDBENCH_VLLM_COMMON_EXTRA_PVC_NAME=${LLMDBENCH_VLLM_COMMON_EXTRA_PVC_NA
 export LLMDBENCH_VLLM_COMMON_EXTRA_PVC_SIZE="${LLMDBENCH_VLLM_COMMON_EXTRA_PVC_SIZE:-10Gi}"
 export LLMDBENCH_VLLM_COMMON_PVC_STORAGE_CLASS="${LLMDBENCH_VLLM_COMMON_PVC_STORAGE_CLASS:-default}"
 export LLMDBENCH_VLLM_COMMON_PVC_DOWNLOAD_TIMEOUT=${LLMDBENCH_VLLM_COMMON_PVC_DOWNLOAD_TIMEOUT:-"2400"}
+export LLMDBENCH_VLLM_COMMON_PVC_ACCESS_MODE=${LLMDBENCH_VLLM_COMMON_PVC_ACCESS_MODE:-"ReadWriteMany"}
 export LLMDBENCH_VLLM_COMMON_HF_TOKEN_KEY="${LLMDBENCH_VLLM_COMMON_HF_TOKEN_KEY:-"HF_TOKEN"}"
 export LLMDBENCH_VLLM_COMMON_HF_TOKEN_NAME=${LLMDBENCH_VLLM_COMMON_HF_TOKEN_NAME:-"llm-d-hf-token"}
 export LLMDBENCH_VLLM_COMMON_FQDN=${LLMDBENCH_VLLM_COMMON_FQDN:-".svc.cluster.local"}

setup/functions.py

@@ -290,7 +290,7 @@ def llmdbench_execute_cmd(
             announce("(stderr not captured)")
 
     if fatal and ecode != 0:
-        announce(f"\ERROR: Exiting with code {ecode}.")
+        announce(f"ERROR: Exiting with code {ecode}.")
         sys.exit(ecode)
 
     return ecode
@@ -393,6 +393,7 @@ def validate_and_create_pvc(
     pvc_name: str,
     pvc_size: str,
     pvc_class: str,
+    pvc_access_mode: str,
     dry_run: bool = False,
 ):
     announce("Provisioning model storage…")
@@ -453,7 +454,7 @@ def validate_and_create_pvc(
             "namespace": namespace,
         },
         "spec": {
-            "accessModes": ["ReadWriteMany"],
+            "accessModes": [f"{pvc_access_mode}"],
             "resources": {"requests": {"storage": pvc_size}},
             "storageClassName": pvc_class,
             "volumeMode": "Filesystem",
@@ -583,6 +584,7 @@ async def wait_for_job(job_name, namespace, timeout=7200, dry_run: bool = False)
     announce(f"Waiting for job {job_name} to complete...")
 
     if dry_run:
+        announce(f"[DRY RUN] Evaluation job {job_name} completed successfully.")
         return True
 
     # use async config loading
@@ -818,6 +820,8 @@ def get_image(
             announce(f'ERROR: Unable to find latest tag for image "{image_full_name}"')
             sys.exit(1)
 
+        announce(f"INFO: resolved image \"{image_full_name}:{image_tag}\" into \"{image_full_name}:{is_latest_tag}\"")
+
     if tag_only == "1":
         return is_latest_tag
     else:

setup/steps/04_ensure_model_namespace_prepared.py

@@ -99,7 +99,8 @@ def main():
                 pvc_name=ev["vllm_common_pvc_name"],
                 pvc_size=ev["vllm_common_pvc_model_cache_size"],
                 pvc_class=ev["vllm_common_pvc_storage_class"],
-                dry_run=ev["control_dry_run"],
+                pvc_access_mode=ev['vllm_common_pvc_access_mode'],
+                dry_run=ev["control_dry_run"]
             )
 
             validate_and_create_pvc(
@@ -110,6 +111,7 @@ def main():
                 pvc_name=ev["vllm_common_extra_pvc_name"],
                 pvc_size=ev["vllm_common_extra_pvc_size"],
                 pvc_class=ev["vllm_common_pvc_storage_class"],
+                pvc_access_mode=ev['vllm_common_pvc_access_mode'],
                 dry_run=ev["control_dry_run"],
             )
 

setup/steps/05_ensure_harness_namespace_prepared.py

@@ -93,7 +93,8 @@ def main():
               pvc_name=volume,
               pvc_size=ev["harness_pvc_size"],
               pvc_class=ev["vllm_common_pvc_storage_class"],
-              dry_run=ev["control_dry_run"],
+              pvc_access_mode=ev['vllm_common_pvc_access_mode'],
+              dry_run=ev["control_dry_run"]
           )
 
           pod_yaml = f"""apiVersion: v1

setup/steps/10_smoketest.py

@@ -109,20 +109,21 @@ def check_deployment(api: pykube.HTTPClient, client: any, ev: dict):
         current_model_ID_label = model_attribute(model, "modelid_label")
 
     if dry_run:
-        pod_ip_list = "127.0.0.4"
-    try:
-        pod_ip_list = []
-        if is_standalone_deployment(ev):
-            pods = client.CoreV1Api().list_namespaced_pod(namespace=ev["vllm_common_namespace"])
-            for pod in pods.items:
-                if pod_string in pod.metadata.name:
+        pod_ip_list = ["127.0.0.4"]
+    else :
+        try:
+            pod_ip_list = []
+            if is_standalone_deployment(ev):
+                pods = client.CoreV1Api().list_namespaced_pod(namespace=ev["vllm_common_namespace"])
+                for pod in pods.items:
+                    if pod_string in pod.metadata.name:
+                        pod_ip_list.append(pod.status.pod_ip)
+            else:
+                pods = client.CoreV1Api().list_namespaced_pod(namespace=ev["vllm_common_namespace"], label_selector=f"llm-d.ai/model={current_model_ID_label},llm-d.ai/role={pod_string}")
+                for pod in pods.items:
                     pod_ip_list.append(pod.status.pod_ip)
-        else:
-            pods = client.CoreV1Api().list_namespaced_pod(namespace=ev["vllm_common_namespace"], label_selector=f"llm-d.ai/model={current_model_ID_label},llm-d.ai/role={pod_string}")
-            for pod in pods.items:
-                pod_ip_list.append(pod.status.pod_ip)
-    except client.ApiException as e:
-        announce(f"ERROR: Unable to find pods in namespace {ev['vllm_common_namespace']}: {e}")
+        except client.ApiException as e:
+            announce(f"ERROR: Unable to find pods in namespace {ev['vllm_common_namespace']}: {e}")
 
     if not pod_ip_list:
         announce(f"ERROR: Unable to find IPs for pods \"{pod_string}\"!")
@@ -131,7 +132,7 @@ def check_deployment(api: pykube.HTTPClient, client: any, ev: dict):
     for pod_ip in pod_ip_list:
         announce(f"       🚀 Testing pod ip \"{pod_ip}\" ...")
         if dry_run:
-            announce(f"       ✅ Pod ip \"{pod_ip}\" responded successfully ({current_model})")
+            announce(f"       ✅ [DRY RUN] Pod ip \"{pod_ip}\" responded successfully ({current_model})")
         else:
             image_url = get_image(ev['llmd_image_registry'], ev['llmd_image_repo'], ev['llmd_image_name'], ev['llmd_image_tag'])
             received_model_name, curl_command_used = get_model_name_from_pod(ev['vllm_common_namespace'], image_url, pod_ip, ev['vllm_common_inference_port'])
@@ -144,7 +145,7 @@ def check_deployment(api: pykube.HTTPClient, client: any, ev: dict):
     announce(f"🚀 Testing service/gateway \"{service_ip}\" (port 80)...")
 
     if dry_run:
-        announce(f"✅ Service responds successfully ({current_model})")
+        announce(f"✅ [DRY RUN] Service responds successfully ({current_model})")
     else:
         image_url = get_image(ev['llmd_image_registry'], ev['llmd_image_repo'], ev['llmd_image_name'], ev['llmd_image_tag'])
         received_model_name, curl_command_used = get_model_name_from_pod(ev['vllm_common_namespace'], image_url, service_ip, "80")