[VLM] Fixing request timeout error, and enable VllmDeployer to fail fast if the underying vllm serve process already failed (#2409)

* enable VllmDeployer to fail fast if the underying vllm process failed.

* example slurm script for submitting jobs

* fix slurm scripts

* small fix

* [Automated Commit] Format Codebase

* Update the readme about the example slurm scripts.

* Change the default endpoint startup timeout to 1 hour in case someone need to download the model for the fisrt time.

* change servr expected qps and target latency

* Change the default dataset repo_id to the new name of the public dataset

* [Automated Commit] Format Codebase

* evaluate the json file with multiprocess

* [Automated Commit] Format Codebase

* change default server_target_latency to 12

* revert evaluation changeS

* [Automated Commit] Format Codebase

* update slurm script

* update slurm script

* revert evaluation.py changes after analysing the discrepancy in is_secondhand f1 score

* [Automated Commit] Format Codebase

* linting

* [Automated Commit] Format Codebase

* lock in model and dataset SHA

* [Automated Commit] Format Codebase

* Specify model quality target and server target latency in the README

* Update loadgen/mlperf.conf

* aligning TestSettings'C++ code with its python binding

* [Automated Commit] Format Codebase

* remove ttft and tpot from mlperf.conf

* Enable CLI to take in user.conf

* [Automated Commit] Format Codebase

* readme

* readme

* rename vl2l -> q3vl

* [Automated Commit] Format Codebase

* empty

* rerun ci

* rerun ci

* Introduce sampling parameters

* [Automated Commit] Format Codebase

* [Automated Commit] Format Codebase

* empty

* move CFLAGS="-std=c++14 -O3" into extra_compile_args of Pybind11Extension

* [Automated Commit] Format Codebase

* enable specifying loadgen source in the Dockerfile

* update slurm scripts

* Maintain None as the default value for the sampling params

* [Automated Commit] Format Codebase

* update readme

* [Automated Commit] Format Codebase

* empty

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: John Calderon <jcalderon@nvidia.com>

Author: 3 people

loadgen/README_BUILD.md

@@ -12,7 +12,7 @@
     pip install absl-py numpy
     git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference
     cd mlperf_inference/loadgen
-    CFLAGS="-std=c++14 -O3" python -m pip install .
+    python -m pip install .
 
 This will fetch the loadgen source, build and install the loadgen as a python module, and run a simple end-to-end demo.
 

loadgen/bindings/python_api.cc

@@ -312,6 +312,8 @@ PYBIND11_MODULE(mlperf_loadgen, m) {
                      &TestSettings::server_num_issue_query_threads)
       .def_readwrite("offline_expected_qps",
                      &TestSettings::offline_expected_qps)
+      .def_readwrite("sample_concatenate_permutation",
+                     &TestSettings::sample_concatenate_permutation)
       .def_readwrite("min_duration_ms", &TestSettings::min_duration_ms)
       .def_readwrite("max_duration_ms", &TestSettings::max_duration_ms)
       .def_readwrite("min_query_count", &TestSettings::min_query_count)
@@ -324,6 +326,14 @@ PYBIND11_MODULE(mlperf_loadgen, m) {
                      &TestSettings::accuracy_log_rng_seed)
       .def_readwrite("accuracy_log_probability",
                      &TestSettings::accuracy_log_probability)
+      .def_readwrite("accuracy_log_sampling_target",
+                     &TestSettings::accuracy_log_sampling_target)
+      .def_readwrite("test05", &TestSettings::test05)
+      .def_readwrite("test05_qsl_rng_seed", &TestSettings::test05_qsl_rng_seed)
+      .def_readwrite("test05_sample_index_rng_seed",
+                     &TestSettings::test05_sample_index_rng_seed)
+      .def_readwrite("test05_schedule_rng_seed",
+                     &TestSettings::test05_schedule_rng_seed)
       .def_readwrite("print_timestamps", &TestSettings::print_timestamps)
       .def_readwrite("performance_issue_unique",
                      &TestSettings::performance_issue_unique)
@@ -333,12 +343,6 @@ PYBIND11_MODULE(mlperf_loadgen, m) {
                      &TestSettings::performance_issue_same_index)
       .def_readwrite("performance_sample_count_override",
                      &TestSettings::performance_sample_count_override)
-      .def_readwrite("test05", &TestSettings::test05)
-      .def_readwrite("test05_qsl_rng_seed", &TestSettings::test05_qsl_rng_seed)
-      .def_readwrite("test05_sample_index_rng_seed",
-                     &TestSettings::test05_sample_index_rng_seed)
-      .def_readwrite("test05_schedule_rng_seed",
-                     &TestSettings::test05_schedule_rng_seed)
       .def_readwrite("use_token_latencies", &TestSettings::use_token_latencies)
       .def_readwrite("ttft_latency", &TestSettings::server_ttft_latency)
       .def_readwrite("tpot_latency", &TestSettings::server_tpot_latency)

loadgen/mlperf.conf

@@ -27,6 +27,7 @@ pointpainting.*.performance_sample_count_override = 1024
 deepseek-r1.*.performance_sample_count_override = 4388
 deepseek-r1-interactive.*.performance_sample_count_override = 4388
 whisper.*.performance_sample_count_override = 1633
+qwen3-vl-235b-a22b.*.performance_sample_count_override = 48289
 # set to 0 to let entire sample set to be performance sample
 3d-unet.*.performance_sample_count_override = 0
 
@@ -69,7 +70,7 @@ llama3_1-8b-interactive.*.sample_concatenate_permutation = 1
 deepseek-r1.*.sample_concatenate_permutation = 1
 deepseek-r1-interactive.*.sample_concatenate_permutation = 1
 whisper.*.sample_concatenate_permutation = 1
-
+qwen3-vl-235b-a22b.*.sample_concatenate_permutation = 1
 *.Server.target_latency = 10
 *.Server.target_latency_percentile = 99
 *.Server.target_duration = 0
@@ -94,7 +95,9 @@ llama3_1-8b-interactive.*.use_token_latencies = 1
 deepseek-r1.*.use_token_latencies = 1
 deepseek-r1-interactive.*.use_token_latencies = 1
 whisper.*.use_token_latencies = 1
-
+# For the VLM benchmark, the model response is relatively short, therefore we track 
+# end-to-end latency instead of token latencies.
+qwen3-vl-235b-a22b.*.use_token_latencies = 0
 # gptj benchmark infers token latencies
 gptj.*.infer_token_latencies = 1
 gptj.*.token_latency_scaling_factor = 69
@@ -140,6 +143,8 @@ deepseek-r1-interactive.Server.target_latency = 0
 deepseek-r1-interactive.Server.ttft_latency = 1500
 deepseek-r1-interactive.Server.tpot_latency = 15
 
+qwen3-vl-235b-a22b.Server.target_latency = 12000
+
 *.Offline.target_latency_percentile = 90
 *.Offline.min_duration = 600000
 
@@ -164,6 +169,7 @@ mixtral-8x7b.Offline.min_query_count = 15000
 rgat.Offline.min_query_count = 788379
 deepseek-r1.Offline.min_query_count = 4388
 whisper.Offline.min_query_count = 1633
+qwen3-vl-235b-a22b.Offline.min_query_count = 48289
 
 # These fields should be defined and overridden by user.conf.
 *.SingleStream.target_latency = 10

loadgen/setup.py

@@ -24,12 +24,12 @@
 #  and binaries. Use one of the gn build targets instead if you want
 #  to avoid poluting the source tree.
 
-from setuptools import Extension, setup
 from pathlib import Path
+
 from pybind11 import get_include
 from pybind11.setup_helpers import Pybind11Extension, build_ext
+from setuptools import setup
 from version_generator import generate_loadgen_version_definitions
-import subprocess
 
 generated_version_source_filename = "generated/version_generated.cc"
 generate_loadgen_version_definitions(generated_version_source_filename, ".")
@@ -42,7 +42,7 @@
     "test_settings.h",
     "issue_query_controller.h",
     "early_stopping.h",
-    "query_dispatch_library.h"
+    "query_dispatch_library.h",
 ]
 
 lib_headers = [
@@ -54,7 +54,7 @@
     "results.h",
     "bindings/c_api.h",
     "version_generator.py",
-    "mlperf_conf.h"
+    "mlperf_conf.h",
 ]
 
 lib_sources = [
@@ -93,18 +93,18 @@
 
 
 try:
-    with open("mlperf.conf", 'r') as file:
+    with open("mlperf.conf", "r") as file:
         conf_contents = file.read()
 
     # Escape backslashes and double quotes
-    conf_contents = conf_contents.replace('\\', '\\\\').replace('"', '\\"')
+    conf_contents = conf_contents.replace("\\", "\\\\").replace('"', '\\"')
 
     # Convert newlines
-    conf_contents = conf_contents.replace('\n', '\\n"\n"')
+    conf_contents = conf_contents.replace("\n", '\\n"\n"')
 
     formatted_content = f'const char* mlperf_conf =\n"{conf_contents}";\n'
 
-    with open("mlperf_conf.h", 'w') as header_file:
+    with open("mlperf_conf.h", "w") as header_file:
         header_file.write(formatted_content)
 
 except IOError as e:
@@ -113,24 +113,25 @@
 mlperf_loadgen_module = Pybind11Extension(
     "mlperf_loadgen",
     define_macros=[
-        ("MAJOR_VERSION",
-         version_split[0]),
-        ("MINOR_VERSION",
-         version_split[1])
+        ("MAJOR_VERSION", version_split[0]),
+        ("MINOR_VERSION", version_split[1]),
     ],
     include_dirs=[".", get_include()],
     sources=mlperf_loadgen_sources,
     depends=mlperf_loadgen_headers,
+    extra_compile_args=["-std=c++14", "-O3"],
 )
 
-setup(name="mlcommons_loadgen",
-      version=version,
-      description="MLPerf Inference LoadGen python bindings",
-      url="https://mlcommons.org/",
-      cmdclass={"build_ext": build_ext},
-      ext_modules=[mlperf_loadgen_module],
-      packages=['mlcommons_loadgen'],
-      package_dir={'mlcommons_loadgen': '.'},
-      include_package_data=True,
-      long_description=mlperf_long_description,
-      long_description_content_type='text/markdown')
+setup(
+    name="mlcommons_loadgen",
+    version=version,
+    description="MLPerf Inference LoadGen python bindings",
+    url="https://mlcommons.org/",
+    cmdclass={"build_ext": build_ext},
+    ext_modules=[mlperf_loadgen_module],
+    packages=["mlcommons_loadgen"],
+    package_dir={"mlcommons_loadgen": "."},
+    include_package_data=True,
+    long_description=mlperf_long_description,
+    long_description_content_type="text/markdown",
+)

loadgen/test_settings.h

@@ -234,10 +234,6 @@ struct TestSettings {
   uint64_t test05_qsl_rng_seed = 0;
   uint64_t test05_sample_index_rng_seed = 0;
   uint64_t test05_schedule_rng_seed = 0;
-
-  /// \brief Load mlperf parameter config from file.
-  int FromConfig(const std::string &path, const std::string &model,
-                 const std::string &scenario, int conf_type = 1);
   /**@}*/
 
   // ==================================
@@ -272,6 +268,10 @@ struct TestSettings {
   bool infer_token_latencies = false;
   uint64_t token_latency_scaling_factor;
   /**@}*/
+
+  /// \brief Load mlperf parameter config from file.
+  int FromConfig(const std::string &path, const std::string &model,
+                 const std::string &scenario, int conf_type = 1);
 };
 
 ///