Merge branch 'master' into submission-guide

Author: arjunsuresh

.github/workflows/build_wheels.yml

@@ -1,4 +1,5 @@
 name: Build loadgen wheels and release them into PYPI
+
 on:
   release:
     types: [published]
@@ -73,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-15]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/test-resnet50.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9" ]
+        python-version: [ "3.10", "3.13" ]
         backend: [ "onnxruntime", "tf" ]
         loadgen-flag: [ "", "--adr.loadgen.tags=_from-pip --pip_loadgen=yes" ]
 

.github/workflows/test-retinanet.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.10" ]
+        python-version: [ "3.10", "3.13" ]
         backend: [ "onnxruntime", "pytorch" ]
 
     steps:

.github/workflows/test-rgat.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.11" ]
+        python-version: [ "3.11", "3.12" ]
         backend: [ "pytorch" ]
         loadgen-flag: [ "" ]
 

.github/workflows/test-submission-checker.yml

@@ -5,7 +5,7 @@ name: Test for MLPerf inference submission checker using MLC script automation
 
 on:
   pull_request:
-    branches: [ "master", "dev" ]
+    branches: [ "master_off", "dev_off" ]
     paths:
       - tools/submission/**
       - .github/workflows/test-submission-checker.yml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.11", "3.12" ]
+        python-version: [ "3.10", "3.12", "3.14" ]
 
     steps:
     - uses: actions/checkout@v3
@@ -31,8 +31,10 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install mlc-scripts
+        python3 -m pip install mlcflow
+        mlc pull repo mlcommons@mlperf-automations --branch=sub-gen-change
         git clone https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --depth 1
+      # python3 -m pip install mlc-scripts
     - name: Test MLPerf inference submission checker
       run: |
-        mlcr run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/mlperf_inference_unofficial_submissions_v5.0 --src_version=v5.0 --extra_args=" --skip-extra-files-in-root-check --skip-extra-accuracy-files-check --skip-all-systems-have-results-check --skip-calibration-check" --quiet 
+        mlcr run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.submission_dir --adr.inference-src.version=custom --input=`pwd`/mlperf_inference_unofficial_submissions_v5.0 --src_version=v5.0 --extra_args=" --skip-extra-files-in-root-check --skip-extra-accuracy-files-check --skip-all-systems-have-results-check --skip-calibration-check" --quiet 

.gitignore

@@ -3,3 +3,6 @@ libmlperf_loadgen.a
 __pycache__/
 generated/
 *.swp
+*.egg-info/
+*.so
+.vscode/

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "language/deepseek-r1/submodules/LiveCodeBench"]
 	path = language/deepseek-r1/submodules/LiveCodeBench
 	url = https://github.com/LiveCodeBench/LiveCodeBench
+[submodule "text_to_video/wan2.2-t2v-14b/submodules/VBench"]
+	path = text_to_video/wan2.2-t2v-14b/submodules/VBench
+	url = https://github.com/Vchitect/VBench

docs/benchmarks/language/scc25_guide/scc25.md

@@ -87,20 +87,10 @@ mlcr generate,inference,submission,_wg-inference \
 
 ### Submit Results
 
-> **Note:**
-Further instructions on the final submission will be published as the deadline approaches.
-
-<!-- Fork the `mlperf-inference-results-scc25` branch of the repository URL at [mlperf-automations](https://github.com/mlcommons/mlperf-automations). 
+MLCommons provides students with a [Submission UI](https://submissions-ui.mlcommons.org/index), where they can upload the generated **.tar** file using their assigned submission ID.
 
-Run the following command after **replacing `--repo_url` with your GitHub fork URL**.
+The deadline for submitting results is 6:00 PM CDT on November 17 (Monday), 2025.
 
-```bash
-mlcr push,github,mlperf,inference,submission \
-   --repo_url=https://github.com/<myfork>/mlperf-automations \
-   --repo_branch=mlperf-inference-results-scc25 \
-   --commit_message="Results on system <HW Name>" \
-   --quiet
-```
+Alternatively, students may use the Submission CLI provided through the MLCFlow automation. To do this, first follow the installation steps in this [guide](../../../install/index.md).
+After installing, follow the instructions under [**Upload the final submission**](https://docs.mlcommons.org/inference/submission/#upload-the-final-submission).
 
-Once uploaded give a Pull Request to the origin repository. Github action will be running there and once 
-finished you can see your submitted results at [https://docs.mlcommons.org/mlperf-automations](https://docs.mlcommons.org/mlperf-automations). -->

graph/R-GAT/tools/download_igbh_full.sh

@@ -33,74 +33,93 @@ cd $1/full/processed
 mkdir paper
 cd paper
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/node_feat.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/node_label_19.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/node_label_19.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/node_label_2K.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/node_label_2K.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/paper_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/paper_id_index_mapping.npy"; exit $?; }
 cd ..
 
 # paper__cites__paper
 mkdir paper__cites__paper
 cd paper__cites__paper
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__cites__paper/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__cites__paper/edge_index.npy"; exit $?; }
 cd ..
 
 # author
 mkdir author
 cd author
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/author/author_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: author/author_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/author/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: author/node_feat.npy"; exit $?; }
 cd ..
 
 # conference
 mkdir conference
 cd conference
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/conference/conference_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: conference/conference_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/conference/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: conference/node_feat.npy"; exit $?; }
 cd ..
 
 # institute
 mkdir institute
 cd institute
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/institute/institute_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: institute/institute_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/institute/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: institute/node_feat.npy"; exit $?; }
 cd ..
 
 # journal
 mkdir journal
 cd journal
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/journal/journal_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: journal/journal_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/journal/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: journal/node_feat.npy"; exit $?; }
 cd ..
 
 # fos
 mkdir fos
 cd fos
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/fos/fos_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: fos/fos_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/fos/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: fos/node_feat.npy"; exit $?; }
 cd ..
 
 # author__affiliated_to__institute
 mkdir author__affiliated_to__institute
 cd author__affiliated_to__institute
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/author__affiliated_to__institute/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: author__affiliated_to__institute/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__published__journal
 mkdir paper__published__journal
 cd paper__published__journal
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__published__journal/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__published__journal/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__topic__fos
 mkdir paper__topic__fos
 cd paper__topic__fos
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__topic__fos/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__topic__fos/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__venue__conference
 mkdir paper__venue__conference
 cd paper__venue__conference
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__venue__conference/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__venue__conference/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__written_by__author
@@ -109,4 +128,4 @@ cd paper__written_by__author
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__written_by__author/edge_index.npy
 cd ..
 
-echo "IGBH-IGBH download complete"
+echo "IGBH-IGBH download complete"

language/deepseek-r1/README.md

@@ -23,6 +23,7 @@ bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/he
 ```
 
 To specify a custom download directory, use the `-d` flag:
+
 ```bash
 bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) \
   -d /path/to/download/directory \
@@ -53,6 +54,7 @@ bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/he
 This will download the full preprocessed dataset file (`mlperf_deepseek_r1_dataset_4388_fp8_eval.pkl`) and the calibration dataset file (`mlperf_deepseek_r1_calibration_dataset_500_fp8_eval.pkl`).
 
 To specify a custom download directory, use the `-d` flag:
+
 ```bash
 bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) \
   -d /path/to/download/directory \
@@ -103,6 +105,8 @@ Launch a Docker container with your preferred backend:
 - **vllm**: vLLM's LLM api-based inference
 - **sglang**: sglang's OpenAI endpoint-based inference
 
+**NOTE**: `sglang` backend uses `sglang==0.5.4` installed into `lmsysorg/sglang:v0.5.2-cu129-b200` base image.
+
 ## Backend-Specific Setup
 
 After launching any Docker container, run the setup script which automatically detects your backend:
@@ -115,6 +119,7 @@ setup.sh
 The setup script creates a virtual environment and configures it differently based on the backend:
 
 #### All Backends
+
 - Virtual environment is **activated** after `setup.sh`
 - Activate backend-specific venv using `source .venv_[pytorch|vllm|sglang]/bin/activate`
 - All commands are to be run using the virtual environment
@@ -159,6 +164,7 @@ The reference implementation includes full support for MLPerf inference benchmar
 ### Running MLPerf Benchmarks
 
 #### Offline Scenario
+
 ```bash
 (.venv_BACKEND) $ python run_mlperf.py \
     --mode offline \
@@ -167,13 +173,25 @@ The reference implementation includes full support for MLPerf inference benchmar
 ```
 
 #### Server Scenario
+
 ```bash
 (.venv_BACKEND) $ python run_mlperf.py \
     --mode server \
     --input-file <input_dataset>.pkl \
     --output-dir mlperf_results
 ```
 
+#### Interactive Scenario
+
+```bash
+(.venv_BACKEND) $ python run_mlperf.py \
+    --mode interactive \
+    --input-file <input_dataset>.pkl \
+    --output-dir mlperf_results
+```
+
+**NOTE:** to enable Speculative Decoding for Sglang Backend, toggle `BACKEND_REGISTRY['sglang']['enable_speculative_decode']` in `utils/backend_registry.py` (disabled by default).
+
 #### Pytorch Backend for Mlperf
 
 PyTorch backend uses distributed execution with `torchrun` and `run_mlperf_mpi.py`:
@@ -188,24 +206,36 @@ PyTorch backend uses distributed execution with `torchrun` and `run_mlperf_mpi.p
 
 ### MLPerf Command Line Options
 
-| Option         | Description                    | Default          |
-| -------------- | ------------------------------ | ---------------- |
-| `--mode`       | Scenario mode (offline/server) | `offline`        |
-| `--accuracy`   | Run accuracy test              | `False`          |
-| `--output-dir` | Output directory for results   | `mlperf_results` |
+| Option         | Description                                | Default          |
+| -------------- | ------------------------------------------ | ---------------- |
+| `--mode`       | Scenario mode (offline/server/interactive) | `offline`        |
+| `--accuracy`   | Run accuracy test                          | `False`          |
+| `--output-dir` | Output directory for results               | `mlperf_results` |
 
 ### Backend Support Matrix
 
 The following table shows which backends support different evaluation and MLPerf operations:
 
-| Backend     | `run_eval.py` | `run_mlperf.py --mode=offline` | `run_mlperf.py --mode=server` |
-| ----------- | ------------- | ------------------------------ | ----------------------------- |
-| pytorch-fp8 | x             | x                              |                               |
-| vllm-fp8    | x             | x                              |                               |
-| sglang-fp8  | x             | x                              | x                             |
+| Backend     | `run_eval.py` | `run_mlperf.py --mode=offline` | `run_mlperf.py --mode=server` | `run_mlperf.py --mode=interactive` |
+| ----------- | ------------- | ------------------------------ | ----------------------------- | ---------------------------------- |
+| pytorch-fp8 | x             | x                              |                               |                                    |
+| vllm-fp8    | x             | x                              |                               |                                    |
+| sglang-fp8  | x             | x                              | x                             | x                                  |
 
 > **Note**: For PyTorch backend, use the `_mpi` versions with `torchrun`. For vLLM and SGLang backends, use the single-process versions without `_mpi`.
 
+## Speculative Decoding
+
+For the DeepSeek-R1 Interactive Scenario, users can enable Speculative Decoding Optimization for the SGLANG Backend by setting the `enable_speculative_decode` flag to `True` in `language/deepseek-r1/utils/backend_registry.py`.
+
+When Enabled, SGLANG backend will run the allowed configuration as per [Inference Policies](https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc) (appendix-speculative-decoding):
+
+| Benchmark   | Scenario    | Speculative Decoding Algorithm                             | Configuration                                           | MTP Head                                       |
+| :---------- | :---------- | :--------------------------------------------------------- | :------------------------------------------------------ | :--------------------------------------------- |
+| DeepSeek-r1 | Interactive | EAGLE-style decoding with deepseek-ai/deepseek-r1 MTP head | `speculative-num-steps=3`, `speculative-eagle-topk=1.0` | https://huggingface.co/deepseek-ai/DeepSeek-R1 |
+
+> Note: ONLY Sglang backend supports speculative-decoding
+
 ## Accuracy Evaluation
 
 **Using MLCFlow Automation**