Merge pull request #820 from mlcommons/dev

Merge Dev

Author: arjunsuresh

.github/workflows/build_wheel_off.yml

@@ -1,6 +1,5 @@
 name: Build wheel and release into PYPI (off now)
 
-
 on: 
   push:
     branches:

.github/workflows/reassign-pr-commits.yml

@@ -0,0 +1,85 @@
+name: Reassign PR Commits
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  reassign:
+    # Only run if the comment is "/reassign" on a Pull Request
+    if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/reassign')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Parse Username
+        id: parse
+        env:
+          COMMENT_BODY: ${{ github.event.comment.body }}
+        run: |
+          # Get the text after /reassign. Default to PR creator if empty.
+          REQUESTED_USER=$(echo "$COMMENT_BODY" | awk '{print $2}')
+          
+          if [ -z "$REQUESTED_USER" ]; then
+            REQUESTED_USER="${{ github.event.issue.user.login }}"
+          fi
+          
+          echo "target_user=$REQUESTED_USER" >> $GITHUB_OUTPUT
+
+      - name: Get User Email from GitHub API
+        id: user_info
+        run: |
+          # Fetch the public email for the target user
+          USER_EMAIL=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/users/${{ steps.parse.outputs.target_user }}" | jq -r '.email')
+          
+          if [ "$USER_EMAIL" == "null" ] || [ -z "$USER_EMAIL" ]; then
+            # Fallback to the GitHub no-reply email format if public email is hidden
+            USER_EMAIL="${{ steps.parse.outputs.target_user }}@users.noreply.github.com"
+          fi
+          
+          echo "email=$USER_EMAIL" >> $GITHUB_OUTPUT
+
+      - name: Checkout PR Branch
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
+
+      - name: Rewrite History
+        run: |
+          # Configure Git
+          git config user.name "${{ steps.parse.outputs.target_user }}"
+          git config user.email "${{ steps.user_info.outputs.email }}"
+          
+          # Use gh CLI to checkout the PR branch locally
+          gh pr checkout ${{ github.event.issue.number }}
+          
+          # Identify the base branch (usually main)
+          BASE_BRANCH=$(gh pr view ${{ github.event.issue.number }} --json baseRefName --template '{{.baseRefName}}')
+          
+          # Rewrite all commits from the point where the PR branched off
+          git filter-branch -f --env-filter "
+            export GIT_AUTHOR_NAME='${{ steps.parse.outputs.target_user }}'
+            export GIT_AUTHOR_EMAIL='${{ steps.user_info.outputs.email }}'
+            export GIT_COMMITTER_NAME='${{ steps.parse.outputs.target_user }}'
+            export GIT_COMMITTER_EMAIL='${{ steps.user_info.outputs.email }}'
+          " origin/$BASE_BRANCH..HEAD
+
+          # Force push the rewritten history
+          git push origin HEAD --force
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: React to Comment
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.reactions.createForIssueComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: context.payload.comment.id,
+              content: 'rocket'
+            })

.github/workflows/test-mlc-script-features.yml

@@ -163,7 +163,7 @@ jobs:
 
         export MLC_SCRIPT_EXTRA_CMD="--adr.python.name=mlperf"
 
-        mlcr run,mlperf,inference,_submission,_short --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=MLCommons --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
+        mlcr run,mlperf,inference --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=MLCommons --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
 
      # Step for Linux/MacOS
     - name: Randomly Execute Step (Linux/MacOS)

script/README.md

@@ -1,6 +1,6 @@
 # MLCommons Automation Scripts
 
-*Last updated: 2026-02-05 19:23:12*
+*Last updated: 2026-02-18 17:10:51*
 
 This directory contains automation scripts for MLPerf benchmarks, AI/ML workflows, and development operations.
 

script/app-mlperf-inference-mlcommons-python/customize.py

@@ -73,7 +73,7 @@ def preprocess(i):
             str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE'])
 
     if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get('MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and (
-            env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'yolo' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL'] or 'pointpainting' in env['MLC_MODEL']) and (env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" or 'pointpainting' in env['MLC_MODEL']):
+            env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'yolo' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL'] or 'pointpainting' in env['MLC_MODEL'] or 'wan-2.2-t2v-a14b' in env['MLC_MODEL']) and (env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" or 'pointpainting' in env['MLC_MODEL']):
         env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \
             env['MLC_MLPERF_LOADGEN_QUERY_COUNT']
 
@@ -638,8 +638,6 @@ def get_run_cmd_reference(
             "classification_and_detection",
             "yolo")
 
-        base_cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} yolo_loadgen.py"""
-
         cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} yolo_loadgen.py \
             --model {x}{env['MLC_ML_MODEL_YOLOV11_PATH']}{x} \
             --dataset-path {x}{env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH']}{x} \
@@ -654,6 +652,30 @@ def get_run_cmd_reference(
         else:
             cmd += " --PerformanceOnly "
 
+    elif "wan-2.2-t2v-a14b" in env['MLC_MODEL']:
+        env['RUN_DIR'] = os.path.join(
+            env['MLC_MLPERF_INFERENCE_SOURCE'],
+            "text_to_video",
+            "wan2.2-t2v-14b",)
+
+        video_output_directory = os.path.join(
+            env['MLC_MLPERF_OUTPUT_DIR'], "generated_videos")
+        os.makedirs(video_output_directory, exist_ok=True)
+
+        torch_distributed_cmd = ""
+
+        if int(env.get('MLC_MLPERF_INFERENCE_NUM_PROCESSES_PER_GPU_NODE', 1)) > 1:
+            torch_distributed_cmd = f"torch.distributed.run --nproc_per_node={env.get('MLC_MLPERF_INFERENCE_NUM_PROCESSES_PER_GPU_NODE')}"
+
+        cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} {torch_distributed_cmd} run_mlperf.py \
+            --model-path {x}{env['MLC_ML_MODEL_WAN2_PATH']}{x} \
+            --dataset {x}{env['MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH']}{x} \
+            --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \
+            --output-dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \
+            --video_output_path {x}{video_output_directory}{x} \
+            {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \
+            {scenario_extra_options} {mode_extra_options}"""
+
     if env.get('MLC_NETWORK_LOADGEN', '') in ["lon", "sut"]:
         cmd = cmd + " " + "--network " + env['MLC_NETWORK_LOADGEN']
         if env.get('MLC_NETWORK_LOADGEN_SUT_SERVERS', []):

script/app-mlperf-inference-mlcommons-python/meta.yaml

@@ -311,6 +311,7 @@ deps:
         - bert-99.9
         - gptj-99
         - gptj-99.9
+        - wan-2.2-t2v-a14b
 
   ## Tensorflow
   - tags: get,generic-python-lib,_tensorflow
@@ -1569,15 +1570,13 @@ variations:
       names:
         - yolo-v11-model-download
       skip_if_env:
-        MLC_USE_ML_MODEL_FROM_HOST: 
-        - true
+        MLC_RUN_STATE_DOCKER:
         - "yes"
     - tags: get,dataset,mlperf-inference,yolo-coco2017-filtered,_mlc,_r2-downloader
       names:
         - yolo-v11-dataset-download
       skip_if_env:
-        MLC_USE_DATASET_FROM_HOST: 
-        - true
+        MLC_RUN_STATE_DOCKER:
         - "yes"
     - tags: get,generic-python-lib,_package.ultralytics
     - tags: get,generic-python-lib,_package.tqdm
@@ -1598,6 +1597,52 @@ variations:
       - yolo_
     env:
       MLC_MODEL: yolo-99
+
+  wan-2.2-t2v-a14b:
+    group: models
+    add_deps_recursive:
+      pytorch:
+        version_min: "2.5.1"
+        version_max: "2.5.1"
+        version_max_usable: "2.5.1"
+      ml-engine-transformers:
+        version_max: "4.51.3"
+        version_max_usable: "4.51.3"
+      torchvision:
+        version_min: "0.20.1"
+        version_max: "0.20.1"
+        version_max_usable: "0.20.1"  
+    env:
+      MLC_MODEL: wan-2.2-t2v-a14b
+    deps:
+    - tags: get-ml-model-wan2,_mlc,_r2-downloader,_wan2_2_t2v_a14b
+      names:
+        - wan2-model-download
+      skip_if_env:
+        MLC_RUN_STATE_DOCKER:
+        - "yes"
+    - tags: get-dataset-mlperf-inference-text-to-video,_mlc,_r2-downloader
+      names:
+        - wan2-dataset-download
+      skip_if_env:
+        MLC_RUN_STATE_DOCKER:
+        - "yes"
+    - tags: get,generic-python-lib,_package.opencv-python
+      version_max: "4.11.0.86"
+      version_max_usable: "4.11.0.86"
+    - tags: get,generic-python-lib,_package.imageio
+    - tags: get,generic-python-lib,_package.imageio-ffmpeg
+    - tags: get,generic-python-lib,_package.diffusers
+      version_max: "0.35.2"
+      version_max_usable: "0.35.2"
+    - tags: get,generic-python-lib,_package.accelerate
+    - tags: get,generic-python-lib,_package.ftfy
+    - tags: get,generic-python-lib,_package.numpy
+      names:
+        - numpy
+    - tags: get,generic-python-lib,_package.flash-attn
+      version_min: "2.8.3"
+      version_max_usable: "2.8.3"
 
   # Target devices
   cpu:

script/app-mlperf-inference/README.md

@@ -1,5 +1,5 @@
 # README for app-mlperf-inference
-This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution.
+This README is automatically generated. Create and add custom content in info.md. Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution.
 
 `mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do
 ```
@@ -85,10 +85,12 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic
 | `--output` | Output from the script passed using the env key `MLC_OUTPUT` |  | `` |
 | `--outdirname` | The directory to store the script output |  | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` |
 | `--outbasename` | The output file/folder name |  | `` |
+| `--search_folder_path` | The folder path where executables of a given script need to be searched. Search is done recursively upto 4 levels. |  | `` |
 | `--name` |  |  | `` |
 | `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved |  | `` |
 | `--skip_compile` | Skip compilation |  | `False` |
 | `--skip_run` | Skip run |  | `False` |
+| `--skip_sudo` | Skip SUDO detection |  | `False` |
 | `--accept_license` | Accept the required license requirement to run the script |  | `False` |
 | `--skip_system_deps` | Skip installing any system dependencies |  | `False` |
 | `--git_ssh` | Use SSH for git repos |  | `False` |
@@ -175,7 +177,10 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic
 - `rgat`
 - `rnnt`
 - `sdxl`
+- `wan-2.2-t2v-a14b`
 - `whisper`
+- `yolo-95` (base: yolo_)
+- `yolo-99` (base: yolo_)
 
 ### Precision
 
@@ -201,7 +206,7 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic
 - `r5.0_default`
 - `r5.1-dev_default`
 - `r5.1_default`
-- `r6.0-dev_default`
+- `r6.0-dev_default` (default)
 
 ### Ungrouped
 
@@ -214,3 +219,4 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic
 - `llama3_1-8b` (base: llama3_1-8b_)
 - `llama3_1-8b-edge` (base: llama3_1-8b_)
 - `power`
+- `yolo_`

script/app-mlperf-inference/customize.py

@@ -180,6 +180,14 @@ def postprocess(i):
             out_baseline_accuracy_string = f""" --output-file {q}{os.path.join(output_dir, "accuracy", "baseline_accuracy.txt")}{q} """
             out_compliance_accuracy_string = f""" --output-file {q}{os.path.join(output_dir, "accuracy", "compliance_accuracy.txt")}{q} """
 
+        elif 'yolo' in model:
+            accuracy_filename = "accuracy-coco.py"
+            accuracy_filepath = os.path.join(
+                env['MLC_MLPERF_INFERENCE_CLASSIFICATION_AND_DETECTION_PATH'], "tools", accuracy_filename)
+            dataset_args = f""" --coco-dir {q}{env.get('MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH')}{q} """
+            accuracy_log_file_option_name = " --mlperf-accuracy-file "
+            datatype_option = ""
+
         elif 'stable-diffusion-xl' in model:
             pass  # No compliance check for now
         elif 'gpt' in model: