Support Text to Video implementation (#798)

Author: anandhu-eng

.github/workflows/build_wheel_off.yml

@@ -1,6 +1,5 @@
 name: Build wheel and release into PYPI (off now)
 
-
 on: 
   push:
     branches:

script/app-mlperf-inference-mlcommons-python/customize.py

@@ -73,7 +73,7 @@ def preprocess(i):
             str(env['MLC_MLPERF_LOADGEN_BATCH_SIZE'])
 
     if env.get('MLC_MLPERF_LOADGEN_QUERY_COUNT', '') != '' and not env.get('MLC_TMP_IGNORE_MLPERF_QUERY_COUNT', False) and (
-            env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'yolo' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL'] or 'pointpainting' in env['MLC_MODEL']) and (env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" or 'pointpainting' in env['MLC_MODEL']):
+            env['MLC_MLPERF_LOADGEN_MODE'] == 'accuracy' or 'gptj' in env['MLC_MODEL'] or 'llama2' in env['MLC_MODEL'] or 'yolo' in env['MLC_MODEL'] or 'mixtral' in env['MLC_MODEL'] or 'llama3' in env['MLC_MODEL'] or 'pointpainting' in env['MLC_MODEL'] or 'wan-2.2-t2v-a14b' in env['MLC_MODEL']) and (env.get('MLC_MLPERF_RUN_STYLE', '') != "valid" or 'pointpainting' in env['MLC_MODEL']):
         env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --count " + \
             env['MLC_MLPERF_LOADGEN_QUERY_COUNT']
 
@@ -638,8 +638,6 @@ def get_run_cmd_reference(
             "classification_and_detection",
             "yolo")
 
-        base_cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} yolo_loadgen.py"""
-
         cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} yolo_loadgen.py \
             --model {x}{env['MLC_ML_MODEL_YOLOV11_PATH']}{x} \
             --dataset-path {x}{env['MLC_ML_DATASET_MLPERF_INFERENCE_YOLO_COCO2017_FILTERED_DATASET_PATH']}{x} \
@@ -654,6 +652,30 @@ def get_run_cmd_reference(
         else:
             cmd += " --PerformanceOnly "
 
+    elif "wan-2.2-t2v-a14b" in env['MLC_MODEL']:
+        env['RUN_DIR'] = os.path.join(
+            env['MLC_MLPERF_INFERENCE_SOURCE'],
+            "text_to_video",
+            "wan2.2-t2v-14b",)
+
+        video_output_directory = os.path.join(
+            env['MLC_MLPERF_OUTPUT_DIR'], "generated_videos")
+        os.makedirs(video_output_directory, exist_ok=True)
+
+        torch_distributed_cmd = ""
+
+        if int(env.get('MLC_MLPERF_INFERENCE_NUM_PROCESSES_PER_GPU_NODE', 1)) > 1:
+            torch_distributed_cmd = f"torch.distributed.run --nproc_per_node={env.get('MLC_MLPERF_INFERENCE_NUM_PROCESSES_PER_GPU_NODE')}"
+
+        cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} {torch_distributed_cmd} run_mlperf.py \
+            --model-path {x}{env['MLC_ML_MODEL_WAN2_PATH']}{x} \
+            --dataset {x}{env['MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH']}{x} \
+            --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \
+            --output-dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \
+            --video_output_path {x}{video_output_directory}{x} \
+            {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \
+            {scenario_extra_options} {mode_extra_options}"""
+
     if env.get('MLC_NETWORK_LOADGEN', '') in ["lon", "sut"]:
         cmd = cmd + " " + "--network " + env['MLC_NETWORK_LOADGEN']
         if env.get('MLC_NETWORK_LOADGEN_SUT_SERVERS', []):

script/app-mlperf-inference-mlcommons-python/meta.yaml

@@ -311,6 +311,7 @@ deps:
         - bert-99.9
         - gptj-99
         - gptj-99.9
+        - wan-2.2-t2v-a14b
 
   ## Tensorflow
   - tags: get,generic-python-lib,_tensorflow
@@ -1569,15 +1570,13 @@ variations:
       names:
         - yolo-v11-model-download
       skip_if_env:
-        MLC_USE_ML_MODEL_FROM_HOST: 
-        - true
+        MLC_RUN_STATE_DOCKER:
         - "yes"
     - tags: get,dataset,mlperf-inference,yolo-coco2017-filtered,_mlc,_r2-downloader
       names:
         - yolo-v11-dataset-download
       skip_if_env:
-        MLC_USE_DATASET_FROM_HOST: 
-        - true
+        MLC_RUN_STATE_DOCKER:
         - "yes"
     - tags: get,generic-python-lib,_package.ultralytics
     - tags: get,generic-python-lib,_package.tqdm
@@ -1598,6 +1597,52 @@ variations:
       - yolo_
     env:
       MLC_MODEL: yolo-99
+
+  wan-2.2-t2v-a14b:
+    group: models
+    add_deps_recursive:
+      pytorch:
+        version_min: "2.5.1"
+        version_max: "2.5.1"
+        version_max_usable: "2.5.1"
+      ml-engine-transformers:
+        version_max: "4.51.3"
+        version_max_usable: "4.51.3"
+      torchvision:
+        version_min: "0.20.1"
+        version_max: "0.20.1"
+        version_max_usable: "0.20.1"  
+    env:
+      MLC_MODEL: wan-2.2-t2v-a14b
+    deps:
+    - tags: get-ml-model-wan2,_mlc,_r2-downloader,_wan2_2_t2v_a14b
+      names:
+        - wan2-model-download
+      skip_if_env:
+        MLC_RUN_STATE_DOCKER:
+        - "yes"
+    - tags: get-dataset-mlperf-inference-text-to-video,_mlc,_r2-downloader
+      names:
+        - wan2-dataset-download
+      skip_if_env:
+        MLC_RUN_STATE_DOCKER:
+        - "yes"
+    - tags: get,generic-python-lib,_package.opencv-python
+      version_max: "4.11.0.86"
+      version_max_usable: "4.11.0.86"
+    - tags: get,generic-python-lib,_package.imageio
+    - tags: get,generic-python-lib,_package.imageio-ffmpeg
+    - tags: get,generic-python-lib,_package.diffusers
+      version_max: "0.35.2"
+      version_max_usable: "0.35.2"
+    - tags: get,generic-python-lib,_package.accelerate
+    - tags: get,generic-python-lib,_package.ftfy
+    - tags: get,generic-python-lib,_package.numpy
+      names:
+        - numpy
+    - tags: get,generic-python-lib,_package.flash-attn
+      version_min: "2.8.3"
+      version_max_usable: "2.8.3"
 
   # Target devices
   cpu:

script/app-mlperf-inference/meta.yaml

@@ -317,6 +317,10 @@ variations:
   reference,yolo_:
     default_variations:
       backend: pytorch
+  
+  reference,wan-2.2-t2v-a14b:
+    default_variations:
+      backend: pytorch
 
   all-models,nvidia-original:
     docker:
@@ -1698,22 +1702,22 @@ variations:
         MLC_ML_MODEL_FILE_WITH_PATH: "on"
 
   yolo_:
-    deps:
-    - tags: get-ml-model-yolov11,_mlc,_r2-downloader
-      names:
-        - yolo-v11-model-download
-      skip_if_env:
-        MLC_USE_ML_MODEL_FROM_HOST: 
-        - true
-        - "yes"
-    - tags: get,dataset,mlperf-inference,yolo-coco2017-filtered,_mlc,_r2-downloader
-      names:
-        - yolo-v11-dataset-download
-      skip_if_env:
-        MLC_USE_DATASET_FROM_HOST: 
-        - true
-        - "yes"
     docker:
+      deps:
+      - tags: get-ml-model-yolov11,_mlc,_r2-downloader
+        names:
+          - yolo-v11-model-download
+        enable_if_env:
+          MLC_USE_ML_MODEL_FROM_HOST: 
+          - true
+          - "yes"
+      - tags: get,dataset,mlperf-inference,yolo-coco2017-filtered,_mlc,_r2-downloader
+        names:
+          - yolo-v11-dataset-download
+        enable_if_env:
+          MLC_USE_DATASET_FROM_HOST: 
+          - true
+          - "yes"
       mounts:
         - "${{ MLC_ML_MODEL_YOLOV11_PATH }}:${{ MLC_ML_MODEL_YOLOV11_PATH }}"
         - "${{ MLC_DATASET_YOLO_COCO2017_FILTERED_PATH }}:${{ MLC_DATASET_YOLO_COCO2017_FILTERED_PATH }}"
@@ -1752,6 +1756,46 @@ variations:
     add_deps_recursive:
       mlperf-inference-implementation:
         tags: _yolo-99
+  
+  wan-2.2-t2v-a14b:
+    group:
+      model
+    env:
+      MLC_MODEL:
+        wan-2.2-t2v-a14b
+    add_deps_recursive:
+      mlperf-inference-implementation:
+        tags: _wan-2.2-t2v-a14b
+    docker:
+      deps:
+      - tags: get-ml-model-wan2,_mlc,_r2-downloader,_wan2_2_t2v_a14b
+        names:
+          - wan2-model-download
+        enable_if_env:
+          MLC_USE_ML_MODEL_FROM_HOST: 
+          - true
+          - "yes"
+      - tags: get-dataset-mlperf-inference-text-to-video,_mlc,_r2-downloader
+        names:
+          - wan2-dataset-download
+        enable_if_env:
+          MLC_USE_DATASET_FROM_HOST: 
+          - true
+          - "yes"
+      mounts:
+        - "${{ MLC_ML_MODEL_WAN2_PATH }}:${{ MLC_ML_MODEL_WAN2_PATH }}"
+        - "${{ MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH }}:${{ MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH }}"
+    # posthook_deps:
+    # - enable_if_env:
+    #     MLC_MLPERF_LOADGEN_MODE:
+    #     - accuracy
+    #     - all
+    #     MLC_MLPERF_ACCURACY_RESULTS_DIR:
+    #     - 'on'
+    #   names:
+    #   - mlperf-accuracy-script
+    #   - wan-accuracy-script
+    #   tags: run,accuracy,mlperf,_wan
 
   mobilenet:
     group:
@@ -1952,6 +1996,10 @@ variations:
       mlperf-inference-implementation:
         tags: _cpu
 
+  # cuda,reference,wan-2.2-t2v-a14b:
+  #   docker:
+  #     base_image: pytorch/pytorch:2.5.1-cuda12.1-cudnn9-devel
+
   cuda,reference:
     docker:
       base_image: nvcr.io/nvidia/pytorch:24.08-py3

script/get-dataset-mlperf-inference-text-to-video/customize.py

@@ -24,6 +24,8 @@ def postprocess(i):
     env = i['env']
 
     if env.get('MLC_DOWNLOAD_MODE', '') != 'dry':
+        env['MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH'] = os.path.join(
+            env['MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH'], 'vbench_prompts.txt')
         env['MLC_ML_DATASET_FILE_WITH_PATH'] = env['MLC_ML_DATASET_MLPERF_INFERENCE_TEXT_TO_VIDEO_DATASET_PATH']
 
     return {'return': 0}

script/get-ml-model-wan2/customize.py

@@ -23,6 +23,8 @@ def postprocess(i):
     env = i['env']
 
     if env.get('MLC_DOWNLOAD_MODE', '') != 'dry':
+        env['MLC_ML_MODEL_WAN2_PATH'] = os.path.join(
+            env['MLC_ML_MODEL_WAN2_PATH'], 'Wan2.2-T2V-A14B-Diffusers')
         env['MLC_ML_MODEL_FILE_WITH_PATH'] = env['MLC_ML_MODEL_WAN2_PATH']
 
     return {'return': 0}

script/run-docker-container/customize.py

@@ -176,7 +176,10 @@ def postprocess(i):
     if is_true(env.get('MLC_DOCKER_PRIVILEGED_MODE', '')):
         run_opts += " --privileged "
 
-    if env.get('MLC_DOCKER_ADD_NUM_GPUS', '') != '':
+    if env.get("MLC_DOCKER_GPU_DEVICES"):
+        for d in env["MLC_DOCKER_GPU_DEVICES"].split(","):
+            run_opts += f" --gpus device={d}"
+    elif env.get('MLC_DOCKER_ADD_NUM_GPUS', '') != '':
         run_opts += " --gpus={}".format(env['MLC_DOCKER_ADD_NUM_GPUS'])
     elif env.get('MLC_DOCKER_ADD_ALL_GPUS', '') != '':
         if env.get('MLC_CONTAINER_TOOL') == "podman":

script/run-docker-container/meta.yaml

@@ -21,6 +21,7 @@ default_env:
 input_mapping:
   all_gpus: MLC_DOCKER_ADD_ALL_GPUS
   num_gpus: MLC_DOCKER_ADD_NUM_GPUS
+  gpu_devices: MLC_DOCKER_GPU_DEVICE_IDS
   base: MLC_DOCKER_IMAGE_BASE
   cache: MLC_DOCKER_CACHE
   mlc_repo: MLC_MLOPS_REPO

script/run-mlperf-inference-app/meta.yaml

@@ -76,6 +76,7 @@ input_mapping:
   model: MLC_MLPERF_MODEL
   multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY
   network: MLC_NETWORK_LOADGEN
+  number_of_processes_per_gpu_node: MLC_MLPERF_INFERENCE_NUM_PROCESSES_PER_GPU_NODE
   nvidia_system_name: MLC_NVIDIA_SYSTEM_NAME
   offline_target_qps: MLC_MLPERF_LOADGEN_OFFLINE_TARGET_QPS
   output_dir: OUTPUT_BASE_DIR
@@ -110,6 +111,7 @@ input_mapping:
   target_latency: MLC_MLPERF_LOADGEN_TARGET_LATENCY
   target_qps: MLC_MLPERF_LOADGEN_TARGET_QPS
   test_query_count: MLC_TEST_QUERY_COUNT
+  test_qps: MLC_MLPERF_INFERENCE_TEST_QPS
   threads: MLC_NUM_THREADS
   sut: MLC_MLPERF_INFERENCE_SUT_VARIATION
   nvidia_llama2_dataset_file_path: MLC_NVIDIA_LLAMA_DATASET_FILE_PATH
@@ -515,6 +517,7 @@ input_description:
     - pointpainting
     - yolo-95
     - yolo-99
+    - wan-2.2-t2v-a14b
     desc: MLPerf model
     sort: 200
   precision: