[Wan22] update video idx (#2530)

* update video idx

* update

* checking

* update

* fix sections

* fix sections

* fix sections

* add

* [Automated Commit] Format Codebase

* run gh action

* [Automated Commit] Format Codebase

* per discussion on MLComm review

* fix the sample.txt

* fix checking path

* promt-0.mp4, ids.mp4

* clean

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com>

Author: 3 people

.github/workflows/auto-update-dev.yml

@@ -3,7 +3,7 @@ name: Auto-Update Dev Branches from Master
 on:
   push:
     branches:
-      - master  # Trigger workflow on commits to 'master' branch.
+      - master  # Trigger workflow on commits to 'master' branch
   workflow_dispatch: {}
 
 jobs:

text_to_video/wan-2.2-t2v-a14b/data/samples_filename_ids.txt

@@ -0,0 +1,10 @@
+A boat sailing leisurely along the Seine River with the Eiffel Tower in background, pixel art-0.mp4, 130.mp4
+A panda drinking coffee in a cafe in Paris, watercolor painting-0.mp4, 106.mp4
+The bund Shanghai, black and white-0.mp4, 84.mp4
+an elephant spraying itself with water using its trunk to cool down-0.mp4, 59.mp4
+a car turning a corner-0.mp4, 12.mp4
+a truck anchored in a tranquil bay-0.mp4, 31.mp4
+The bund Shanghai, in cyberpunk style-0.mp4, 86.mp4
+Gwen Stacy reading a book, in cyberpunk style-0.mp4, 122.mp4
+skyscraper-0.mp4, 223.mp4 
+a shark is swimming in the ocean, animated style-0.mp4, 96.mp4

tools/submission/submission_checker/checks/accuracy_check.py

@@ -2,6 +2,7 @@
 from ..constants import *
 from ..loader import SubmissionLogs
 from ..configuration.configuration import Config
+from ..utils import check_extra_files
 import re
 import os
 
@@ -25,6 +26,9 @@ class AccuracyCheck(BaseCheck):
     - `loadgen_errors_check`: Fails if Loadgen reported non-ignored errors.
     - `dataset_check`: Verifies the reported sample count matches the
         configured dataset size unless the check is skipped.
+    - `extra_files_check`: For benchmarks in REQUIRED_ACC_BENCHMARK (e.g.
+        stable-diffusion-xl, wan-2.2-t2v-a14b), verifies required extra
+        artifacts (e.g. images/, videos/) exist in the accuracy directory.
 
     Attributes:
             submission_logs (SubmissionLogs): Holder for submission log paths
@@ -78,6 +82,7 @@ def setup_checks(self):
         self.checks.append(self.accuracy_json_check)
         self.checks.append(self.loadgen_errors_check)
         self.checks.append(self.dataset_check)
+        self.checks.append(self.extra_files_check)
 
     def accuracy_result_check(self):
         """Validate reported accuracy metrics in `accuracy.txt`.
@@ -234,3 +239,34 @@ def dataset_check(self):
             )
             return False
         return True
+
+    def extra_files_check(self):
+        """Verify required extra accuracy files for certain benchmarks.
+
+        For models in REQUIRED_ACC_BENCHMARK (e.g. stable-diffusion-xl
+        images, wan-2.2-t2v-a14b videos), ensures the accuracy directory
+        contains the required subdirs and files. Skipped if
+        skip_extra_accuracy_files_check is set.
+
+        Returns:
+            bool: True if the check is skipped, the model has no extra
+                requirements, or all required files exist; False otherwise.
+        """
+        if self.config.skip_extra_accuracy_files_check:
+            return True
+        if self.model not in REQUIRED_ACC_BENCHMARK:
+            return True
+        if self.config.version not in REQUIRED_ACC_BENCHMARK[self.model]:
+            return True
+        acc_dir = os.path.dirname(self.path)
+        target_files = REQUIRED_ACC_BENCHMARK[self.model][self.config.version]
+        extra_files_pass, missing_files = check_extra_files(
+            acc_dir, target_files)
+        if not extra_files_pass:
+            self.log.error(
+                "%s expected to have the following extra files (%s)",
+                acc_dir,
+                missing_files,
+            )
+            return False
+        return True

tools/submission/submission_checker/constants.py

@@ -1123,6 +1123,22 @@
                 "2289",
             ]
         },
+    },
+    "wan-2.2-t2v-a14b": {
+        "v6.0": {
+            "videos": [
+                "130",
+                "106",
+                "84",
+                "59",
+                "12",
+                "31", 
+                "86",
+                "122", 
+                "233", 
+                "96",
+            ]            
+        },
     }
 }
 REQUIRED_MEASURE_FILES = ["user.conf", "README.md"]
@@ -1701,6 +1717,7 @@
     "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST08/verify_accuracy.txt",
     "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST08/verify_accuracy.txt",
 }
+
 TEST07_ACC_PATH = {
     "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST07/verify_accuracy.txt",
     "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST07/verify_accuracy.txt",

tools/submission/submission_checker/utils.py

@@ -57,8 +57,13 @@ def check_extra_files(path, target_files):
             for target_file in target_files[dir]:
                 if target_file not in files:
                     check_pass = False
-                    missing_files.append(
-                        f"{os.path.join(path, dir, target_file)}.png")
+                    if "images" in dir:
+                        missing_files.append(
+                            f"{os.path.join(path, dir, target_file)}.png")
+                    if "videos" in dir:
+                        missing_files.append(
+                            f"{os.path.join(path, dir, target_file)}.mp4")
+
             if "captions" not in files:
                 missing_files.append(
                     f"{os.path.join(path, dir, 'captions.txt')}")