1212import sys
1313from pathlib import Path
1414
15+
1516def setup_logging ():
1617 """Setup logging configuration."""
1718 logging .basicConfig (
@@ -24,36 +25,37 @@ def setup_logging():
2425def parse_results (output_path ):
2526 """
2627 Parse VBench evaluation results and print summary.
27-
28+
2829 Args:
2930 output_path: Path to evaluation results directory
3031 """
3132 output_path = Path (output_path )
32-
33+
3334 # Find the most recent eval_results file (contains scores)
3435 result_files = sorted (output_path .glob ("results_*_eval_results.json" ))
3536 if not result_files :
3637 logging .warning (f"No results found in { output_path } " )
3738 return
38-
39+
3940 result_file = result_files [- 1 ]
40-
41+
4142 try :
4243 with open (result_file , 'r' ) as f :
4344 results = json .load (f )
44-
45+
4546 # Print summary in MLPerf-style format
46- print ("\n " + "=" * 60 )
47+ print ("\n " + "=" * 60 )
4748 print ("VBench Evaluation Results" )
48- print ("=" * 60 )
49-
50- # Extract dimension scores (VBench format: {dimension_name: [avg_score, [video_results]], ...})
49+ print ("=" * 60 )
50+
51+ # Extract dimension scores (VBench format: {dimension_name: [avg_score,
52+ # [video_results]], ...})
5153 if results :
5254 print ("\n Dimension Scores:" )
5355 print ("-" * 60 )
5456 total_score = 0
5557 num_dimensions = 0
56-
58+
5759 for dimension , value in sorted (results .items ()):
5860 # VBench stores [avg_score, list_of_video_results]
5961 if isinstance (value , list ) and len (value ) > 0 :
@@ -62,24 +64,25 @@ def parse_results(output_path):
6264 total_score += score
6365 num_dimensions += 1
6466 print (f" { dimension :30s} : { score :6.4f} " )
65-
67+
6668 if num_dimensions > 0 :
6769 overall_avg = total_score / num_dimensions
6870 print ("-" * 60 )
6971 print (f" { 'Overall Average' :30s} : { overall_avg :6.4f} " )
70-
71- print ("=" * 60 )
72+
73+ print ("=" * 60 )
7274 print (f"Detailed results: { result_file } " )
73- print ("=" * 60 + "\n " )
74-
75+ print ("=" * 60 + "\n " )
76+
7577 except Exception as e :
7678 logging .error (f"Failed to parse results: { e } " )
7779 import traceback
7880 traceback .print_exc ()
7981
8082
8183def main ():
82- parser = argparse .ArgumentParser (description = "VBench evaluation for Wan2.2 T2V videos" )
84+ parser = argparse .ArgumentParser (
85+ description = "VBench evaluation for Wan2.2 T2V videos" )
8386 parser .add_argument (
8487 "--videos-path" ,
8588 type = str ,
@@ -111,20 +114,20 @@ def main():
111114 default = 8 ,
112115 help = "Number of GPUs to use for evaluation (default: 8)"
113116 )
114-
117+
115118 args = parser .parse_args ()
116-
119+
117120 setup_logging ()
118-
121+
119122 # Validate inputs
120123 videos_path = Path (args .videos_path )
121124 if not videos_path .exists ():
122125 logging .error (f"Videos path does not exist: { videos_path } " )
123126 return 1
124-
127+
125128 output_path = Path (args .output_path )
126129 output_path .mkdir (parents = True , exist_ok = True )
127-
130+
128131 logging .info ("=" * 60 )
129132 logging .info ("VBench Evaluation" )
130133 logging .info ("=" * 60 )
@@ -133,8 +136,9 @@ def main():
133136 logging .info (f"GPUs: { args .num_gpus } " )
134137 logging .info (f"Dimensions: { ', ' .join (args .dimensions )} " )
135138 logging .info ("=" * 60 )
136-
137- vbench_script = Path (__file__ ).parent / "submodules" / "VBench" / "evaluate.py"
139+
140+ vbench_script = Path (__file__ ).parent / "submodules" / \
141+ "VBench" / "evaluate.py"
138142 cmd = [
139143 "python" , "-m" , "torch.distributed.run" ,
140144 f"--nproc_per_node={ args .num_gpus } " ,
@@ -144,21 +148,21 @@ def main():
144148 "--load_ckpt_from_local=True" ,
145149 "--dimension"
146150 ] + args .dimensions
147-
151+
148152 logging .info ("\n Executing VBench evaluation..." )
149153 logging .info (f"Command: { ' ' .join (cmd )} " )
150154 logging .info ("" )
151-
155+
152156 # Run evaluation
153157 try :
154158 result = subprocess .run (cmd , check = True )
155-
159+
156160 # Parse and print results
157161 logging .info ("\n Parsing evaluation results..." )
158162 parse_results (output_path )
159-
163+
160164 return 0
161-
165+
162166 except subprocess .CalledProcessError as e :
163167 logging .error (f"Evaluation failed with exit code { e .returncode } " )
164168 return e .returncode
0 commit comments