fix lint

rootfs · rootfs · commit 844083149423 · 2025-09-28T16:24:14.000Z
Signed-off-by: Huamin Chen &lt;hchen@redhat.com&gt;
diff --git a/bench/plot_comprehensive_results.py b/bench/plot_comprehensive_results.py
diff --git a/bench/vllm_semantic_router_bench/cli.py b/bench/vllm_semantic_router_bench/cli.py
@@ -40,7 +40,15 @@ def main():
     test_parser.add_argument(
         "--dataset",
         required=True,
-        choices=["mmlu", "arc", "arc-challenge", "gpqa", "truthfulqa", "commonsenseqa", "hellaswag"],
+        choices=[
+            "mmlu",
+            "arc",
+            "arc-challenge",
+            "gpqa",
+            "truthfulqa",
+            "commonsenseqa",
+            "hellaswag",
+        ],
         help="Dataset to test",
     )
     test_parser.add_argument(
@@ -68,7 +76,15 @@ def main():
     compare_parser.add_argument(
         "--dataset",
         required=True,
-        choices=["mmlu", "arc", "arc-challenge", "gpqa", "truthfulqa", "commonsenseqa", "hellaswag"],
+        choices=[
+            "mmlu",
+            "arc",
+            "arc-challenge",
+            "gpqa",
+            "truthfulqa",
+            "commonsenseqa",
+            "hellaswag",
+        ],
         help="Dataset to benchmark",
     )
     compare_parser.add_argument(
@@ -119,7 +135,14 @@ def main():
     comprehensive_parser.add_argument(
         "--datasets",
         nargs="+",
-        default=["mmlu", "arc-challenge", "gpqa", "truthfulqa", "commonsenseqa", "hellaswag"],
+        default=[
+            "mmlu",
+            "arc-challenge",
+            "gpqa",
+            "truthfulqa",
+            "commonsenseqa",
+            "hellaswag",
+        ],
         help="Datasets to benchmark",
     )
     comprehensive_parser.add_argument(
diff --git a/bench/vllm_semantic_router_bench/dataset_factory.py b/bench/vllm_semantic_router_bench/dataset_factory.py
@@ -23,6 +23,7 @@
 )
 from .dataset_implementations.gsm8k_dataset import GSM8KDataset
 from .dataset_implementations.hellaswag_dataset import HellaSwagDataset
+
 # from .dataset_implementations.math_dataset import MATHDataset  # Disabled - dataset not available
 from .dataset_implementations.mmlu_dataset import MMLUDataset
 from .dataset_implementations.openbookqa_dataset import OpenBookQADataset
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/aqua_rat_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/aqua_rat_dataset.py
@@ -84,7 +84,7 @@ def load_dataset(
         if samples_per_category:
             np.random.seed(seed)
             random.seed(seed)
-            
+
             sample_size = min(samples_per_category, len(df))
             df = df.sample(n=sample_size, random_state=seed)
 
@@ -101,7 +101,8 @@ def load_dataset(
             for option in raw_options:
                 # Remove letter prefix like "A)", "B)", etc.
                 import re
-                cleaned = re.sub(r'^[A-E]\)', '', option).strip()
+
+                cleaned = re.sub(r"^[A-E]\)", "", option).strip()
                 options.append(cleaned)
 
             question = Question(
@@ -115,7 +116,7 @@ def load_dataset(
                     "difficulty": "Moderate",
                     "type": "algebraic_word_problem",
                     "rationale": rationale,
-                }
+                },
             )
             questions.append(question)
 
@@ -132,8 +133,10 @@ def load_dataset(
 
     def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
         """Format prompt for AQUA-RAT questions."""
-        options_text = "\n".join([f"{chr(65+i)}) {opt}" for i, opt in enumerate(question.options)])
-        
+        options_text = "\n".join(
+            [f"{chr(65+i)}) {opt}" for i, opt in enumerate(question.options)]
+        )
+
         if prompt_style == "plain":
             return f"""Solve this algebraic word problem:
 
@@ -145,7 +148,7 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 ANSWER: [letter]
 
 For example: ANSWER: A"""
-        
+
         elif prompt_style == "explicit_cot":
             return f"""Solve this algebraic word problem step by step:
 
@@ -165,6 +168,6 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 ANSWER: [letter]
 
 For example: ANSWER: A"""
-        
+
         else:
             raise ValueError(f"Unknown prompt style: {prompt_style}")
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/drop_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/drop_dataset.py
@@ -84,7 +84,7 @@ def load_dataset(
         if samples_per_category:
             np.random.seed(seed)
             random.seed(seed)
-            
+
             sample_size = min(samples_per_category, len(df))
             df = df.sample(n=sample_size, random_state=seed)
 
@@ -115,7 +115,7 @@ def load_dataset(
                     "type": "discrete_reasoning",
                     "passage": passage,
                     "question_only": question_text,
-                }
+                },
             )
             questions.append(question)
 
@@ -141,7 +141,7 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 ANSWER: [your answer]
 
 For example: ANSWER: 68.5 or ANSWER: germans or ANSWER: Centenary Medal"""
-        
+
         elif prompt_style == "explicit_cot":
             return f"""{question.question}
 
@@ -156,6 +156,6 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 ANSWER: [your answer]
 
 For example: ANSWER: 68.5 or ANSWER: germans or ANSWER: Centenary Medal"""
-        
+
         else:
             raise ValueError(f"Unknown prompt style: {prompt_style}")
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/gsm8k_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/gsm8k_dataset.py
@@ -84,7 +84,7 @@ def load_dataset(
         if samples_per_category:
             np.random.seed(seed)
             random.seed(seed)
-            
+
             sample_size = min(samples_per_category, len(df))
             df = df.sample(n=sample_size, random_state=seed)
 
@@ -93,11 +93,12 @@ def load_dataset(
         for _, row in df.iterrows():
             question_text = row["question"]
             answer_text = row["answer"]
-            
+
             # Extract the final numerical answer from the solution
             import re
+
             # GSM8K answers end with "#### [number]"
-            answer_match = re.search(r'####\s*([0-9,.-]+)', answer_text)
+            answer_match = re.search(r"####\s*([0-9,.-]+)", answer_text)
             correct_answer = answer_match.group(1) if answer_match else "Unknown"
 
             question = Question(
@@ -111,7 +112,7 @@ def load_dataset(
                     "difficulty": "Elementary",
                     "type": "word_problem",
                     "solution": answer_text,
-                }
+                },
             )
             questions.append(question)
 
@@ -137,7 +138,7 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 ANSWER: [number]
 
 For example: ANSWER: 42"""
-        
+
         elif prompt_style == "explicit_cot":
             return f"""Solve this math word problem step by step, showing all your work:
 
@@ -154,6 +155,6 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 ANSWER: [number]
 
 For example: ANSWER: 42"""
-        
+
         else:
             raise ValueError(f"Unknown prompt style: {prompt_style}")
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/math_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/math_dataset.py
@@ -115,10 +115,11 @@ def load_dataset(
             # The solution contains the final answer in \boxed{} format
             question_text = row["problem"]
             solution = row["solution"]
-            
+
             # Extract boxed answer as the correct answer
             import re
-            boxed_match = re.search(r'\\boxed\{([^}]+)\}', solution)
+
+            boxed_match = re.search(r"\\boxed\{([^}]+)\}", solution)
             correct_answer = boxed_match.group(1) if boxed_match else "Unknown"
 
             question = Question(
@@ -132,7 +133,7 @@ def load_dataset(
                     "level": row.get("level", "Unknown"),
                     "subject": row["type"],
                     "solution": solution,
-                }
+                },
             )
             questions.append(question)
 
@@ -151,7 +152,7 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
         """Format prompt for MATH questions."""
         if prompt_style == "plain":
             return f"Solve this mathematics problem step by step:\n\n{question.question}\n\nProvide your final answer in the format: Answer: [your answer]"
-        
+
         elif prompt_style == "explicit_cot":
             return f"""Solve this mathematics problem step by step, showing all your work:
 
@@ -165,6 +166,6 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 5. Verify your answer
 
 Provide your final answer in the format: Answer: [your answer]"""
-        
+
         else:
             raise ValueError(f"Unknown prompt style: {prompt_style}")
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/openbookqa_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/openbookqa_dataset.py
@@ -84,7 +84,7 @@ def load_dataset(
         if samples_per_category:
             np.random.seed(seed)
             random.seed(seed)
-            
+
             sample_size = min(samples_per_category, len(df))
             df = df.sample(n=sample_size, random_state=seed)
 
@@ -94,13 +94,16 @@ def load_dataset(
             question_stem = row["question_stem"]
             choices = row["choices"]
             answer_key = row["answerKey"]  # A, B, C, D
-            
+
             # Extract options from choices
             # Handle different possible structures for choices
             if isinstance(choices, dict) and "text" in choices:
                 options = choices["text"]
             elif isinstance(choices, list):
-                options = [choice["text"] if isinstance(choice, dict) else choice for choice in choices]
+                options = [
+                    choice["text"] if isinstance(choice, dict) else choice
+                    for choice in choices
+                ]
             else:
                 options = [str(choices)]  # Fallback
 
@@ -115,7 +118,7 @@ def load_dataset(
                     "difficulty": "Elementary",
                     "type": "science_reasoning",
                     "requires_fact_combination": True,
-                }
+                },
             )
             questions.append(question)
 
@@ -132,8 +135,10 @@ def load_dataset(
 
     def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
         """Format prompt for OpenBookQA questions."""
-        options_text = "\n".join([f"{chr(65+i)}) {opt}" for i, opt in enumerate(question.options)])
-        
+        options_text = "\n".join(
+            [f"{chr(65+i)}) {opt}" for i, opt in enumerate(question.options)]
+        )
+
         if prompt_style == "plain":
             return f"""Question: {question.question}
 
@@ -142,7 +147,7 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 Think about what scientific facts and principles apply to this question.
 
 Provide your answer in the format 'Answer: [letter]'."""
-        
+
         elif prompt_style == "explicit_cot":
             return f"""Question: {question.question}
 
@@ -157,6 +162,6 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 5. Select the best answer based on scientific principles
 
 Show your scientific reasoning step by step, then provide your answer in the format 'Answer: [letter]'."""
-        
+
         else:
             raise ValueError(f"Unknown prompt style: {prompt_style}")
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/sciq_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/sciq_dataset.py
@@ -84,7 +84,7 @@ def load_dataset(
         if samples_per_category:
             np.random.seed(seed)
             random.seed(seed)
-            
+
             sample_size = min(samples_per_category, len(df))
             df = df.sample(n=sample_size, random_state=seed)
 
@@ -93,13 +93,13 @@ def load_dataset(
         for _, row in df.iterrows():
             question_text = row["question"]
             correct_answer = row["correct_answer"]
-            
+
             # Build options list
             options = [
                 row["correct_answer"],
-                row["distractor1"], 
+                row["distractor1"],
                 row["distractor2"],
-                row["distractor3"]
+                row["distractor3"],
             ]
             # Shuffle options and find correct index
             random.seed(42)  # Fixed seed for reproducible option order
@@ -118,8 +118,10 @@ def load_dataset(
                 metadata={
                     "difficulty": "Moderate",
                     "type": "science_multiple_choice",
-                    "support": row.get("support", ""),  # Background passage if available
-                }
+                    "support": row.get(
+                        "support", ""
+                    ),  # Background passage if available
+                },
             )
             questions.append(question)
 
@@ -136,20 +138,22 @@ def load_dataset(
 
     def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
         """Format prompt for SciQ questions."""
-        options_text = "\n".join([f"{chr(65+i)}) {opt}" for i, opt in enumerate(question.options)])
-        
+        options_text = "\n".join(
+            [f"{chr(65+i)}) {opt}" for i, opt in enumerate(question.options)]
+        )
+
         # Add support passage if available
         support_text = ""
         if question.metadata and question.metadata.get("support"):
             support_text = f"Background: {question.metadata['support']}\n\n"
-        
+
         if prompt_style == "plain":
             return f"""{support_text}Question: {question.question}
 
 {options_text}
 
 Provide your answer in the format 'Answer: [letter]'."""
-        
+
         elif prompt_style == "explicit_cot":
             return f"""{support_text}Question: {question.question}
 
@@ -164,6 +168,6 @@ def format_prompt(self, question: Question, prompt_style: str = "plain") -> str:
 5. Select the best answer
 
 Show your scientific reasoning step by step, then provide your answer in the format 'Answer: [letter]'."""
-        
+
         else:
             raise ValueError(f"Unknown prompt style: {prompt_style}")
diff --git a/bench/vllm_semantic_router_bench/dataset_implementations/strategyqa_dataset.py b/bench/vllm_semantic_router_bench/dataset_implementations/strategyqa_dataset.py
diff --git a/bench/vllm_semantic_router_bench/router_reason_bench_multi_dataset.py b/bench/vllm_semantic_router_bench/router_reason_bench_multi_dataset.py

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@`
`23`	`23`	`)`
`24`	`24`	`from .dataset_implementations.gsm8k_dataset import GSM8KDataset`
`25`	`25`	`from .dataset_implementations.hellaswag_dataset import HellaSwagDataset`
	`26`	`+`
`26`	`27`	`# from .dataset_implementations.math_dataset import MATHDataset # Disabled - dataset not available`
`27`	`28`	`from .dataset_implementations.mmlu_dataset import MMLUDataset`
`28`	`29`	`from .dataset_implementations.openbookqa_dataset import OpenBookQADataset`