revert untouched files

Author: neginraoof

eval/chat_benchmarks/MTBench/fastchat/data/clean_sharegpt.py

@@ -5,7 +5,6 @@
 Usage:
 python3 -m fastchat.data.clean_sharegpt --in sharegpt_html.json --out sharegpt_clean.json
 """
-
 import argparse
 from concurrent.futures import ProcessPoolExecutor
 import json
@@ -20,7 +19,9 @@
 
 div_pattern = re.compile("<div.*?>")
 span_pattern = re.compile("<span.*?>")
-code_lang_pattern = re.compile("```\s*" + "(.*?)" + "(?:Copy code)+" + "(.+?)" + "\s*?```", re.DOTALL)
+code_lang_pattern = re.compile(
+    "```\s*" + "(.*?)" + "(?:Copy code)+" + "(.+?)" + "\s*?```", re.DOTALL
+)
 code_lang_format = "```\g<1>\n\g<2>\n```"
 regenerate_pattern = re.compile("\d+ / \d+")
 copy_chars_pattern = re.compile("Copy\d+ chars / \d+ words")
@@ -154,7 +155,9 @@ def clean_html_all(content, begin, end):
     content = content[begin:end]
     processed = []
     with ProcessPoolExecutor() as executor:
-        for result in tqdm(executor.map(clean_html_one_sample, content), total=len(content)):
+        for result in tqdm(
+            executor.map(clean_html_one_sample, content), total=len(content)
+        ):
             processed.append(result)
 
     visited = {}

eval/chat_benchmarks/MTBench/fastchat/data/extract_gpt4_only.py

@@ -3,7 +3,6 @@
 
 Usage: python3 -m fastchat.data.extract_gpt4_only --in sharegpt.json
 """
-
 import argparse
 import json
 

eval/chat_benchmarks/MTBench/fastchat/data/extract_single_round.py

@@ -3,7 +3,6 @@
 
 Usage: python3 -m fastchat.data.extract_single_round --in sharegpt.json
 """
-
 import argparse
 import json
 

eval/chat_benchmarks/MTBench/fastchat/data/filter_wrong_format.py

@@ -5,7 +5,6 @@
 python3 -m fastchat.data.filter_wrong_format --in input.json --out output.json
 
 """
-
 import argparse
 import json
 import re

eval/chat_benchmarks/MTBench/fastchat/data/get_stats.py

@@ -26,7 +26,9 @@ def tokenize_one_sample(c):
 def tokenize_dataset(content):
     processed = []
     with ProcessPoolExecutor() as executor:
-        for result in tqdm(executor.map(tokenize_one_sample, content), total=len(content)):
+        for result in tqdm(
+            executor.map(tokenize_one_sample, content), total=len(content)
+        ):
             processed.append(result)
 
     return processed
@@ -57,7 +59,9 @@ def compute_stats(content):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--in-file", type=str)
-    parser.add_argument("--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf")
+    parser.add_argument(
+        "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf"
+    )
     args = parser.parse_args()
 
     content = json.load(open(args.in_file, "r"))

eval/chat_benchmarks/MTBench/fastchat/data/hardcoded_questions.py

@@ -1,7 +1,6 @@
 """
 Hardcoded question and answers.
 """
-
 import json
 
 

eval/chat_benchmarks/MTBench/fastchat/data/inspect_data.py

@@ -2,7 +2,6 @@
 Usage:
 python3 -m fastchat.data.inspect_data --in sharegpt_20230322_clean_lang_split.json
 """
-
 import argparse
 import json
 import random

eval/chat_benchmarks/MTBench/fastchat/data/optional_clean.py

@@ -8,7 +8,6 @@
 Requirement:
 pip3 install polyglot pyicu pycld2
 """
-
 import argparse
 import json
 import re

eval/chat_benchmarks/MTBench/fastchat/data/optional_replace.py

@@ -7,7 +7,6 @@
 Requirement:
 pip3 install transformers tqdm
 """
-
 import argparse
 import json
 import traceback
@@ -16,7 +15,9 @@
 from tqdm import tqdm
 
 
-def replace_special_tokens(tokenizer: transformers.PreTrainedTokenizer, text: str) -> str:
+def replace_special_tokens(
+    tokenizer: transformers.PreTrainedTokenizer, text: str
+) -> str:
     if not text:
         return text
 

eval/chat_benchmarks/MTBench/fastchat/data/prepare_all.py

@@ -9,14 +9,20 @@
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--prefix", type=str, default="~/datasets/sharegpt_20230521")
-    parser.add_argument("--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf")
+    parser.add_argument(
+        "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf"
+    )
     parser.add_argument("--seq-len", type=int, default=4096)
     args = parser.parse_args()
 
     in_prefix = args.prefix
     model_path = args.model_name_or_path
     seq_len = args.seq_len
-    prefix = f"{in_prefix}_{seq_len}".replace("4096", "4k").replace("8192", "8k").replace("16384", "16k")
+    prefix = (
+        f"{in_prefix}_{seq_len}".replace("4096", "4k")
+        .replace("8192", "8k")
+        .replace("16384", "16k")
+    )
 
     cmd_list = [
         f"python3 -m fastchat.data.clean_sharegpt --in {in_prefix}_html.json --out {prefix}_clean.json",