format

Author: infwinston

fastchat/serve/monitor/basic_stats.py

@@ -142,7 +142,7 @@ def report_basic_stats(log_files):
         height=300,
         width=1200,
     )
-    
+
     # calculate conversion rate for each day (vote / chat)
     conversion_rate = {}
     for date in chat_dates_counts.index:
@@ -163,7 +163,6 @@ def report_basic_stats(log_files):
         width=1200,
     )
 
-
     # Model call counts
     model_hist_all = df_all[df_all["type"] == "chat"]["model"].value_counts()
     model_hist_1_day = df_1_day[df_1_day["type"] == "chat"]["model"].value_counts()

fastchat/serve/monitor/clean_battle_data.py

@@ -146,7 +146,7 @@ def replace_model_name(old_name, tstamp):
         "dumbledore-v3": "llama-3.2-vision-90b-instruct",
         "potter-v1": "llama-3.2-vision-11b-instruct",
         "sharp-game-player-v1": "llama-3.2-3b-instruct",
-        "zeus-flare-thunder-v1": "llama-3.2-1b-instruct",        
+        "zeus-flare-thunder-v1": "llama-3.2-1b-instruct",
         "qwen-vl-max-0809": "qwen2-vl-72b",
         "gemini-1.5-pro-002-test-sp": "gemini-1.5-pro-002",
         "gemini-1.5-flash-test-5": "gemini-1.5-flash-002",
@@ -185,7 +185,7 @@ def replace_model_name(old_name, tstamp):
             return old_name
         else:
             return old_name + "-old"
-    if old_name  == "eureka-chatbot":
+    if old_name == "eureka-chatbot":
         if tstamp > 1721651521:
             return "eureka-chatbot-v2"
         else:
@@ -293,12 +293,17 @@ def process_data_txt2img(
             if row["models"][0] is None or row["models"][1] is None:
                 count_dict["invalid"] += 1
                 continue
-            if not isinstance(row["models"][0], str) or not isinstance(row["models"][1], str):
+            if not isinstance(row["models"][0], str) or not isinstance(
+                row["models"][1], str
+            ):
                 count_dict["invalid"] += 1
                 continue
 
             # Resolve model names
-            models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+            models_public = [
+                remove_html(row["models"][0]),
+                remove_html(row["models"][1]),
+            ]
             if "model_name" in row["states"][0]:
                 models_hidden = [
                     row["states"][0]["model_name"],
@@ -356,12 +361,12 @@ def process_data_txt2img(
                     "anony_bothbad_vote": 0,
                     "anony_leftvote": 0,
                     "anony_rightvote": 0,
-                    "sanitized_id": shortuuid.uuid()
+                    "sanitized_id": shortuuid.uuid(),
                 }
             all_ips[ip]["count"] += 1
             if flag_anony:
                 all_ips[ip]["anony_count"] += 1
-                all_ips[ip]["anony_"+row["type"]] += 1
+                all_ips[ip]["anony_" + row["type"]] += 1
 
             if sanitize_ip:
                 user_id = f"{all_ips[ip]['sanitized_id']}"
@@ -389,6 +394,7 @@ def process_data_txt2img(
         )
     return battles, count_dict, count_leak, all_ips
 
+
 def process_data(
     data,
     exclude_model_names,
@@ -433,12 +439,17 @@ def process_data(
             if row["models"][0] is None or row["models"][1] is None:
                 count_dict["invalid"] += 1
                 continue
-            if not isinstance(row["models"][0], str) or not isinstance(row["models"][1], str):
+            if not isinstance(row["models"][0], str) or not isinstance(
+                row["models"][1], str
+            ):
                 count_dict["invalid"] += 1
                 continue
 
             # Resolve model names
-            models_public = [remove_html(row["models"][0]), remove_html(row["models"][1])]
+            models_public = [
+                remove_html(row["models"][0]),
+                remove_html(row["models"][1]),
+            ]
             if "model_name" in row["states"][0]:
                 models_hidden = [
                     row["states"][0]["model_name"],
@@ -484,7 +495,6 @@ def process_data(
                 print(state["messages"][state["offset"]][1])
                 raise ValueError
 
-
             # Drop conversations if the model names are leaked
             messages = ""
             for i in range(2):
@@ -576,12 +586,12 @@ def process_data(
                     "anony_bothbad_vote": 0,
                     "anony_leftvote": 0,
                     "anony_rightvote": 0,
-                    "sanitized_id": shortuuid.uuid()
+                    "sanitized_id": shortuuid.uuid(),
                 }
             all_ips[ip]["count"] += 1
             if flag_anony:
                 all_ips[ip]["anony_count"] += 1
-                all_ips[ip]["anony_"+row["type"]] += 1
+                all_ips[ip]["anony_" + row["type"]] += 1
 
             if sanitize_ip:
                 user_id = f"{all_ips[ip]['sanitized_id']}"
@@ -607,13 +617,25 @@ def process_data(
                 )
 
             user_tokens = sum(
-                [conv["num_tokens"] for conv in conversation_a if conv["role"] == "user"]
+                [
+                    conv["num_tokens"]
+                    for conv in conversation_a
+                    if conv["role"] == "user"
+                ]
             )
             assistant_a_tokens = sum(
-                [conv["num_tokens"] for conv in conversation_a if conv["role"] == "assistant"]
+                [
+                    conv["num_tokens"]
+                    for conv in conversation_a
+                    if conv["role"] == "assistant"
+                ]
             )
             assistant_b_tokens = sum(
-                [conv["num_tokens"] for conv in conversation_b if conv["role"] == "assistant"]
+                [
+                    conv["num_tokens"]
+                    for conv in conversation_b
+                    if conv["role"] == "assistant"
+                ]
             )
             context_tokens_a = sum([conv["num_tokens"] for conv in conversation_a[:-1]])
             context_tokens_b = sum([conv["num_tokens"] for conv in conversation_b[:-1]])
@@ -702,30 +724,36 @@ def clean_battle_data(
                     all_ips[ip]["count"] += sub_all_ips[ip]["count"]
                     all_ips[ip]["anony_count"] += sub_all_ips[ip]["anony_count"]
                     all_ips[ip]["anony_tievote"] += sub_all_ips[ip]["anony_tievote"]
-                    all_ips[ip]["anony_bothbad_vote"] += sub_all_ips[ip]["anony_bothbad_vote"]
+                    all_ips[ip]["anony_bothbad_vote"] += sub_all_ips[ip][
+                        "anony_bothbad_vote"
+                    ]
                     all_ips[ip]["anony_leftvote"] += sub_all_ips[ip]["anony_leftvote"]
                     all_ips[ip]["anony_rightvote"] += sub_all_ips[ip]["anony_rightvote"]
 
     battles.sort(key=lambda x: x["tstamp"])
     last_updated_tstamp = battles[-1]["tstamp"]
-    
+
     battles = pd.DataFrame(battles)
-    
+
     # drop rows with same question_id
     print(f"before drop dups #battles: {len(battles)}")
     battles = battles.drop_duplicates(subset=["question_id"], keep="first")
     battles = battles.reset_index(drop=True)
     print(f"#battles: {len(battles)}")
 
-    battles = battles[battles["anony"]].reset_index(drop=True) if anony_only else battles
+    battles = (
+        battles[battles["anony"]].reset_index(drop=True) if anony_only else battles
+    )
     if run_dedup and not (vision or txt2img):
         print("Running deduplication...")
         battles = utils.dedup_process(battles)
         num_dedup_battles = sum(battles["dedup_tag"].apply(lambda x: x["sampled"]))
         print(f"#dedup_battles: {num_dedup_battles}")
     else:
         print("Skip deduplication...")
-        dedup_tags = np.array([{"high_freq": False, "sampled": True} for _ in range(len(battles))])
+        dedup_tags = np.array(
+            [{"high_freq": False, "sampled": True} for _ in range(len(battles))]
+        )
         battles["dedup_tag"] = dedup_tags
 
     last_updated_datetime = datetime.datetime.fromtimestamp(
@@ -746,7 +774,9 @@ def clean_battle_data(
     for votetype in ["tievote", "bothbad_vote", "leftvote", "rightvote"]:
         vote_key = "anony_" + votetype
         userid_key = "sanitized_id" if sanitize_ip else "ip"
-        top_30_users = sorted(all_ips.values(), key=lambda x: x[vote_key], reverse=True)[:30]
+        top_30_users = sorted(
+            all_ips.values(), key=lambda x: x[vote_key], reverse=True
+        )[:30]
         top_30_ip_id = ["arena_user_" + ip[userid_key] for ip in top_30_users]
         battles_top_30_ips = battles[battles["judge"].isin(top_30_ip_id)]
         print(f"Top 30 IPs #battles: {len(battles_top_30_ips)}")
@@ -755,13 +785,15 @@ def clean_battle_data(
         for user in top_30_users:
             user_ip = user["ip"]
             user_id = "arena_user_" + user[userid_key]
-            
+
             ip_battles = battles_top_30_ips[battles_top_30_ips["judge"] == user_id]
             win_count = len(ip_battles[ip_battles["winner"] == "model_a"])
             tie_count = len(ip_battles[ip_battles["winner"] == "tie"])
             loss_count = len(ip_battles[ip_battles["winner"] == "model_b"])
-            print(f"{user_id}: model_a {win_count}, tie {tie_count}, mobel_b {loss_count}, {user_ip}")
-    
+            print(
+                f"{user_id}: model_a {win_count}, tie {tie_count}, mobel_b {loss_count}, {user_ip}"
+            )
+
     return battles
 
 
@@ -784,8 +816,14 @@ def clean_battle_data(
     ban_ip_list = json.load(open(args.ban_ip_file)) if args.ban_ip_file else None
 
     battles = clean_battle_data(
-        log_files, args.exclude_model_names or [], ban_ip_list, args.sanitize_ip, anony_only=args.anony_only,
-        run_dedup=args.run_dedup, vision=args.vision, txt2img=args.txt2img
+        log_files,
+        args.exclude_model_names or [],
+        ban_ip_list,
+        args.sanitize_ip,
+        anony_only=args.anony_only,
+        run_dedup=args.run_dedup,
+        vision=args.vision,
+        txt2img=args.txt2img,
     )
     last_updated_tstamp = battles.iloc[-1]["tstamp"]
     cutoff_date = datetime.datetime.fromtimestamp(
@@ -801,7 +839,9 @@ def clean_battle_data(
         print(f"Write cleaned data to {output}")
 
     if not args.txt2img:
-        battles = battles.drop(columns=["conversation_a", "conversation_b", "question_id"])
+        battles = battles.drop(
+            columns=["conversation_a", "conversation_b", "question_id"]
+        )
     print("Samples:")
     print(battles[:5])
 

fastchat/serve/monitor/clean_chat_data.py

@@ -41,7 +41,9 @@ def get_log_files(max_num_files=None, is_vision=False):
             prefix = ""
             if is_vision:
                 prefix = "vision-tmp-"
-            name = os.path.expanduser(f"~/fastchat_logs/server{i}/{prefix}{d}-conv.json")
+            name = os.path.expanduser(
+                f"~/fastchat_logs/server{i}/{prefix}{d}-conv.json"
+            )
             if os.path.exists(name):
                 filenames.append(name)
     max_num_files = max_num_files or len(filenames)
@@ -120,9 +122,7 @@ def clean_chat_data(log_files, action_type, remove_prompt=False, exclude_models=
             msg = x["content"]
             if isinstance(x["content"], list):
                 msg = x["content"][0]
-            x["num_tokens"] = len(
-                encoding.encode(msg, allowed_special="all")
-            )
+            x["num_tokens"] = len(encoding.encode(msg, allowed_special="all"))
             messages_concat += msg.lower()
 
             if remove_prompt:
@@ -185,21 +185,27 @@ def clean_chat_data(log_files, action_type, remove_prompt=False, exclude_models=
     parser.add_argument("--action-type", type=str, default="chat")
     parser.add_argument("--max-num-files", type=int)
     parser.add_argument("--vision", action="store_true")
-    parser.add_argument("--start-time", type=str) # example: 2024-08-01
-    parser.add_argument("--end-time", type=str) # example: 2024-08-01
+    parser.add_argument("--start-time", type=str)  # example: 2024-08-01
+    parser.add_argument("--end-time", type=str)  # example: 2024-08-01
     parser.add_argument("--remove-prompt", action="store_true")
     parser.add_argument("--exclude-models", type=str, nargs="+", default=[])
     args = parser.parse_args()
 
     log_files = get_log_files(args.max_num_files, args.vision)
     # print(log_files)
-    chats = clean_chat_data(log_files, args.action_type, args.remove_prompt, args.exclude_models)
+    chats = clean_chat_data(
+        log_files, args.action_type, args.remove_prompt, args.exclude_models
+    )
     print(len(chats))
     # convert to dataframe
     chats = pd.DataFrame(chats)
     if args.start_time is not None:
-        chats = chats[pd.to_datetime(chats["tstamp"], unit="s") >= pd.to_datetime(args.start_time)]
-        chats = chats[pd.to_datetime(chats["tstamp"], unit='s') < pd.to_datetime(args.end_time)]
+        chats = chats[
+            pd.to_datetime(chats["tstamp"], unit="s") >= pd.to_datetime(args.start_time)
+        ]
+        chats = chats[
+            pd.to_datetime(chats["tstamp"], unit="s") < pd.to_datetime(args.end_time)
+        ]
         print(len(chats))
 
     last_updated_tstamp = chats.iloc[-1]["tstamp"]

fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py

@@ -98,11 +98,13 @@ def to_remove(x):
 #     "lmsys/vicuna-7b-v1.5", use_fast=False
 # )
 
+
 def num_tokens_from_string(string: str) -> int:
     encoding = tiktoken.encoding_for_model("gpt-4")
     num_tokens = len(encoding.encode(string))
     return num_tokens
 
+
 prompts = []
 responses = []
 for conv in df["conversation"]: