update

Author: infwinston

fastchat/serve/monitor/basic_stats.py

@@ -16,11 +16,13 @@
 LOG_ROOT_DIR = "~/fastchat_logs"
 
 
-def get_log_files(max_num_files=None):
+def get_log_files(max_num_files=None, is_vision=False):
     log_root = os.path.expanduser(LOG_ROOT_DIR)
     filenames = []
     for i in range(NUM_SERVERS):
         for filename in os.listdir(f"{log_root}/server{i}"):
+            if is_vision and not filename.startswith("vision-"):
+                continue
             if filename.endswith("-conv.json"):
                 filepath = f"{log_root}/server{i}/{filename}"
                 name_tstamp_tuple = (filepath, os.path.getmtime(filepath))
@@ -39,7 +41,12 @@ def load_log_files(filename):
     for retry in range(5):
         try:
             for l in open(filename):
-                row = json.loads(l)
+                try:
+                    row = json.loads(l)
+                except json.decoder.JSONDecodeError:
+                    print(f"JSONDecodeError: {l}")
+                    continue
+
                 data.append(
                     dict(
                         type=row["type"],
@@ -232,9 +239,11 @@ def report_basic_stats(log_files):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--max-num-files", type=int)
+    parser.add_argument("--vision", action="store_true")
+
     args = parser.parse_args()
 
-    log_files = get_log_files(args.max_num_files)
+    log_files = get_log_files(args.max_num_files, args.vision)
     basic_stats = report_basic_stats(log_files)
 
     print(basic_stats["action_hist_md"] + "\n")

fastchat/serve/monitor/clean_battle_data.py

@@ -130,9 +130,22 @@ def replace_model_name(old_name, tstamp):
         "im-also-a-late-june-chatbot": "gemma-2-9b-it",
         "gemini-test-1": "gemini-1.5-pro-exp-0801",
         "gemini-test-2": "gemini-1.5-pro-exp-0801",
-        "anonymous-chatbot": "chatgpt-4o-latest",
+        "anonymous-chatbot": "chatgpt-4o-latest-20240808",
         "toto-mini": "jamba-1.5-mini",
         "toto-medium": "jamba-1.5-large",
+        "gemini-test-3": "gemini-1.5-pro-exp-0827",
+        "engine-test": "gemini-1.5-flash-exp-0827",
+        "little-engine-test": "gemini-1.5-flash-8b-exp-0827",
+        "chatgpt-4o-latest": "chatgpt-4o-latest-20240808",
+        "anonymous-chatbot-0903": "chatgpt-4o-latest-20240903",
+        "the-real-chatbot-v1": "llama-3.1-405b-instruct-bf16",
+        "llama-3.1-405b-instruct": "llama-3.1-405b-instruct-fp8",
+        "gemini-test-5": "gemini-1.5-pro-002-test",
+        "dumbledore-v3": "llama-3.2-vision-90b-instruct",
+        "potter-v1": "llama-3.2-vision-11b-instruct",
+        "sharp-game-player-v1": "llama-3.2-3b-instruct",
+        "zeus-flare-thunder-v1": "llama-3.2-1b-instruct",
+        "qwen-vl-max-0809": "qwen2-vl-72b",
     }
     if old_name in ["gpt-4", "gpt-3.5-turbo"]:
         if tstamp > 1687849200:
@@ -166,7 +179,7 @@ def replace_model_name(old_name, tstamp):
     if old_name == "deepseek-coder-v2" and tstamp > 1721663428:
         return "deepseek-coder-v2-0724"
     if old_name == "meta-llama-3.1-405b-instruct" and tstamp > 1721847659:
-        return "llama-3.1-405b-instruct"
+        return "llama-3.1-405b-instruct-fp8"
     if old_name == "meta-llama-3.1-70b-instruct-sp" and tstamp > 1721847659:
         return "llama-3.1-70b-instruct"
     if old_name == "meta-llama-3.1-8b-instruct-sp" and tstamp > 1721847659:
@@ -185,8 +198,12 @@ def read_file(filename, vision=False):
     for retry in range(5):
         try:
             # lines = open(filename).readlines()
-            for l in open(filename):
-                row = json.loads(l)
+            for i, l in enumerate(open(filename)):
+                try:
+                    row = json.loads(l)
+                except json.JSONDecodeError as e:
+                    print(f"JSON Decode Error in file {filename} line {i}")
+                    continue
                 if row["type"] in VOTES:
                     data.append(row)
             break
@@ -225,6 +242,7 @@ def process_data(
     count_dict = {
         "anony": 0,
         "invalid": 0,
+        "encoding_invalid": 0,
         "leaked_identity": 0,
         "banned": 0,
         "error": 0,
@@ -312,6 +330,13 @@ def process_data(
                     else:
                         flag_none_msg = True
 
+            # test if message string can be encoded to utf-8
+            try:
+                messages.encode("utf-8")
+            except:
+                count_dict["encoding_invalid"] += 1
+                continue
+
             if vision != flag_vision:
                 count_dict["invalid"] += 1
                 continue
@@ -353,7 +378,7 @@ def process_data(
             if exclude_model_names:
                 exclude = False
                 for exclude_model in exclude_model_names:
-                    if exclude_model in models[0] or exclude_model in models[1]:
+                    if models[0] == exclude_model or models[1] == exclude_model:
                         count_dict["exclude_model"] += 1
                         exclude = True
                         break

fastchat/serve/monitor/clean_chat_data.py

@@ -10,6 +10,8 @@
 import os
 from pytz import timezone
 import time
+import pandas as pd
+import tiktoken
 
 from tqdm import tqdm
 
@@ -26,16 +28,20 @@
 )
 
 
-def get_log_files(max_num_files=None):
+def get_log_files(max_num_files=None, is_vision=False):
     dates = []
-    for month in range(4, 12):
-        for day in range(1, 33):
-            dates.append(f"2023-{month:02d}-{day:02d}")
+    for year in range(2023, 2025):
+        for month in range(1, 13):
+            for day in range(1, 33):
+                dates.append(f"{year}-{month:02d}-{day:02d}")
 
     filenames = []
     for d in dates:
         for i in range(NUM_SERVERS):
-            name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
+            prefix = ""
+            if is_vision:
+                prefix = "vision-tmp-"
+            name = os.path.expanduser(f"~/fastchat_logs/server{i}/{prefix}{d}-conv.json")
             if os.path.exists(name):
                 filenames.append(name)
     max_num_files = max_num_files or len(filenames)
@@ -44,7 +50,8 @@ def get_log_files(max_num_files=None):
     return filenames
 
 
-def clean_chat_data(log_files, action_type):
+def clean_chat_data(log_files, action_type, remove_prompt=False):
+    encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
     raw_data = []
     for filename in tqdm(log_files, desc="read files"):
         for retry in range(5):
@@ -65,11 +72,15 @@ def clean_chat_data(log_files, action_type):
     ct_invalid_conv_id = 0
     ct_invalid = 0
     ct_network_error = 0
+    ct_img_chat = 0
+    ct_csam = 0
     for row in raw_data:
         try:
             if action_type in ["chat", "upvote", "downvote"]:
                 state = row["state"]
                 model = row["model"]
+                if state.get("has_csam_image", False):
+                    ct_csam += 1
             elif action_type == "leftvote":
                 state = row["states"][0]
                 model = row["states"][0]["model_name"]
@@ -92,17 +103,31 @@ def clean_chat_data(log_files, action_type):
         model = replace_model_name(model, row["tstamp"])
 
         try:
-            lang_code = detect_language(state["messages"][state["offset"]][1])
-        except IndexError:
+            msg = state["messages"][state["offset"]][1]
+            if isinstance(msg, list):
+                ct_img_chat += 1
+                msg = msg[0]
+            lang_code = detect_language(msg)
+            if not all(isinstance(x["content"][0], str) for x in conversation):
+                ct_invalid += 1
+        except (IndexError, TypeError):
             ct_invalid += 1
             continue
 
-        if not all(isinstance(x["content"], str) for x in conversation):
-            ct_invalid += 1
-            continue
+        # add token length
+        messages_concat = ""
+        for x in conversation:
+            msg = x["content"]
+            if isinstance(x["content"], list):
+                msg = x["content"][0]
+            x["num_tokens"] = len(
+                encoding.encode(msg, allowed_special="all")
+            )
+            messages_concat += msg.lower()
 
-        messages = "".join([x["content"] for x in conversation]).lower()
-        if NETWORK_ERROR_MSG in messages:
+            if remove_prompt:
+                x.pop("content")
+        if NETWORK_ERROR_MSG in messages_concat:
             ct_network_error += 1
             continue
 
@@ -141,10 +166,10 @@ def clean_chat_data(log_files, action_type):
         dedup_chats.append(chats[i])
 
     print(
-        f"#raw: {len(raw_data)}, #chat: {len(chats)}, #dedup_chat: {len(dedup_chats)}"
+        f"#raw: {len(raw_data)}, #chat: {len(chats)}, #dedup_chat: {len(dedup_chats)}, #csam: {ct_csam}"
     )
     print(
-        f"#invalid_conv_id: {ct_invalid_conv_id}, #network_error: {ct_network_error}, #invalid: {ct_invalid}"
+        f"#invalid_conv_id: {ct_invalid_conv_id}, #network_error: {ct_network_error}, #invalid: {ct_invalid}, #img-chat: {ct_img_chat}"
     )
     print(f"#models: {len(all_models)}, {all_models}")
     print(f"last-updated: {last_updated_datetime}")
@@ -156,16 +181,30 @@ def clean_chat_data(log_files, action_type):
     parser = argparse.ArgumentParser()
     parser.add_argument("--action-type", type=str, default="chat")
     parser.add_argument("--max-num-files", type=int)
+    parser.add_argument("--vision", action="store_true")
+    parser.add_argument("--start-time", type=str) # example: 2024-08-01
+    parser.add_argument("--end-time", type=str) # example: 2024-08-01
+    parser.add_argument("--remove-prompt", action="store_true")
     args = parser.parse_args()
 
-    log_files = get_log_files(args.max_num_files)
-    chats = clean_chat_data(log_files, args.action_type)
-    last_updated_tstamp = chats[-1]["tstamp"]
+    log_files = get_log_files(args.max_num_files, args.vision)
+    # print(log_files)
+    chats = clean_chat_data(log_files, args.action_type, args.remove_prompt)
+    print(len(chats))
+    # convert to dataframe
+    chats = pd.DataFrame(chats)
+    if args.start_time is not None:
+        chats = chats[pd.to_datetime(chats["tstamp"], unit="s") >= pd.to_datetime(args.start_time)]
+        chats = chats[pd.to_datetime(chats["tstamp"], unit='s') < pd.to_datetime(args.end_time)]
+        print(len(chats))
+
+    last_updated_tstamp = chats.iloc[-1]["tstamp"]
     cutoff_date = datetime.datetime.fromtimestamp(
         last_updated_tstamp, tz=timezone("US/Pacific")
     ).strftime("%Y%m%d")
 
     output = f"clean_{args.action_type}_conv_{cutoff_date}.json"
-    with open(output, "w") as fout:
-        json.dump(chats, fout, indent=2, ensure_ascii=False)
+    # with open(output, "w") as fout:
+    #     json.dump(chats, fout, indent=2, ensure_ascii=False)
+    chats.to_json(output, orient="records", indent=2, force_ascii=False)
     print(f"Write cleaned data to {output}")

fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py

@@ -15,6 +15,7 @@
 import plotly.express as px
 import plotly.graph_objects as go
 from tqdm import tqdm
+import tiktoken
 
 import plotly.io as pio
 
@@ -91,11 +92,16 @@ def to_remove(x):
 fig.show()
 fig.write_image("daily_conversation_count.pdf")
 
-import transformers
+# import transformers
 
-tokenizer = transformers.AutoTokenizer.from_pretrained(
-    "lmsys/vicuna-7b-v1.5", use_fast=False
-)
+# tokenizer = transformers.AutoTokenizer.from_pretrained(
+#     "lmsys/vicuna-7b-v1.5", use_fast=False
+# )
+
+def num_tokens_from_string(string: str) -> int:
+    encoding = tiktoken.encoding_for_model("gpt-4")
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
 
 prompts = []
 responses = []
@@ -110,10 +116,25 @@ def to_remove(x):
 print(f"#responses: {len(responses)}")
 
 
-prompt_lens = [len(tokenizer(x).input_ids) for x in tqdm(prompts)]
-print()
-print(f"mean prompt len: {np.mean(prompt_lens):.2f}")
+prompt_lens = []
+response_lens = []
+for x in tqdm(prompts):
+    try:
+        if isinstance(x, list):
+            x = x[0]
+        prompt_lens.append(num_tokens_from_string(x))
+    except Exception as e:
+        print(f"Error processing prompt: {e}")
+        print(x)
+
+for x in tqdm(responses):
+    try:
+        if isinstance(x, list):
+            x = x[0]
+        response_lens.append(num_tokens_from_string(x))
+    except Exception as e:
+        print(f"Error processing prompt: {e}")
+        print(x)
 
-response_lens = [len(tokenizer(x).input_ids) if x else 0 for x in tqdm(responses)]
-print()
+print(f"mean prompt len: {np.mean(prompt_lens):.2f}")
 print(f"mean response len: {np.mean(response_lens):.2f}")