Merge with unified vision arena

Author: BabyChouSr

fastchat/serve/gradio_block_arena_vision.py

@@ -199,8 +199,19 @@ def convert_images_to_conversation_format(images):
     return conv_images
 
 
-def add_text(state, model_selector, chat_input, request: gr.Request):
-    text, images = chat_input["text"], chat_input["files"]
+def add_text(state, model_selector, chat_input, context: Context, request: gr.Request):
+    if isinstance(chat_input, dict):
+        text, images = chat_input["text"], chat_input["files"]
+    else:
+        text, images = chat_input, []
+
+    if (
+        len(images) > 0
+        and model_selector in context.text_models
+        and model_selector not in context.vision_models
+    ):
+        gr.Warning(f"{model_selector} is a text-only model. Image is ignored.")
+        images = []
     ip = get_ip(request)
     logger.info(f"add_text. ip: {ip}. len: {len(text)}")
 
@@ -222,11 +233,13 @@ def add_text(state, model_selector, chat_input, request: gr.Request):
     images = convert_images_to_conversation_format(images)
 
     # Use the first state to get the moderation response because this is based on user input so it is independent of the model
+    moderation_image_input = images[0] if len(images) > 0 else None
     moderation_type_to_response_map = (
         state.content_moderator.image_and_text_moderation_filter(
-            images[0], text, [state.model_name], do_moderation=False
+            moderation_image_input, text, [state.model_name], do_moderation=False
         )
     )
+
     text_flagged, nsfw_flag, csam_flag = (
         moderation_type_to_response_map["text_moderation"]["flagged"],
         moderation_type_to_response_map["nsfw_moderation"]["flagged"],
@@ -245,7 +258,9 @@ def add_text(state, model_selector, chat_input, request: gr.Request):
         state.conv.append_message(state.conv.roles[0], post_processed_text)
         state.skip_next = True
         gr.Warning(MODERATION_MSG)
-        return (state, gradio_chatbot_before_user_input, None) + (no_change_btn,) * 5
+        return (state, gradio_chatbot_before_user_input, None, "", no_change_btn) + (
+            no_change_btn,
+        ) * 5
 
     if (len(state.conv.messages) - state.conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
         logger.info(f"conversation turn limit. ip: {ip}. text: {text}")

fastchat/serve/gradio_block_arena_vision_anony.py

@@ -322,18 +322,12 @@ def add_text(
     images = convert_images_to_conversation_format(images)
 
     # Use the first state to get the moderation response because this is based on user input so it is independent of the model
-    if len(images) > 0:
-        moderation_type_to_response_map = states[
-            0
-        ].content_moderator.image_and_text_moderation_filter(
-            images[0], text, model_list, do_moderation=True
-        )
-    else:
-        moderation_type_to_response_map = states[
-            0
-        ].content_moderator.image_and_text_moderation_filter(
-            None, text, model_list, do_moderation=True
-        )
+    moderation_image_input = images[0] if len(images) > 0 else None
+    moderation_type_to_response_map = states[
+        0
+    ].content_moderator.image_and_text_moderation_filter(
+        moderation_image_input, text, model_list, do_moderation=True
+    )
     text_flagged, nsfw_flag, csam_flag = (
         moderation_type_to_response_map["text_moderation"]["flagged"],
         moderation_type_to_response_map["nsfw_moderation"]["flagged"],

fastchat/serve/gradio_block_arena_vision_named.py

@@ -36,7 +36,6 @@
     convert_images_to_conversation_format,
     enable_multimodal_keep_input,
     enable_multimodal_clear_input,
-    enable_multimodal,
     disable_multimodal,
     invisible_text,
     invisible_btn,
@@ -254,18 +253,13 @@ def add_text(
     images = convert_images_to_conversation_format(images)
 
     # Use the first state to get the moderation response because this is based on user input so it is independent of the model
-    if len(images) > 0:
-        moderation_type_to_response_map = states[
-            0
-        ].content_moderator.image_and_text_moderation_filter(
-            images[0], text, model_list, do_moderation=False
-        )
-    else:
-        moderation_type_to_response_map = states[
-            0
-        ].content_moderator.image_and_text_moderation_filter(
-            None, text, model_list, do_moderation=False
-        )
+    moderation_image_input = images[0] if len(images) > 0 else None
+    moderation_type_to_response_map = states[
+        0
+    ].content_moderator.image_and_text_moderation_filter(
+        moderation_image_input, text, model_list, do_moderation=False
+    )
+
     text_flagged, nsfw_flag, csam_flag = (
         moderation_type_to_response_map["text_moderation"]["flagged"],
         moderation_type_to_response_map["nsfw_moderation"]["flagged"],
@@ -309,7 +303,7 @@ def add_text(
         return (
             states
             + gradio_chatbot_list
-            + [None]
+            + [None, "", no_change_btn]
             + [
                 no_change_btn,
             ]

fastchat/serve/gradio_web_server.py

@@ -477,7 +477,7 @@ def bot_response(
         if state.content_moderator.text_flagged or state.content_moderator.nsfw_flagged:
             start_tstamp = time.time()
             finish_tstamp = start_tstamp
-            conv.save_new_images(
+            state.conv.save_new_images(
                 has_csam_images=state.has_csam_image,
                 use_remote_storage=use_remote_storage,
             )

fastchat/serve/gradio_web_server_multi.py

@@ -192,7 +192,10 @@ def build_demo(context: Context, elo_results_file: str, leaderboard_table_file):
             if elo_results_file:
                 with gr.Tab("🏆 Leaderboard", id=3):
                     build_leaderboard_tab(
-                        elo_results_file, leaderboard_table_file, show_plot=True
+                        elo_results_file,
+                        leaderboard_table_file,
+                        arena_hard_leaderboard=None,
+                        show_plot=True,
                     )
 
             with gr.Tab("ℹ️ About Us", id=4):

fastchat/serve/monitor/monitor.py

@@ -864,6 +864,8 @@ def build_leaderboard_tab(
                         vision=True,
                         show_plot=show_plot,
                     )
+
+            model_to_score = {}
             if arena_hard_leaderboard is not None:
                 with gr.Tab("Arena-Hard-Auto", id=3):
                     dataFrame = arena_hard_process(
@@ -883,7 +885,6 @@ def build_leaderboard_tab(
                             "avg_tokens": "Average Tokens",
                         }
                     )
-                    model_to_score = {}
                     for i in range(len(dataFrame)):
                         model_to_score[dataFrame.loc[i, "Model"]] = dataFrame.loc[
                             i, "Win-rate"