Queries all selected models at same time

In earlier script, even though we select 4 models, it queries first model, waits for its output, after it is done, it moves to second model and waits for its output, after it is done, it moves to third model and so on.. 
Changed it to Parallel execution so that all models are queried at the same time.

Author: ChayScripts

Horizontal View - app.py

@@ -1,6 +1,7 @@
 import streamlit as st
 import requests
 import time
+import concurrent.futures
 
 st.set_page_config(page_title="LLM Comparison", layout="wide")
 
@@ -50,12 +51,16 @@ def remove_model(index):
         st.session_state.selected_models.pop(index)
 
 for i in range(st.session_state.model_count):
-    col1, col2 = st.columns([0.97, 0.02]) 
+    col1, col2 = st.columns([0.97, 0.02])
     with col1:
+        # Ensure the list is long enough
+        if i >= len(st.session_state.selected_models):
+            st.session_state.selected_models.append("") 
+        
         st.session_state.selected_models[i] = st.selectbox(
             f"Model {i+1}",
             models_available,
-            index=0 if i >= len(st.session_state.selected_models) or not st.session_state.selected_models[i] else models_available.index(st.session_state.selected_models[i]),
+            index=0 if i >= len(st.session_state.selected_models) or not st.session_state.selected_models[i] else (models_available.index(st.session_state.selected_models[i]) if st.session_state.selected_models[i] in models_available else 0),
             key=f"model_select_{i}"
         )
     with col2:
@@ -72,47 +77,75 @@ def remove_model(index):
 with col_run:
     run_clicked = st.button("Run Models", type="primary")
 
+def query_ollama_model(model_name, prompt_text):
+    """Function to query a single Ollama model."""
+    try:
+        start_time = time.time()
+        res = requests.post(
+            "http://localhost:11434/api/generate",
+            json={"model": model_name, "prompt": prompt_text, "stream": False},
+            headers={"Content-Type": "application/json"},
+        )
+        res.raise_for_status()
+        response_data = res.json()
+        end_time = time.time()
+
+        duration = round(end_time - start_time, 2)
+        content = response_data.get("response", "")
+        eval_count = response_data.get("eval_count", len(content.split()))
+        eval_rate = response_data.get("eval_rate", round(eval_count / duration, 2) if duration > 0 else 0)
+
+        return {
+            "model": model_name,
+            "duration": duration,
+            "eval_count": eval_count,
+            "eval_rate": eval_rate,
+            "response": content
+        }
+    except Exception as e:
+        return {
+            "model": model_name,
+            "duration": 0,
+            "eval_count": 0,
+            "eval_rate": 0,
+            "response": f"Error: {e}"
+        }
+
 if run_clicked and prompt and selected_models_filtered:
     responses = []
-
-    response_placeholders = [st.empty() for _ in selected_models_filtered]
-
-    for i, model in enumerate(selected_models_filtered):
-        try:
-            with st.spinner(f"Running {model}..."):
-                start_time = time.time()
-                res = requests.post(
-                    "http://localhost:11434/api/generate",
-                    json={"model": model, "prompt": prompt, "stream": False},
-                    headers={"Content-Type": "application/json"},
-                )
-                res.raise_for_status()
-                response_data = res.json()
-                end_time = time.time()
-
-            duration = round(end_time - start_time, 2)
-            content = response_data.get("response", "")
-            eval_count = response_data.get("eval_count", len(content.split()))
-            eval_rate = response_data.get("eval_rate", round(eval_count / duration, 2))
-
-            responses.append({
-                "model": model,
-                "duration": duration,
-                "eval_count": eval_count,
-                "eval_rate": eval_rate,
-                "response": content
-            })
-        except Exception as e:
-            responses.append({
-                "model": model,
-                "duration": 0,
-                "eval_count": 0,
-                "eval_rate": 0,
-                "response": f"Error: {e}"
-            })
-
-    cols = st.columns(len(responses))
-    for i, res in enumerate(responses):
+    
+    # Create placeholders for immediate feedback
+    response_placeholders = {model: st.empty() for model in selected_models_filtered}
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(selected_models_filtered)) as executor:
+        # Submit tasks to the thread pool
+        future_to_model = {executor.submit(query_ollama_model, model, prompt): model for model in selected_models_filtered}
+        
+        # Iterate as futures complete
+        for future in concurrent.futures.as_completed(future_to_model):
+            model_name = future_to_model[future]
+            try:
+                res = future.result()
+                responses.append(res)
+            except Exception as exc:
+                responses.append({
+                    "model": model_name,
+                    "duration": 0,
+                    "eval_count": 0,
+                    "eval_rate": 0,
+                    "response": f"Error: {exc}"
+                })
+
+    # Sort responses by the order of selected models for consistent display
+    ordered_responses = []
+    for model in selected_models_filtered:
+        for res in responses:
+            if res["model"] == model:
+                ordered_responses.append(res)
+                break
+
+    cols = st.columns(len(ordered_responses))
+    for i, res in enumerate(ordered_responses):
         with cols[i]:
             st.markdown(
                 f"### <span style='color:#3366cc'>{res['model']}</span>" if i % 2 == 0 else f"### <span style='color:#cc0000'>{res['model']}</span>",