Refactor basic chat web app and add image analysis app

Author: sd109

.gitignore

@@ -13,3 +13,9 @@ test-values.y[a]ml
 # Helm chart stuff
 chart/Chart.lock
 chart/charts
+
+# Python stuff
+**/build/
+**/*.egg-info/
+**/flagged/
+web-apps/**/overrides.yml

web-apps/chat-interface/Dockerfile renamed to web-apps/basic-chat/Dockerfile

@@ -1,16 +1,18 @@
 FROM python:3.11-slim
 
-COPY requirements.txt requirements.txt
+ARG DIR=chat-interface
+
+COPY $DIR/requirements.txt requirements.txt
+COPY utils utils
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY purge-google-fonts.sh purge-google-fonts.sh
 RUN bash purge-google-fonts.sh
 
 WORKDIR /app
 
-COPY *.py .
+COPY $DIR/*.py .
 
-COPY defaults.yml .
-# COPY overrides.yml .
+COPY $DIR/defaults.yml .
 
 ENTRYPOINT ["python3", "app.py"]

web-apps/chat-interface/app.py renamed to web-apps/basic-chat/app.py

@@ -1,23 +1,77 @@
-import sys
 import logging
+import openai
+
 import gradio as gr
-from urllib.parse import urljoin
-from config import AppSettings
 
+from urllib.parse import urljoin
 from langchain.schema import HumanMessage, AIMessage, SystemMessage
 from langchain_openai import ChatOpenAI
-import openai
+from typing import Dict, List
+from pydantic import BaseModel, ConfigDict
+from utils import LLMParams, load_settings
 
 logging.basicConfig()
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-logger.info("Starting app")
 
-settings = AppSettings.load()
-if len(sys.argv) > 1:
-    settings.hf_model_name = sys.argv[1]
-logger.info("App settings: %s", settings)
+class AppSettings(BaseModel):
+    # Basic config
+    host_address: str
+    backend_url: str
+    model_name: str
+    model_instruction: str
+    page_title: str
+    llm_params: LLMParams
+    # Theme customisation
+    theme_params: Dict[str, str | list]
+    theme_params_extended: Dict[str, str]
+    css_overrides: str | None
+    custom_javascript: str | None
+    # Error on typos and suppress warnings for fields with 'model_' prefix
+    model_config = ConfigDict(protected_namespaces=(), extra="forbid")
+
+
+# class AppSettings(BaseModel):
+#     hf_model_name: str = Field(
+#         description="The model to use when constructing the LLM Chat client. This should match the model name running on the vLLM backend",
+#     )
+#     backend_url: HttpUrl = Field(
+#         description="The address of the OpenAI compatible API server (either in-cluster or externally hosted)"
+#     )
+#     page_title: str = Field(default="Large Language Model")
+#     page_description: Optional[str] = Field(default=None)
+#     hf_model_instruction: str = Field(
+#         default="You are a helpful and cheerful AI assistant. Please respond appropriately."
+#     )
+
+#     # Model settings
+
+#     # For available parameters, see https://docs.vllm.ai/en/latest/dev/sampling_params.html
+#     # which is based on https://platform.openai.com/docs/api-reference/completions/create
+#     llm_max_tokens: int = Field(default=500)
+#     llm_temperature: float = Field(default=0)
+#     llm_top_p: float = Field(default=1)
+#     llm_top_k: float = Field(default=-1)
+#     llm_presence_penalty: float = Field(default=0, ge=-2, le=2)
+#     llm_frequency_penalty: float = Field(default=0, ge=-2, le=2)
+
+#     # UI theming
+
+#     # Variables explicitly passed to gradio.theme.Default()
+#     # For example:
+#     # {"primary_hue": "red"}
+#     theme_params: dict[str, Union[str, List[str]]] = Field(default_factory=dict)
+#     # Overrides for theme.body_background_fill property
+#     theme_background_colour: Optional[str] = Field(default=None)
+#     # Provides arbitrary CSS and JS overrides to the UI,
+#     # see https://www.gradio.app/guides/custom-CSS-and-JS
+#     css_overrides: Optional[str] = Field(default=None)
+#     custom_javascript: Optional[str] = Field(default=None)
+
+
+settings = AppSettings(**load_settings())
+logger.info(settings)
 
 backend_url = str(settings.backend_url)
 backend_health_endpoint = urljoin(backend_url, "/health")
@@ -36,29 +90,19 @@ class PossibleSystemPromptException(Exception):
 
 llm = ChatOpenAI(
     base_url=urljoin(backend_url, "v1"),
-    model=settings.hf_model_name,
+    model=settings.model_name,
     openai_api_key="required-but-not-used",
-    temperature=settings.llm_temperature,
-    max_tokens=settings.llm_max_tokens,
-    # model_kwargs={
-    #     "top_p": settings.llm_top_p,
-    #     "frequency_penalty": settings.llm_frequency_penalty,
-    #     "presence_penalty": settings.llm_presence_penalty,
-    #     # Additional parameters supported by vLLM but not OpenAI API
-    #     # https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#extra-parameters
-    #     "extra_body": {
-    #         "top_k": settings.llm_top_k,
-    #     }
-    top_p=settings.llm_top_p,
-    frequency_penalty=settings.llm_frequency_penalty,
-    presence_penalty=settings.llm_presence_penalty,
-    # Additional parameters supported by vLLM but not OpenAI API
-    # https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#extra-parameters
+    temperature=settings.llm_params.temperature,
+    max_tokens=settings.llm_params.max_tokens,
+    top_p=settings.llm_params.top_p,
+    frequency_penalty=settings.llm_params.frequency_penalty,
+    presence_penalty=settings.llm_params.presence_penalty,
     extra_body={
-        "top_k": settings.llm_top_k,
+        "top_k": settings.llm_params.top_k,
     },
     streaming=True,
 )
+logger.info(llm)
 
 
 def inference(latest_message, history):
@@ -67,13 +111,13 @@ def inference(latest_message, history):
 
     try:
         if INCLUDE_SYSTEM_PROMPT:
-            context = [SystemMessage(content=settings.hf_model_instruction)]
+            context = [SystemMessage(content=settings.model_instruction)]
         else:
             context = []
         for i, (human, ai) in enumerate(history):
             if not INCLUDE_SYSTEM_PROMPT and i == 0:
                 # Mimic system prompt by prepending it to first human message
-                human = f"{settings.hf_model_instruction}\n\n{human}"
+                human = f"{settings.model_instruction}\n\n{human}"
             context.append(HumanMessage(content=human))
             context.append(AIMessage(content=(ai or "")))
         context.append(HumanMessage(content=latest_message))
@@ -131,8 +175,8 @@ def inference(latest_message, history):
 
 # UI theming
 theme = gr.themes.Default(**settings.theme_params)
-if settings.theme_background_colour:
-    theme.body_background_fill = settings.theme_background_colour
+theme.set(**settings.theme_params_extended)
+# theme.set(text)
 
 
 def inference_wrapper(*args):
@@ -153,7 +197,7 @@ def inference_wrapper(*args):
 
 
 # Build main chat interface
-with gr.ChatInterface(
+app = gr.ChatInterface(
     inference_wrapper,
     chatbot=gr.Chatbot(
         # Height of conversation window in CSS units (string) or pixels (int)
@@ -167,24 +211,54 @@ def inference_wrapper(*args):
         scale=7,
     ),
     title=settings.page_title,
-    description=settings.page_description,
     retry_btn="Retry",
     undo_btn="Undo",
     clear_btn="Clear",
     analytics_enabled=False,
     theme=theme,
     css=settings.css_overrides,
     js=settings.custom_javascript,
-) as app:
-    logger.debug("Gradio chat interface config: %s", app.config)
-    # For running locally in tilt dev setup
-    if len(sys.argv) > 2 and sys.argv[2] == "localhost":
-        app.launch()
-    # For running on cluster
-    else:
-        app.queue(
-            # Allow 10 concurrent requests to backend
-            # vLLM backend should be clever enough to
-            # batch these requests appropriately.
-            default_concurrency_limit=10,
-        ).launch(server_name="0.0.0.0")
+)
+logger.debug("Gradio chat interface config: %s", app.config)
+app.queue(
+    # Allow 10 concurrent requests to backend
+    # vLLM backend should be clever enough to
+    # batch these requests appropriately.
+    default_concurrency_limit=10,
+).launch(server_name=settings.host_address)
+
+# with gr.ChatInterface(
+#     inference_wrapper,
+#     chatbot=gr.Chatbot(
+#         # Height of conversation window in CSS units (string) or pixels (int)
+#         height="68vh",
+#         show_copy_button=True,
+#     ),
+#     textbox=gr.Textbox(
+#         placeholder="Ask me anything...",
+#         container=False,
+#         # Ratio of text box to submit button width
+#         scale=7,
+#     ),
+#     title=settings.page_title,
+#     description=settings.page_description,
+#     retry_btn="Retry",
+#     undo_btn="Undo",
+#     clear_btn="Clear",
+#     analytics_enabled=False,
+#     theme=theme,
+#     css=settings.css_overrides,
+#     js=settings.custom_javascript,
+# ) as app:
+#     logger.debug("Gradio chat interface config: %s", app.config)
+#     # For running locally in tilt dev setup
+#     if len(sys.argv) > 2 and sys.argv[2] == "localhost":
+#         app.launch()
+#     # For running on cluster
+#     else:
+#         app.queue(
+#             # Allow 10 concurrent requests to backend
+#             # vLLM backend should be clever enough to
+#             # batch these requests appropriately.
+#             default_concurrency_limit=10,
+#         ).launch(server_name=settings.host_address)

web-apps/basic-chat/defaults.yml

@@ -0,0 +1,31 @@
+
+model_name:
+model_instruction: "You are a helpful and cheerful AI assistant. Please respond appropriately."
+backend_url:
+host_address: 0.0.0.0
+
+page_title: Large Language Model
+
+# LLM request parameters
+# See https://platform.openai.com/docs/api-reference/chat/create
+# and https://docs.vllm.ai/en/v0.6.0/serving/openai_compatible_server.html#extra-parameters
+llm_params:
+  max_tokens:
+  temperature: 0
+  top_p:
+  top_k:
+  frequency_penalty:
+  presence_penalty:
+
+# Gradio theme constructor parameters (e.g. 'primary_hue')
+# See https://www.gradio.app/guides/theming-guide
+theme_params: {}
+
+# Gradio theme .set(...) parameters
+# See https://www.gradio.app/guides/theming-guide#extending-themes-via-set
+theme_params_extended: {}
+
+# Additional CSS and JS overrides
+# See https://www.gradio.app/guides/custom-CSS-and-JS
+css_overrides:
+custom_javascript:

web-apps/chat-interface/gradio-client-test.py renamed to web-apps/basic-chat/gradio-client-test.py

@@ -4,4 +4,4 @@ openai
 langchain
 langchain_openai
 pydantic
-pydantic_settings
+-e ../utils

web-apps/chat-interface/requirements.txt renamed to web-apps/basic-chat/requirements.txt

@@ -2,14 +2,12 @@
 set -e
 
 build() {
-    pushd $1 > /dev/null
-    if [[ -f Dockerfile ]]; then
+    if [[ -f $1/Dockerfile ]]; then
         echo Building $1 docker image
-        docker build . -t ghcr.io/stackhpc/azimuth-llm-$1
+        docker build . -t ghcr.io/stackhpc/azimuth-llm-$1 -f $1/Dockerfile
     else
         echo No Dockerfile found for $1
     fi
-    popd > /dev/null
 }
 
 # If a single app is provided as a