feat: UI component consistency through conversation

Author: velias

libs/next_gen_ui_agent/agent.py

@@ -126,7 +126,9 @@ async def component_selection(
                 )
 
             input_to_dynamic_selection = AgentInput(
-                user_prompt=input["user_prompt"], input_data=to_dynamic_selection
+                user_prompt=input["user_prompt"],
+                input_data=to_dynamic_selection,
+                previous_user_prompts=input.get("previous_user_prompts"),
             )
             from_dynamic_selection = (
                 await self._component_selection_strategy.select_components(

libs/next_gen_ui_agent/component_selection.py

@@ -15,16 +15,16 @@
 from pydantic_core import from_json
 
 ui_components_description_supported = """
-* one-card - component to visualize multiple fields from one-item data. One image can be shown if url is available together with other fields. Array of simple values from one-item data can be shown as a field. Array of objects can't be shown as a field.
-* video-player - component to play video from one-item data. Videos like trailers, promo videos. Data must contain url pointing to the video to be shown, e.g. https://www.youtube.com/watch?v=v-PjgYDrg70
-* image - component to show one image from one-item data. Images like posters, covers, pictures. Do not use for video! Select it if no other fields are necessary to be shown. Data must contain url pointing to the image to be shown, e.g. https://www.images.com/v-PjgYDrg70.jpeg
+- 'one-card': component to show multiple fields from one-item data. One image can be shown if url is available together with other fields. Array of simple values from one-item data can be shown as a field. Array of objects can't be shown as a field.
+- 'video-player': component to play video from one-item data. Videos like trailers, promo videos. Data must contain url pointing to the video to be shown, e.g. https://www.youtube.com/watch?v=v-PjgYDrg70
+- 'image': component to show one image from one-item data. Images like posters, covers, pictures. Do not use for video! Do not use if other fields has to be shown, use `one-card` instead! Data must contain url pointing to the image to be shown, e.g. https://www.images.com/v-PjgYDrg70.jpeg
 """
 
 ui_components_description_all = (
     ui_components_description_supported
     + """
-* table - component to visualize array of objects with more than 6 items and small number of shown fields with short values.
-* set-of-cards - component to visualize array of objects with less than 6 items, or high number of shown fields and fields with long values.
+- 'table': component to show array of objects with more than 6 items and small number of shown fields with short values.
+- 'set-of-cards': component to show array of objects with less than 6 items, or high number of shown fields and fields with long values.
 """.strip()
 )
 
@@ -64,7 +64,12 @@ async def select_components(
         logger.debug("---CALL component_selection---")
         components = await asyncio.gather(
             *[
-                self.component_selection_run(inference, input["user_prompt"], data)
+                self.component_selection_run(
+                    inference,
+                    input["user_prompt"],
+                    data,
+                    input.get("previous_user_prompts"),
+                )
                 for data in input["input_data"]
             ]
         )
@@ -79,6 +84,7 @@ async def perform_inference(
         user_prompt: str,
         json_data: Any,
         input_data_id: str,
+        previous_user_prompts: list[str] | None = None,
     ) -> list[str]:
         """Run Component Selection inference."""
 
@@ -89,17 +95,23 @@ async def perform_inference(
             # logger.debug(user_prompt)
             # logger.debug(input_data)
 
-        sys_msg_content = f"""You are helpful and advanced user interface design assistant. Based on the "User query" and JSON formatted "Data", select the best UI component to visualize the "Data" to the user.
+        sys_msg_content = """You are helpful and advanced user interface design assistant. Based on the "User query" and JSON formatted "Data", select the best UI component to visualize the "Data" to the user.
 Generate response in the JSON format only. Select one component only into "component".
 Provide the title for the component in "title".
 Provide reason for the component selection in the "reasonForTheComponentSelection".
 Provide your confidence for the component selection as a percentage in the "confidenceScore".
 Provide list of "fields" to be visualized in the UI component. Select only relevant data fields to be presented in the component. Do not bloat presentation. Show all the important info about the data item. Mainly include information the user asks for in User query.
 If the selected UI component requires specific fields mentioned in its description, provide them. Provide "name" for every field.
 For every field provide "data_path" containing JSONPath to get the value from the Data. Do not use any formatting or calculation in the "data_path".
+"""
 
-Select one from there UI components: {get_ui_components_description(self.unsupported_components)}
-    """
+        sys_msg_content += f"""
+Select one from these UI components: {get_ui_components_description(self.unsupported_components)}
+"""
+        if previous_user_prompts and len(previous_user_prompts) > 0:
+            sys_msg_content += """
+Generate response for the current "User query" and "Data". Keep current UI component and its configuration consistent with the expected UI component generated previously for the "Previous user queries" if applicable for the current "User query" and "Data".
+"""
 
         sys_msg_content += """
 Response example for multi-item data:
@@ -129,12 +141,24 @@ async def perform_inference(
         # we have to parse JSON data to reduce arrays
         data = reduce_arrays(json_data, 6)
 
-        prompt = f"""=== User query ===
-    {user_prompt}
+        prompt = f"""User query:
+{user_prompt}
+"""
+
+        if previous_user_prompts and len(previous_user_prompts) > 0:
+            prompt += f"""
+Previous user queries:
+- {"\n- ".join(previous_user_prompts)}
+"""
 
-    === Data ===
-    {str(data)}
-        """
+        prompt += f"""
+Data:
+{str(data)}
+"""
+
+        # TODO remove this after testing
+        # if previous_user_prompts and len(previous_user_prompts) > 0:
+        #    print("prompt:", prompt)
 
         logger.debug("LLM system message:\n%s", sys_msg_content)
         logger.debug("LLM prompt:\n%s", prompt)
@@ -162,6 +186,7 @@ async def component_selection_run(
         inference: InferenceBase,
         user_prompt: str,
         input_data: InputData,
+        previous_user_prompts: list[str] | None = None,
     ) -> UIComponentMetadata:
         """Run Component Selection task."""
 
@@ -173,7 +198,7 @@ async def component_selection_run(
             json_data = wrap_json_data(json_data, input_data.get("type"))
 
         inference_output = await self.perform_inference(
-            inference, user_prompt, json_data, input_data_id
+            inference, user_prompt, json_data, input_data_id, previous_user_prompts
         )
 
         try:

libs/next_gen_ui_agent/types.py

@@ -71,6 +71,8 @@ class AgentInput(TypedDict):
 
     user_prompt: str
     """User prompt to be processed."""
+    previous_user_prompts: NotRequired[list[str] | None]
+    """Previous user prompts - used to achieve UI component consistency through the conversation. List starts with the most recent prompt from the histroy and goes to older."""
     input_data: list[InputData]
     """Input data to be processed - one or more can be provided."""
 

tests/ui_consistency_poc/README.md

@@ -0,0 +1,11 @@
+# Research/PoC of the UI component consistency during conversation
+
+[![Module Category](https://img.shields.io/badge/Module%20Category-Testing/Evaluation-darkmagenta)](https://github.com/RedHat-UX/next-gen-ui-agent)
+[![Module Status](https://img.shields.io/badge/Module%20Status-Tech%20Preview-orange)](https://github.com/RedHat-UX/next-gen-ui-agent)
+
+This module contains code used for UI component consistency during conversation research/PoC and detailed results
+
+## Provides
+
+* `one_component.py` - runnable code for one component consistency research PoC - you can select LLM at the begining (uses embedded Llama Stack). Writes out to console and files in `one_component_results/` directory
+* `one_component_results/` - directory with results from `one_component.py`. Files names are LLM used for the run.

tests/ui_consistency_poc/__init__.py

@@ -0,0 +1,245 @@
+import asyncio
+import os
+from next_gen_ui_agent import NextGenUIAgent
+from next_gen_ui_agent.types import InputData
+from next_gen_ui_llama_stack_embedded import init_inference_from_env
+
+
+if True:
+    # Ollama models:
+    LLAMASTACK_CONFIG_PATH_DEFAULT = "tests/ai_eval_components/llamastack-ollama.yaml"
+    #INFERENCE_MODEL_DEFAULT = "granite3.2:2b"
+    #INFERENCE_MODEL_DEFAULT = "granite3.3:2b"
+    INFERENCE_MODEL_DEFAULT="granite3.3:8b"
+    #INFERENCE_MODEL_DEFAULT="ollama/llama3.2:latest"
+
+else:
+    # Gemini API models:
+    LLAMASTACK_CONFIG_PATH_DEFAULT = "tests/ai_eval_components/llamastack-m.gemini.yaml"
+    INFERENCE_MODEL_DEFAULT="gemini/gemini-2.0-flash-lite"
+    #INFERENCE_MODEL_DEFAULT="gemini/gemini-2.0-flash"
+    #INFERENCE_MODEL_DEFAULT="gemini/gemini-2.5-flash-lite"
+    #INFERENCE_MODEL_DEFAULT="gemini/gemini-2.5-flash"
+    os.environ["INFERENCE_MODEL"] = INFERENCE_MODEL_DEFAULT
+
+BY_PREVIOUS_COMPONENT_PROMPT_PRFIX = '\nKeep UI component and its configuration consistent with previously shown '
+BY_PREVIOUS_COMPONENT_PROMPT_SUFFIX = ' and only update it to match the "User Query" and "Data" if applicable pushing requested info up, otherwise replace it with another one.'
+
+#TOYSTORY_DATA = '{"movie": {"title": "Toy Story", "pictureUrl": "https://example.com/poster.jpg", "imdb" : { "rating": 8.3 }, "countries": ["USA"], "actors": ["Tom Hanks", "Tim Allen", "Jim Varney", "Don Rickles"], "released": 1995}}'
+TOYSTORY_DATA = '''{
+    "movie": {
+        "languages": [
+            "English"
+        ],
+        "year": 1995,
+        "pictureUrl": "https://image.tmdb.org/t/p/w440_and_h660_face/uXDfjJbdP4ijW5hWSBrPrlKpxab.jpg",
+        "runtime": 81,
+        "movieId": "1",
+        "imdb": {
+            "votes": 591836,
+            "id": "0114709",
+            "rating": 8.3
+        },
+        "countries": [
+            "USA",
+            "Germany",
+            "Czech Republic"
+        ],
+        "trailerUrl": "https://www.youtube.com/watch?v=v-PjgYDrg70",
+        "title": "Toy Story",
+        "url": "https://themoviedb.org/movie/862",
+        "revenue": 373554033,
+        "tmdbId": "862",
+        "plot": "A cowboy doll is profoundly threatened and jealous when a new spaceman figure supplants him as top toy in a boy's room.",
+        "released": "1995-11-22",
+        "budget": 30000000
+    },
+    "actors": [
+        "Jim Varney",
+        "Tim Allen",
+        "Tom Hanks",
+        "Don Rickles"
+    ]
+}'''
+
+PROMPT_MOVIE_INFO = "Show me basic info about Toy Story."
+PROMPT_MOVIE_RELEASED = "When was the movie released?"
+PROMPT_MOVIE_POSTER = "Show me the poster only."
+PROMPT_MOVIE_SUBSCRIPTIONS = "Show me my subscriptions"
+
+def open_results_file():
+    """
+    Generate a valid filename from INFERENCE_MODEL_DEFAULT and open it for writing.
+    Keeps only the part after the last '/' and adds .txt suffix.
+    If file exists, it will be replaced.
+    Creates the file in the 'one_component_results' subfolder relative to this script's directory.
+    
+    Returns:
+        file: Open file handle for writing
+    """
+    import os
+    # Get the directory where this script is located
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    results_dir = os.path.join(script_dir, 'one_component_results')
+    
+    # Create the subfolder if it doesn't exist
+    os.makedirs(results_dir, exist_ok=True)
+    
+    # Extract the part after the last '/'
+    filename = INFERENCE_MODEL_DEFAULT.split('/')[-1]
+    # Add .txt suffix
+    full_filename = os.path.join(results_dir, f"{filename}.txt")
+
+    print(f"\nWriting results to {full_filename} ...\n")
+
+    # Open file for writing, replacing if it exists
+    return open(full_filename, 'w')
+
+# open results file
+results_file = open_results_file()
+
+def print_response(title, llm_response):
+    response_text = f"{title}{llm_response[0].model_dump_json(indent=2)}\n\n"
+    print(response_text, end='')
+    results_file.write(response_text)
+    results_file.flush()
+    
+def print_header(title):
+    header_text = f"{title}\n\n"
+    print(header_text, end='')
+    results_file.write(header_text)
+    results_file.flush()
+
+if __name__ == "__main__":
+
+    inference = init_inference_from_env(
+        default_model=INFERENCE_MODEL_DEFAULT,
+        default_config_file=LLAMASTACK_CONFIG_PATH_DEFAULT,
+    )
+
+    agent = NextGenUIAgent(config = {"inference": inference, "unsupported_components": True})
+
+    print_header("*** 1. Current: Info about Toy Story. Previous: subscriptions table - change of the data entity ***")
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_INFO+BY_PREVIOUS_COMPONENT_PROMPT_PRFIX+"'table' with title 'My Subscriptions' and fields ['subscription.id', 'subscription.endDate']"+BY_PREVIOUS_COMPONENT_PROMPT_SUFFIX,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response one previous component: ", llm_response)
+    
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_INFO,
+            "previous_user_prompts": [PROMPT_MOVIE_SUBSCRIPTIONS],
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response multipleprevious prompts: ", llm_response)
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_INFO,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response no history: ", llm_response)
+
+# ----------------- 2 -----------------
+    print_header("*** 2. Current: When was the movie released? Previous: Info about Toy Story - additional info asked about data entity from previous conversation step ***")
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_RELEASED+BY_PREVIOUS_COMPONENT_PROMPT_PRFIX+"'one-card' with title 'Toy Story' and fields ['movie.title', 'movie.pictureUrl', 'movie.imdb.rating', 'movie.imdb.year']"+BY_PREVIOUS_COMPONENT_PROMPT_SUFFIX,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response one previous component: ", llm_response)
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_RELEASED,
+            "previous_user_prompts": [PROMPT_MOVIE_INFO],
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response multiple previous prompts: ", llm_response)
+    
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_RELEASED,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response no history: ", llm_response)
+
+# ----------------- 3 -----------------
+    print_header("*** 3. Current: Show me the poster only. Previous: Info about Toy Story - the same data entity, but user prompt asks for different UI component  ***")
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_POSTER+BY_PREVIOUS_COMPONENT_PROMPT_PRFIX+"'one-card' with title 'Toy Story' and fields ['movie.title', 'movie.released', 'movie.imdb.rating']"+BY_PREVIOUS_COMPONENT_PROMPT_SUFFIX,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response one previous component: ", llm_response)
+    
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_POSTER,
+            "previous_user_prompts": [PROMPT_MOVIE_INFO],
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response multiple previous prompts: ", llm_response)
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_POSTER,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response no history: ", llm_response)
+
+# ----------------- 4 -----------------
+    print_header("*** 4. Current: When was the movie released? Previous: Poster only, Info about Toy Story, Subscriptions table - multiple previous conversation steps including entity change and then component type change to another and back ***")
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_RELEASED+BY_PREVIOUS_COMPONENT_PROMPT_PRFIX+"'image' with title 'Toy Story Poster' and fields ['movie.pictureUrl']"+BY_PREVIOUS_COMPONENT_PROMPT_SUFFIX,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response one previous component: ", llm_response)
+    
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_RELEASED,
+            "previous_user_prompts": [PROMPT_MOVIE_POSTER, PROMPT_MOVIE_INFO, PROMPT_MOVIE_SUBSCRIPTIONS],
+            #"previous_user_prompts": [PROMPT_MOVIE_POSTER],
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response multiple previous prompts: ", llm_response)
+
+    llm_response = asyncio.run(agent.component_selection(
+        input={
+            "user_prompt": PROMPT_MOVIE_RELEASED,
+            "input_data": [InputData({"id": "1", "data": TOYSTORY_DATA})],
+        },
+    ))
+    
+    print_response("Response no history: ", llm_response)
+