Structured output branch

Author: BenConstable9

image_processing/src/image_processing/requirements.txt

@@ -5,7 +5,7 @@ aiohttp==3.11.11
 aiosignal==1.3.2
 annotated-types==0.7.0
 anyio==4.8.0
-attrs==24.3.0
+attrs==25.1.0
 azure-ai-documentintelligence==1.0.0
 azure-ai-textanalytics==5.3.0
 azure-ai-vision-imageanalysis==1.0.0
@@ -15,7 +15,7 @@ azure-functions==1.21.3
 azure-identity==1.19.0
 azure-search==1.0.0b2
 azure-search-documents==11.6.0b8
-azure-storage-blob==12.24.0
+azure-storage-blob==12.24.1
 beautifulsoup4==4.12.3
 blis==0.7.11
 bs4==0.0.2
@@ -38,7 +38,7 @@ fsspec==2024.12.0
 h11==0.14.0
 httpcore==1.0.7
 httpx==0.28.1
-huggingface-hub==0.27.1
+huggingface-hub==0.28.0
 idna==3.10
 isodate==0.7.2
 jinja2==3.1.5
@@ -50,15 +50,15 @@ marisa-trie==1.2.1
 markdown-it-py==3.0.0
 markupsafe==3.0.2
 mdurl==0.1.2
-model2vec==0.3.7
+model2vec==0.3.8
 msal==1.31.1
 msal-extensions==1.2.0
 msrest==0.7.1
 multidict==6.1.0
 murmurhash==1.0.12
 numpy==1.26.4
 oauthlib==3.2.2
-openai==1.60.0
+openai==1.60.2
 openpyxl==3.1.5
 packaging==24.2
 pandas==2.2.3
@@ -67,7 +67,7 @@ portalocker==2.10.1
 preshed==3.0.9
 propcache==0.2.1
 pycparser==2.22 ; platform_python_implementation != 'PyPy'
-pydantic==2.10.5
+pydantic==2.10.6
 pydantic-core==2.27.2
 pygments==2.19.1
 pyjwt==2.10.1

text_2_sql/.env.example

@@ -5,6 +5,7 @@ Text2Sql__DatabaseEngine=<DatabaseEngine> # TSQL or PostgreSQL or Snowflake or D
 Text2Sql__UseQueryCache=<Determines if the Query Cache will be used to speed up query generation. Defaults to True.> # True or False
 Text2Sql__PreRunQueryCache=<Determines if the results from the Query Cache will be pre-run to speed up answer generation. Defaults to True.> # True or False
 Text2Sql__UseColumnValueStore=<Determines if the Column Value Store will be used for schema selection Defaults to True.> # True or False
+Text2Sql__GenerateFollowUpQuestions=<Determines if follow up questions will be generated. Defaults to True.> # True or False
 
 # Open AI Connection Details
 OpenAI__CompletionDeployment=<openAICompletionDeploymentId. Used for data dictionary creator>

text_2_sql/autogen/pyproject.toml

@@ -9,9 +9,9 @@ authors = [
 requires-python = ">=3.11"
 dependencies = [
     "aiostream>=0.6.4",
-    "autogen-agentchat==0.4.2",
-    "autogen-core==0.4.2",
-    "autogen-ext[azure,openai]==0.4.2",
+    "autogen-agentchat==0.4.3",
+    "autogen-core==0.4.3",
+    "autogen-ext[azure,openai]==0.4.3",
     "grpcio>=1.68.1",
     "pyyaml>=6.0.2",
     "text_2_sql_core",

text_2_sql/autogen/src/autogen_text_2_sql/creators/llm_agent_creator.py

@@ -7,6 +7,10 @@
 from autogen_text_2_sql.creators.llm_model_creator import LLMModelCreator
 from jinja2 import Template
 import logging
+from text_2_sql_core.structured_outputs import (
+    AnswerAgentWithFollowUpQuestionsAgentOutput,
+    UserMessageRewriteAgentOutput,
+)
 
 
 class LLMAgentCreator:
@@ -106,10 +110,20 @@ def create(cls, name: str, **kwargs) -> AssistantAgent:
             for tool in agent_file["tools"]:
                 tools.append(cls.get_tool(sql_helper, tool))
 
+        structured_output = None
+        if agent_file.get("structured_output", False):
+            # Import the structured output agent
+            if name == "answer_agent_with_follow_up_questions":
+                structured_output = AnswerAgentWithFollowUpQuestionsAgentOutput
+            elif name == "user_message_rewrite_agent":
+                structured_output = UserMessageRewriteAgentOutput
+
         agent = AssistantAgent(
             name=name,
             tools=tools,
-            model_client=LLMModelCreator.get_model(agent_file["model"]),
+            model_client=LLMModelCreator.get_model(
+                agent_file["model"], structured_output=structured_output
+            ),
             description=cls.get_property_and_render_parameters(
                 agent_file, "description", kwargs
             ),

text_2_sql/autogen/src/autogen_text_2_sql/creators/llm_model_creator.py

@@ -12,7 +12,9 @@
 
 class LLMModelCreator:
     @classmethod
-    def get_model(cls, model_name: str) -> AzureOpenAIChatCompletionClient:
+    def get_model(
+        cls, model_name: str, structured_output=None
+    ) -> AzureOpenAIChatCompletionClient:
         """Retrieves the model based on the model name.
 
         Args:
@@ -22,9 +24,9 @@ def get_model(cls, model_name: str) -> AzureOpenAIChatCompletionClient:
         Returns:
             AzureOpenAIChatCompletionClient: The model client."""
         if model_name == "4o-mini":
-            return cls.gpt_4o_mini_model()
+            return cls.gpt_4o_mini_model(structured_output=structured_output)
         elif model_name == "4o":
-            return cls.gpt_4o_model()
+            return cls.gpt_4o_model(structured_output=structured_output)
         else:
             raise ValueError(f"Model {model_name} not found")
 
@@ -46,7 +48,9 @@ def get_authentication_properties(cls) -> dict:
         return token_provider, api_key
 
     @classmethod
-    def gpt_4o_mini_model(cls) -> AzureOpenAIChatCompletionClient:
+    def gpt_4o_mini_model(
+        cls, structured_output=None
+    ) -> AzureOpenAIChatCompletionClient:
         token_provider, api_key = cls.get_authentication_properties()
         return AzureOpenAIChatCompletionClient(
             azure_deployment=os.environ["OpenAI__MiniCompletionDeployment"],
@@ -61,10 +65,11 @@ def gpt_4o_mini_model(cls) -> AzureOpenAIChatCompletionClient:
                 "json_output": True,
             },
             temperature=0,
+            response_format=structured_output,
         )
 
     @classmethod
-    def gpt_4o_model(cls) -> AzureOpenAIChatCompletionClient:
+    def gpt_4o_model(cls, structured_output=None) -> AzureOpenAIChatCompletionClient:
         token_provider, api_key = cls.get_authentication_properties()
         return AzureOpenAIChatCompletionClient(
             azure_deployment=os.environ["OpenAI__CompletionDeployment"],
@@ -79,4 +84,5 @@ def gpt_4o_model(cls) -> AzureOpenAIChatCompletionClient:
                 "json_output": True,
             },
             temperature=0,
+            response_format=structured_output,
         )

text_2_sql/text_2_sql_core/src/text_2_sql_core/payloads/interaction_payloads.py

@@ -86,6 +86,9 @@ class Source(InteractionPayloadBase):
             default_factory=list, alias="decomposedUserMessages"
         )
         sources: list[Source] = Field(default_factory=list)
+        follow_up_questions: list[str] | None = Field(
+            default=None, alias="followUpQuestions"
+        )
         assistant_state: dict | None = Field(default=None, alias="assistantState")
 
     payload_type: Literal[PayloadType.ANSWER_WITH_SOURCES] = Field(

text_2_sql/text_2_sql_core/src/text_2_sql_core/prompts/answer_agent_with_follow_up_questions.yaml

@@ -0,0 +1,34 @@
+model: "4o-mini"
+description: "An agent that generates a response to a user's question."
+system_message: |
+  <role_and_objective>
+    You are Senior Data Analystm, specializing in providing data driven answers to a user's question. Use the general business use case of '{{ use_case }}' to aid understanding of the user's question. You should provide a clear and concise response based on the information obtained from the SQL queries and their results. Adopt a data-driven approach to generate the response.
+  </role_and_objective>
+
+  <system_information>
+    You are part of an overall system that provides Text2SQL and subsequent data analysis functionality only. You will be passed a result from multiple SQL queries, you must formulate a response to the user's question using this information.
+    You can assume that the SQL queries are correct and that the results are accurate.
+    You and the wider system can only generate SQL queries and process the results of these queries. You cannot access any external resources.
+    The main ability of the system is to perform natural language understanding and generate SQL queries from the user's question. These queries are then automatically run against the database and the results are passed to you.
+  </system_information>
+
+  <instructions>
+
+    Use the information obtained to generate a response to the user's question. The question has been broken down into a series of SQL queries and you need to generate a response based on the results of these queries.
+
+    Do not use any external resources to generate the response. The response should be based solely on the information provided in the SQL queries and their results.
+
+    You have no access to the internet or any other external resources. You can only use the information provided in the SQL queries and their results, to generate the response.
+
+    You can use Markdown and Markdown tables to format the response. You MUST use the information obtained from the SQL queries to generate the response.
+
+    If the user is asking about your capabilities, use the <system_information> to explain what you do.
+
+    Make sure your response directly addresses every part of the user's question.
+
+    Finally, generate 3 data driven follow-up questions based on the information obtained from the SQL queries and their results. Think carefully about what questions may arise from the data and how they can be used to further analyze the data.
+
+  </instructions>
+
+  <output_
+structured_output: true

text_2_sql/text_2_sql_core/src/text_2_sql_core/prompts/sql_schema_selection_agent.yaml

@@ -96,3 +96,4 @@ system_message: |
   <key_relationships>
     {{ relationship_paths }}
   </key_relationships>
+structured_output: true

text_2_sql/text_2_sql_core/src/text_2_sql_core/prompts/user_message_rewrite_agent.yaml

@@ -175,3 +175,4 @@ system_message: |
        - Multiple queries get comparable data
        - Final step compares results
   </combination_patterns>
+structured_output: true

text_2_sql/text_2_sql_core/src/text_2_sql_core/structured_outputs/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+from text_2_sql_core.structured_outputs.sql_schema_selection_agent import (
+    SQLSchemaSelectionAgentOutput,
+)
+from text_2_sql_core.structured_outputs.user_message_rewrite_agent import (
+    UserMessageRewriteAgentOutput,
+)
+from text_2_sql_core.structured_outputs.answer_agent_with_follow_up_questions import (
+    AnswerAgentWithFollowUpQuestionsAgentOutput,
+)
+
+__all__ = [
+    "AnswerAgentWithFollowUpQuestionsAgentOutput",
+    "SQLSchemaSelectionAgentOutput",
+    "UserMessageRewriteAgentOutput",
+]