Clean up SK code (#69)

Author: BenConstable9

…l/rag_with_prompt_based_text_2_sql.ipynb …teration 2 - Prompt Based Text2SQL.ipynbtext_2_sql/semantic_kernel/rag_with_prompt_based_text_2_sql.ipynb renamed to text_2_sql/semantic_kernel/Iteration 2 - Prompt Based Text2SQL.ipynb text_2_sql/semantic_kernel/rag_with_prompt_based_text_2_sql.ipynb renamed to text_2_sql/semantic_kernel/Iteration 2 - Prompt Based Text2SQL.ipynb

@@ -239,15 +239,14 @@
         "\n",
         "    # Create important information prompt that contains the SQL database information.\n",
         "    engine_specific_rules = \"Use TOP X to limit the number of rows returned instead of LIMIT X. NEVER USE LIMIT X as it produces a syntax error.\"\n",
-        "    important_information_prompt = f\"\"\"\n",
+        "    sql_database_information_prompt = f\"\"\"\n",
         "    [SQL DATABASE INFORMATION]\n",
-        "    {sql_plugin.system_prompt(engine_specific_rules=engine_specific_rules)}\n",
+        "    {sql_plugin.sql_prompt_injection(engine_specific_rules=engine_specific_rules)}\n",
         "    [END SQL DATABASE INFORMATION]\n",
         "    \"\"\"\n",
         "\n",
         "    arguments = KernelArguments()\n",
-        "    arguments[\"chat_history\"] = chat_history\n",
-        "    arguments[\"important_information\"] = important_information_prompt\n",
+        "    arguments[\"sql_database_information\"] = sql_database_information_prompt\n",
         "    arguments[\"user_input\"] = question\n",
         "\n",
         "    logging.info(\"Question: %s\", question)\n",

…l/rag_with_vector_based_text_2_sql.ipynb …ions 3 & 4 - Vector Based Text2SQL.ipynbtext_2_sql/semantic_kernel/rag_with_vector_based_text_2_sql.ipynb renamed to text_2_sql/semantic_kernel/Iterations 3 & 4 - Vector Based Text2SQL.ipynb text_2_sql/semantic_kernel/rag_with_vector_based_text_2_sql.ipynb renamed to text_2_sql/semantic_kernel/Iterations 3 & 4 - Vector Based Text2SQL.ipynb

@@ -227,15 +227,14 @@
         "\n",
         "    # Create important information prompt that contains the SQL database information.\n",
         "    engine_specific_rules = \"Use TOP X to limit the number of rows returned instead of LIMIT X. NEVER USE LIMIT X as it produces a syntax error.\"\n",
-        "    important_information_prompt = f\"\"\"\n",
+        "    sql_database_information_prompt = f\"\"\"\n",
         "    [SQL DATABASE INFORMATION]\n",
-        "    {await sql_plugin.system_prompt(engine_specific_rules=engine_specific_rules, question=question)}\n",
+        "    {await sql_plugin.sql_prompt_injection(engine_specific_rules=engine_specific_rules, question=question)}\n",
         "    [END SQL DATABASE INFORMATION]\n",
         "    \"\"\"\n",
         "\n",
         "    arguments = KernelArguments()\n",
-        "    arguments[\"chat_history\"] = chat_history\n",
-        "    arguments[\"important_information\"] = important_information_prompt\n",
+        "    arguments[\"sql_database_information\"] = sql_database_information_prompt\n",
         "    arguments[\"user_input\"] = question\n",
         "\n",
         "    logging.info(\"Question: %s\", question)\n",

text_2_sql/semantic_kernel/README.md

@@ -16,10 +16,8 @@ As the query cache is shared between users (no data is stored in the cache), a n
 
 ## Provided Notebooks & Scripts
 
-- `./rag_with_prompt_based_text_2_sql.ipynb` provides example of how to utilise the Prompt Based Text2SQL plugin to query the database.
-- `./rag_with_vector_based_text_2_sql.ipynb` provides example of how to utilise the Vector Based Text2SQL plugin to query the database. The query cache plugin will be enabled or disabled depending on the environmental parameters.
-- `./rag_with_ai_search_and_text_2_sql.ipynb` provides an example of how to use the Text2SQL and an AISearch plugin in parallel to automatically retrieve data from the most relevant source to answer the query.
-    - This setup is useful for a production application as the SQL Database is unlikely to be able to answer all the questions a user may ask.
+- `./Iteration 2 - Prompt Based Text2SQL.ipynb` provides example of how to utilise the Prompt Based Text2SQL plugin to query the database.
+- `./Iterations 3 & 4 - Vector Based Text2SQL.ipynb` provides example of how to utilise the Vector Based Text2SQL plugin to query the database. The query cache plugin will be enabled or disabled depending on the environmental parameters.
 - `./time_comparison_script.py` provides a utility script for performing time based comparisons between the different approaches.
 
 ### ai-search.py

text_2_sql/semantic_kernel/plugins/ai_search_plugin/__init__.py

@@ -37,7 +37,7 @@ def load_entities(self):
                 entity_name = entity_object["EntityName"].lower()
                 self.entities[entity_name] = entity_object
 
-    def system_prompt(self, engine_specific_rules: str | None = None) -> str:
+    def sql_prompt_injection(self, engine_specific_rules: str | None = None) -> str:
         """Get the schemas for the database entities and provide a system prompt for the user.
 
         Returns:
@@ -60,7 +60,7 @@ def system_prompt(self, engine_specific_rules: str | None = None) -> str:
             engine_specific_rules = f"""\n        The following {
                 self.target_engine} Syntax rules must be adhered to.\n        {engine_specific_rules}"""
 
-        system_prompt = f"""Use the names and descriptions of {self.target_engine} entities provided in ENTITIES LIST to decide which entities to query if you need to retrieve information from the database. Use the 'GetEntitySchema()' function to get more details of the schema of the view you want to query.
+        sql_prompt_injection = f"""Use the names and descriptions of {self.target_engine} entities provided in ENTITIES LIST to decide which entities to query if you need to retrieve information from the database. Use the 'GetEntitySchema()' function to get more details of the schema of the view you want to query.
 
         Always then use the 'RunSQLQuery()' function to run the SQL query against the database. Never just return the SQL query as the answer.
 
@@ -86,7 +86,7 @@ def system_prompt(self, engine_specific_rules: str | None = None) -> str:
 
         The source title to cite is the 'entity_name' property. The source reference is the SQL query used. The source chunk is the result of the SQL query used to answer the user query in Markdown table format. e.g. {{ 'title': "vProductAndDescription", 'chunk': '| ProductID | Name              | ProductModel | Culture | Description                      |\\n|-----------|-------------------|--------------|---------|----------------------------------|\\n| 101       | Mountain Bike     | MT-100       | en      | A durable bike for mountain use. |\\n| 102       | Road Bike         | RB-200       | en      | Lightweight bike for road use.   |\\n| 103       | Hybrid Bike       | HB-300       | fr      | Vélo hybride pour usage mixte.   |\\n', 'reference': 'SELECT ProductID, Name, ProductModel, Culture, Description FROM vProductAndDescription WHERE Culture = \"en\";' }}"""
 
-        return system_prompt
+        return sql_prompt_injection
 
     @kernel_function(
         description="Get the detailed schema of an entity in the Database. Use the entity and the column returned to formulate a SQL query. The view name or table name must be one of the ENTITY NAMES defined in the [ENTITIES LIST]. Only use the column names obtained from GetEntitySchema() when constructing a SQL query, do not make up column names.",

text_2_sql/semantic_kernel/plugins/ai_search_plugin/ai_search_plugin.py

@@ -211,7 +211,7 @@ async def fetch_queries_from_cache(self, question: str) -> str:
 
         return formatted_sql_cache_string
 
-    async def system_prompt(
+    async def sql_prompt_injection(
         self, engine_specific_rules: str | None = None, question: str | None = None
     ) -> str:
         """Get the schemas for the database entities and provide a system prompt for the user.
@@ -261,7 +261,7 @@ async def system_prompt(
 
             Check the above schemas carefully to see if they can be used to formulate a SQL query. If you need additional schemas, use 'GetEntitySchema()' function to search for the most relevant schemas for the data that you wish to obtain."""
 
-        system_prompt = f"""{query_prompt}
+        sql_prompt_injection = f"""{query_prompt}
 
         If needed, use the 'RunSQLQuery()' function to run the SQL query against the database. Never just return the SQL query as the answer.
 
@@ -280,7 +280,7 @@ async def system_prompt(
 
         The source title to cite is the 'EntityName' property. The source reference is the SQL query used. The source chunk is the result of the SQL query used to answer the user query in Markdown table format. e.g. {{ 'title': "vProductAndDescription", 'chunk': '| ProductID | Name              | ProductModel | Culture | Description                      |\\n|-----------|-------------------|--------------|---------|----------------------------------|\\n| 101       | Mountain Bike     | MT-100       | en      | A durable bike for mountain use. |\\n| 102       | Road Bike         | RB-200       | en      | Lightweight bike for road use.   |\\n| 103       | Hybrid Bike       | HB-300       | fr      | Vélo hybride pour usage mixte.   |\\n', 'reference': 'SELECT ProductID, Name, ProductModel, Culture, Description FROM vProductAndDescription WHERE Culture = \"en\";' }}"""
 
-        return system_prompt
+        return sql_prompt_injection
 
     @kernel_function(
         description="Gets the schema of a view or table in the SQL Database by selecting the most relevant entity based on the search term. Extract key terms from the user question and use these as the search term. Several entities may be returned. Only use when the provided schemas in the system prompt are not sufficient to answer the question.",

text_2_sql/semantic_kernel/plugins/prompt_based_sql_plugin/prompt_based_sql_plugin.py

@@ -12,47 +12,35 @@ template: |
 
   The response must meet the following requirements:
 
-  [RESPONSE OUTPUT REQUIREMENTS]
+  The answer MUST be in JSON format:
+  {
+    "answer": "<GENERATED ANSWER>",
+    "sources": [
+      {"title": <SOURCE 1 TITLE>, "chunk": <SOURCE 1 CONTEXT CHUNK>, "reference": "<SOURCE 1 REFERENCE>"},
+      {"title": <SOURCE 2 TITLE>, "chunk": <SOURCE 2 CONTEXT CHUNK>, "reference": "<SOURCE 2 REFERENCE>"}
+    ]
+  }
 
-    The answer MUST be in JSON format:
-    {
-      "answer": "<GENERATED ANSWER>",
-      "sources": [
-        {"title": <SOURCE 1 TITLE>, "chunk": <SOURCE 1 CONTEXT CHUNK>, "reference": "<SOURCE 1 REFERENCE>"},
-        {"title": <SOURCE 2 TITLE>, "chunk": <SOURCE 2 CONTEXT CHUNK>, "reference": "<SOURCE 2 REFERENCE>"}
-      ]
-    }
+  [ANSWER PROPERTY REQUIREMENTS]
+    - **Calculations**:
+      Use context-provided values and explain calculations briefly.
+    - **Structure**:
+      Responses must be direct, easy to understand, and formatted using Markdown.
+      Use Level 3 and 4 headings, bold sub-headings, and lists where appropriate. Keep font size consistent.
+    - **Citations**:
+      Factual statements must be cited using numbered references like [1]. Each citation must match a source in the 'sources' object.
 
-    [ANSWER PROPERTY REQUIREMENTS]
-      - **Language & Tone**:
-        Use British English, business-friendly language that is professional and clear.
-      - **Content Restrictions**:
-        Avoid profanity, offensive language, and code. Rephrase or omit inappropriate content.
-      - **Information Sources**:
-        Only use provided functions and important information. Prioritize SQL Database data in case of conflicts.
-      - **Calculations**:
-        Use context-provided values and explain calculations briefly.
-      - **Structure**:
-        Responses must be direct, easy to understand, and formatted using Markdown.
-        Use Level 3 and 4 headings, bold sub-headings, and lists where appropriate. Keep font size consistent.
-      - **Citations**:
-        Factual statements must be cited using numbered references like [1]. Each citation must match a source in the 'sources' object.
+  [SOURCES PROPERTY REQUIREMENTS]
+    - **Reference Inclusion**:
+      All cited content must have a corresponding reference in the 'sources' object.
+    - **Source Format**:
+      Each source must follow this format: {"title": "<SOURCE TITLE>", "chunk": "<SOURCE CONTEXT CHUNK>", "reference": "<SOURCE REFERENCE>"}
+    - **Source Chunk**:
+      Include a concise, unedited snippet of relevant context in the 'chunk' property.
+    - **Mandatory Citations**:
+      Every source listed must be cited at least once in the answer.
 
-    [SOURCES PROPERTY REQUIREMENTS]
-      - **Reference Inclusion**:
-        All cited content must have a corresponding reference in the 'sources' object.
-      - **Source Format**:
-        Each source must follow this format: {"title": "<SOURCE TITLE>", "chunk": "<SOURCE CONTEXT CHUNK>", "reference": "<SOURCE REFERENCE>"}
-      - **Source Chunk**:
-        Include a concise, unedited snippet of relevant context in the 'chunk' property.
-      - **Mandatory Citations**:
-        Every source listed must be cited at least once in the answer.
-
-  [IMPORTANT INFORMATION]
-
-  {{$important_information}}
-
-  [END]
+  {{$sql_database_information}}
   </message>
   {{$chat_history}}
   <message role="user">{{$user_input}}</message>
@@ -62,7 +50,7 @@ input_variables:
   - name: user_input
     description: The user input
     is_required: true
-  - name: important_information
+  - name: sql_database_information
     description: Useful information for the chatbot
     is_required: true
 output_variable: