Fix bug in Databricks Data dicationary (#97)

Author: BenConstable9

adi_function_app/mark_up_cleaner.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 import logging
 import json
-import re
+import regex as re
 
 
 def get_sections(text: str) -> list:

deploy_ai_search/src/deploy_ai_search/deploy.py

@@ -15,28 +15,30 @@ def deploy_config(arguments: argparse.Namespace):
 
     Args:
         arguments (argparse.Namespace): The arguments passed to the script"""
+
+    suffix = None if args.suffix == "None" else args.suffix
     if arguments.index_type == "rag":
         index_config = RagDocumentsAISearch(
-            suffix=arguments.suffix,
+            suffix=suffix,
             rebuild=arguments.rebuild,
             enable_page_by_chunking=arguments.enable_page_chunking,
         )
     elif arguments.index_type == "text_2_sql_schema_store":
         index_config = Text2SqlSchemaStoreAISearch(
-            suffix=arguments.suffix,
+            suffix=suffix,
             rebuild=arguments.rebuild,
             single_data_dictionary_file=arguments.single_data_dictionary_file,
         )
     elif arguments.index_type == "text_2_sql_query_cache":
         index_config = Text2SqlQueryCacheAISearch(
-            suffix=arguments.suffix,
+            suffix=suffix,
             rebuild=arguments.rebuild,
             single_query_cache_file=arguments.single_query_cache_file,
             enable_query_cache_indexer=arguments.enable_query_cache_indexer,
         )
     elif arguments.index_type == "text_2_sql_column_value_store":
         index_config = Text2SqlColumnValueStoreAISearch(
-            suffix=arguments.suffix,
+            suffix=suffix,
             rebuild=arguments.rebuild,
         )
     else:

text_2_sql/text_2_sql_core/src/text_2_sql_core/data_dictionary/databricks_data_dictionary_creator.py

@@ -112,7 +112,7 @@ def extract_distinct_values_sql_query(
         Returns:
             str: The SQL query to extract distinct values from a column.
         """
-        return f"""SELECT DISTINCT {column.name} FROM {self.catalog}.{entity.entity} WHERE {column.name} IS NOT NULL ORDER BY {column.name} DESC;"""
+        return f"""SELECT DISTINCT {column.name} FROM {self.catalog}.{entity.entity_schema}.{entity.entity} WHERE {column.name} IS NOT NULL ORDER BY {column.name} DESC;"""
 
 
 if __name__ == "__main__":

text_2_sql/text_2_sql_core/src/text_2_sql_core/prompts/answer_agent.yaml

@@ -14,4 +14,5 @@ system_message:
       {'sql_result_snippet': <SQL QUERY RESULT 2>, 'sql_query_used': '<SOURCE 2 SQL QUERY>', 'explanation': '<EXPLANATION OF SQL QUERY 2>'},
     ]
   }
+
   "