Update rag examples

Author: BenConstable9

deploy_ai_search/src/deploy_ai_search/.env.example

@@ -0,0 +1,27 @@
+
+FunctionApp__Endpoint=<FunctionAppEndpoint>
+FunctionApp__Key=<FunctionAppKey>
+FunctionApp__LayoutAnalysis__FunctionName=layout_analysis
+FunctionApp__FigureAnalysis__FunctionName=figure_analysis
+FunctionApp__LayoutAndFigureMerger__FunctionName=layout_and_figure_merger
+FunctionApp__MarkUpCleaner__FunctionName=mark_up_cleaner
+FunctionApp__SemanticTextChunker__FunctionName=semantic_text_chunker
+FunctionApp__AppRegistrationResourceId=<App registration in form api://appRegistrationclientId if using identity based connections>
+IdentityType=key # system_assigned or user_assigned or key
+AIService__AzureSearchOptions__Endpoint=<AzureSearchEndpoint>
+AIService__AzureSearchOptions__Identity__ClientId=<clientId if using user assigned identity>
+AIService__AzureSearchOptions__Key=<AzureSearchKey>
+AIService__AzureSearchOptions__UsePrivateEndpoint=false
+AIService__AzureSearchOptions__Identity__FQName=<fully qualified name of the identity if using user assigned identity>
+StorageAccount__FQEndpoint=<Fully qualified endpoint in form ResourceId=resourceId if using identity based connections>
+StorageAccount__ConnectionString=<StorageAccountConnectionString>
+OpenAI__ApiKey=<OpenAIKey>
+OpenAI__Endpoint=<OpenAIEndpoint>
+OpenAI__EmbeddingModel="text-embedding-ada-002"
+OpenAI__EmbeddingDeployment="text-embedding-ada-002"
+OpenAI__EmbeddingDimensions=1536
+Text2Sql__DatabaseEngine=<DatabaseEngine>
+StorageAccount__RagDocuments__Container=<RagDocumentsContainer>
+StorageAccount__Text2SqlQueryCache__Container=<Text2SqlQueryCacheContainer>
+StorageAccount__Text2SqlSchemaStore__Container=<Text2SqlSchemaStoreContainer>
+StorageAccount__Text2SqlColumnValueStore__Container=<Text2SqlColumnValueStoreContainer>

deploy_ai_search/src/deploy_ai_search/ai_search.py

@@ -253,7 +253,7 @@ def get_mark_up_cleaner_skill(self, context, source) -> WebApiSkill:
 
         return mark_up_cleaner_skill
 
-    def get_text_split_skill(
+    def get_semantic_chunker_skill(
         self,
         context,
         source,
@@ -292,7 +292,7 @@ def get_text_split_skill(
         ]
 
         semantic_text_chunker_skill = WebApiSkill(
-            name="Mark Up Cleaner Skill",
+            name="Semantic Chunker Skill",
             description="Skill to clean the data before sending to embedding",
             context=context,
             uri=self.environment.get_custom_skill_function_url("semantic_text_chunker"),

deploy_ai_search/src/deploy_ai_search/environment.py

@@ -210,7 +210,7 @@ def function_app_mark_up_cleaner_route(self) -> str:
         """
         This function returns function app data cleanup function name
         """
-        return os.environ.get("FunctionApp__PreEmbeddingCleaner__FunctionName")
+        return os.environ.get("FunctionApp__MarkUpCleaner__FunctionName")
 
     @property
     def function_app_semantic_text_chunker_route(self) -> str:
@@ -220,18 +220,25 @@ def function_app_semantic_text_chunker_route(self) -> str:
         return os.environ.get("FunctionApp__SemanticTextChunker__FunctionName")
 
     @property
-    def function_app_adi_route(self) -> str:
+    def function_app_layout_analysis_route(self) -> str:
         """
         This function returns function app adi name
         """
-        return os.environ.get("FunctionApp__ADI__FunctionName")
+        return os.environ.get("FunctionApp__LayoutAnalysis__FunctionName")
 
     @property
-    def function_app_key_phrase_extractor_route(self) -> str:
+    def function_app_figure_analysis_route(self) -> str:
         """
-        This function returns function app keyphrase extractor name
+        This function returns function app figure analysis name
         """
-        return os.environ.get("FunctionApp__KeyPhraseExtractor__FunctionName")
+        return os.environ.get("FunctionApp__FigureAnalysis__FunctionName")
+
+    @property
+    def function_app_layout_and_figure_merger_route(self) -> str:
+        """
+        This function returns function app layout and figure merger name
+        """
+        return os.environ.get("FunctionApp__LayoutAndFigureMerger__FunctionName")
 
     @property
     def open_ai_embedding_dimensions(self) -> str:
@@ -260,10 +267,12 @@ def get_custom_skill_function_url(self, skill_type: str):
         """
         if skill_type == "mark_up_cleaner":
             route = self.function_app_mark_up_cleaner_route
-        elif skill_type == "adi":
-            route = self.function_app_adi_route
-        elif skill_type == "key_phrase_extraction":
-            route = self.function_app_key_phrase_extractor_route
+        elif skill_type == "layout_analysis":
+            route = self.function_app_layout_analysis_route
+        elif skill_type == "figure_analysis":
+            route = self.function_app_figure_analysis
+        elif skill_type == "layout_and_figure_merger":
+            route = self.function_app_layout_and_figure_merger
         elif skill_type == "semantic_text_chunker":
             route = self.function_app_semantic_text_chunker_route
         else:

deploy_ai_search/src/deploy_ai_search/rag_documents.py

@@ -179,9 +179,13 @@ def get_skills(self) -> list:
         Returns:
             list: The skillsets  used in the indexer"""
 
-        adi_skill = self.get_adi_skill(self.enable_page_by_chunking)
+        layout_skill = self.get_layout_analysis_skill(self.enable_page_by_chunking)
 
-        text_split_skill = self.get_text_split_skill(
+        figure_skill = self.get_figure_analysis_skill()
+
+        merger_skill = self.get_layout_and_figure_merger_skill()
+
+        text_split_skill = self.get_semantic_chunker_skill(
             "/document", "/document/extracted_content/content"
         )
 
@@ -195,13 +199,17 @@ def get_skills(self) -> list:
 
         if self.enable_page_by_chunking:
             skills = [
-                adi_skill,
+                layout_skill,
+                figure_skill,
+                merger_skill,
                 mark_up_cleaner_skill,
                 embedding_skill,
             ]
         else:
             skills = [
-                adi_skill,
+                layout_skill,
+                figure_skill,
+                merger_skill,
                 text_split_skill,
                 mark_up_cleaner_skill,
                 embedding_skill,