[9.2] [Obs AI Assistant] aware of new .integration_knowledge* system index (#237085) (#239263)

arturoliduena · kibanamachine · neptunian · web-flow · commit 41a47a2ff222 · 2025-10-16T11:21:32.000+02:00
Closes elastic/obs-ai-team#380 # Backport This will backport the following commits from `main` to `9.2`: - [[Obs AI Assistant] aware of new .integration_knowledge* system index (#237085)](#237085)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Sandra G <neptunian@users.noreply.github.com> Co-authored-by: Viduni Wickramarachchi <viduni.ushanka@gmail.com>
diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts
@@ -205,3 +205,12 @@ export enum ConversationAccess {
   SHARED = 'shared',
   PRIVATE = 'private',
 }
+
+export interface IntegrationKnowledgeBaseEntry {
+  content: string;
+  package_name: string;
+  filename: string;
+  version: string;
+  path: string;
+  installed_at: string;
+}
diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts
@@ -13,7 +13,11 @@ import { encode } from 'gpt-tokenizer';
 import { isLockAcquisitionError } from '@kbn/lock-manager';
 import type { DocumentationManagerAPI } from '@kbn/product-doc-base-plugin/server/services/doc_manager';
 import { resourceNames } from '..';
-import type { Instruction, KnowledgeBaseEntry } from '../../../common/types';
+import type {
+  Instruction,
+  IntegrationKnowledgeBaseEntry,
+  KnowledgeBaseEntry,
+} from '../../../common/types';
 import { KnowledgeBaseEntryRole, KnowledgeBaseType } from '../../../common/types';
 import { getAccessQuery, getUserAccessFilters } from '../util/get_access_query';
 import { getCategoryQuery } from '../util/get_category_query';
@@ -36,6 +40,8 @@ import { getInferenceIdFromWriteIndex } from './get_inference_id_from_write_inde
 import { createOrUpdateKnowledgeBaseIndexAssets } from '../index_assets/create_or_update_knowledge_base_index_assets';
 import { LEGACY_CUSTOM_INFERENCE_ID } from '../../../common/preconfigured_inference_ids';
 
+const INTEGRATION_KNOWLEDGE_INDEX = '.integration_knowledge';
+
 interface Dependencies {
   core: CoreSetup<ObservabilityAIAssistantPluginStartDependencies>;
   esClient: {
@@ -112,6 +118,64 @@ export class KnowledgeBaseService {
     }));
   }
 
+  private async recallFromIntegrationsKnowledge({
+    queries,
+    esClient,
+  }: {
+    queries: Array<{ text: string; boost?: number }>;
+    esClient: { asCurrentUser: ElasticsearchClient; asInternalUser: ElasticsearchClient };
+  }): Promise<RecalledEntry[]> {
+    // Check if the .integration_knowledge index exists before searching for documents
+    // This has to be done with `.search` since `.exists` and `.get` can't be performed
+    // with the internal system user (lack of permissions)
+    try {
+      await esClient.asInternalUser.search({
+        index: INTEGRATION_KNOWLEDGE_INDEX,
+        size: 0,
+      });
+    } catch (error) {
+      // If there's an error checking the index existence, assume it doesn't exist and don't attempt to recall
+      this.dependencies.logger.debug(
+        `Failed to access the index: "${INTEGRATION_KNOWLEDGE_INDEX}". Skipping integration knowledge recall.`
+      );
+      this.dependencies.logger.debug(error);
+      return [];
+    }
+
+    try {
+      // Search the .integration_knowledge index using semantic search on the content field
+      const response = await esClient.asInternalUser.search<IntegrationKnowledgeBaseEntry>({
+        index: INTEGRATION_KNOWLEDGE_INDEX,
+        query: {
+          bool: {
+            should: queries.map(({ text, boost = 1 }) => ({
+              semantic: {
+                field: 'content',
+                query: text,
+                boost,
+              },
+            })),
+          },
+        },
+        size: 10,
+        _source: ['package_name', 'filename', 'content', 'version'],
+      });
+      return response.hits.hits.map((hit) => ({
+        text: hit._source?.content!,
+        labels: { filename: hit._source?.filename ?? '', version: hit._source?.version ?? '' },
+        title: hit._source?.package_name,
+        esScore: hit._score!,
+        id: hit._id!,
+      }));
+    } catch (error) {
+      this.dependencies.logger.error(
+        `Error recalling from index "${INTEGRATION_KNOWLEDGE_INDEX}": ${error?.message}`
+      );
+      this.dependencies.logger.debug(error);
+      return [];
+    }
+  }
+
   recall = async ({
     user,
     queries,
@@ -137,40 +201,55 @@ export class KnowledgeBaseService {
       () => `Recalling entries from KB for queries: "${JSON.stringify(queries)}"`
     );
 
-    const [documentsFromKb, documentsFromConnectors] = await Promise.all([
-      this.recallFromKnowledgeBase({
-        user,
-        queries,
-        categories,
-        namespace,
-      }).catch((error) => {
-        if (isInferenceEndpointMissingOrUnavailable(error)) {
-          throwKnowledgeBaseNotReady(error);
-        }
-        throw error;
-      }),
-      recallFromSearchConnectors({
-        esClient,
-        uiSettingsClient,
-        queries,
-        core: this.dependencies.core,
-        logger: this.dependencies.logger,
-      }).catch((error) => {
-        this.dependencies.logger.error('Error getting data from search indices');
-        this.dependencies.logger.debug(error);
-        return [];
-      }),
-    ]);
+    const [documentsFromKb, documentsFromConnectors, documentsFromIntegrations] = await Promise.all(
+      [
+        this.recallFromKnowledgeBase({
+          user,
+          queries,
+          categories,
+          namespace,
+        }).catch((error) => {
+          if (isInferenceEndpointMissingOrUnavailable(error)) {
+            throwKnowledgeBaseNotReady(error);
+          }
+          throw error;
+        }),
+        recallFromSearchConnectors({
+          esClient,
+          uiSettingsClient,
+          queries,
+          core: this.dependencies.core,
+          logger: this.dependencies.logger,
+        }).catch((error) => {
+          this.dependencies.logger.error('Error getting data from search indices');
+          this.dependencies.logger.debug(error);
+          return [];
+        }),
+        this.recallFromIntegrationsKnowledge({
+          esClient,
+          queries,
+        }).catch((error) => {
+          this.dependencies.logger.error(
+            `Error getting data from ${INTEGRATION_KNOWLEDGE_INDEX} index`
+          );
+          this.dependencies.logger.debug(error);
+          return [];
+        }),
+      ]
+    );
 
     this.dependencies.logger.debug(
       `documentsFromKb: ${JSON.stringify(documentsFromKb.slice(0, 5), null, 2)}`
     );
     this.dependencies.logger.debug(
       `documentsFromConnectors: ${JSON.stringify(documentsFromConnectors.slice(0, 5), null, 2)}`
     );
+    this.dependencies.logger.debug(
+      `documentsFromIntegrations: ${JSON.stringify(documentsFromIntegrations.slice(0, 5), null, 2)}`
+    );
 
     const sortedEntries = orderBy(
-      documentsFromKb.concat(documentsFromConnectors),
+      [...documentsFromKb, ...documentsFromConnectors, ...documentsFromIntegrations],
       'esScore',
       'desc'
     ).slice(0, limit.size ?? 20);
