[Fleet] Add integration_knowledge system index for Fleet (#132506)

* added privilege and test

* [CI] Auto commit changes from spotless

* changed privileges

* removed conflict i missed

* [CI] Auto commit changes from spotless

* updated test

* rolled back test to be correct to investigate failure

* added additional indexAbstraction function with actual type to avoid adding the wrong privileges

* changed test to true for now due to BWC logic

* [CI] Auto commit changes from spotless

* changed to let index management be the responsibility of ES

* Removed unneeded function

* [CI] Auto commit changes from spotless

* fixed formatting of variable in json

* Update x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java

Co-authored-by: Jen Huang <[email protected]>

* Update x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java

Co-authored-by: Jen Huang <[email protected]>

* added changelog and removed whitespace

* actually remove whitespace

* [CI] Auto commit changes from spotless

* changed area in changelog

* fixed indicies IT to use correct url

* fixed mapping issues by updating plugins needed to use types, also fixed tests to assert on the correct field

* changed index to externally mananged, and removed permissive warnings

* [CI] Auto commit changes from spotless

* made changes to support semantic search with the .md content being added in content field

---------

Co-authored-by: elasticsearchmachine <[email protected]>
Co-authored-by: Jen Huang <[email protected]>

Author: 3 people

docs/changelog/132506.yaml

@@ -0,0 +1,5 @@
+pr: 132506
+summary: Add .integration_knowledge system index for usage by AI assistants
+area: Infra/Core
+type: feature
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java

@@ -174,6 +174,14 @@ static RoleDescriptor kibanaSystem(String name) {
                     .privileges("write", "delete", "create_index")
                     .allowRestrictedIndices(true)
                     .build(),
+                // Integrations knowledge base: Fleet creates, manages, and uses this index to store knowledge base documents to be consumed
+                // by AI assistants to support integrations
+                // assistants.
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".integration_knowledge*")
+                    .privileges("read", "write", "create_index")
+                    .allowRestrictedIndices(true)
+                    .build(),
                 // Other Fleet indices. Kibana reads and writes to these indices to manage
                 // Elastic Agents.
                 RoleDescriptor.IndicesPrivileges.builder()

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java

@@ -103,7 +103,8 @@ public class TestRestrictedIndices {
                     SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-[0-9]+*", "fleet policies"),
                     SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-leader*", "fleet policies leader"),
                     SystemIndexDescriptorUtils.createUnmanaged(".fleet-servers*", "fleet servers"),
-                    SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts")
+                    SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts"),
+                    SystemIndexDescriptorUtils.createUnmanaged(".integration_knowledge*", "fleet integration knowledge base")
                 ),
                 List.of(
                     new SystemDataStreamDescriptor(

x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json

@@ -0,0 +1,38 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1",
+    "index.hidden": true
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}",
+        "managed_index_mappings_version": ${fleet.managed.index.version},
+        "description": "Integration package knowledge base content storage"
+      },
+      "properties": {
+        "filename": {
+          "type": "keyword"
+        },
+        "content": {
+          "type": "semantic_text",
+          "inference_id": ".elser-2-elasticsearch",
+          "model_settings": {
+            "service": "elasticsearch",
+            "task_type": "sparse_embedding"
+          }
+        },
+        "version": {
+          "type": "version"
+        },
+        "package_name": {
+          "type": "keyword"
+        },
+        "installed_at": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/fleet/build.gradle

@@ -24,7 +24,10 @@ dependencies {
   javaRestTestImplementation(project(path: xpackModule('core')))
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
   compileOnly project(path: xpackModule('ilm'))
+  testImplementation project(path: xpackModule('mapper-version'))
 
   clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('mapper-version'))
+  clusterModules project(xpackModule('inference'))
   clusterModules project(':modules:data-streams')
 }

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java

@@ -17,6 +17,8 @@ public abstract class AbstractFleetIT extends ESRestTestCase {
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("x-pack-fleet")
         .module("x-pack-ilm")
+        .module("mapper-version")
+        .module("x-pack-inference")
         .module("data-streams")
         .setting("xpack.security.enabled", "true")
         .setting("xpack.security.autoconfiguration.enabled", "false")

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java

@@ -293,4 +293,20 @@ public void verifyFileDeliveryDataILMPolicyExists() throws Exception {
             assertThat(policyMap.size(), equalTo(2));
         });
     }
+
+    public void testCreationOfIntegrationKnowledge() throws Exception {
+        Request request = new Request("PUT", ".integration_knowledge");
+        Response response = client().performRequest(request);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+
+        request = new Request("GET", ".integration_knowledge/_mapping");
+        response = client().performRequest(request);
+        String responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, containsString("content"));
+
+        request = new Request("GET", ".integration_knowledge-7/_mapping");
+        response = client().performRequest(request);
+        responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, containsString("content"));
+    }
 }

x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java

@@ -87,6 +87,7 @@ public class Fleet extends Plugin implements SystemIndexPlugin {
     private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1;
     private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1;
     private static final int FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION = 1;
+    private static final int FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION = 1;
 
     @Override
     public Collection<?> createComponents(PluginServices services) {
@@ -111,7 +112,8 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
             fleetPoliciesSystemIndexDescriptor(),
             fleetPoliciesLeaderSystemIndexDescriptor(),
             fleetServersSystemIndexDescriptors(),
-            fleetArtifactsSystemIndexDescriptors()
+            fleetArtifactsSystemIndexDescriptors(),
+            fleetIntegrationKnowledgeSystemIndexDescriptor()
         );
     }
 
@@ -267,6 +269,26 @@ private static SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() {
             .build();
     }
 
+    private static SystemIndexDescriptor fleetIntegrationKnowledgeSystemIndexDescriptor() {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest();
+        request.source(
+            loadTemplateSource("/fleet-integration-knowledge.json", FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION),
+            XContentType.JSON
+        );
+
+        return SystemIndexDescriptor.builder()
+            .setType(Type.EXTERNAL_MANAGED)
+            .setAllowedElasticProductOrigins(ALLOWED_PRODUCTS)
+            .setOrigin(FLEET_ORIGIN)
+            .setMappings(request.mappings())
+            .setSettings(request.settings())
+            .setPrimaryIndex(".integration_knowledge-" + CURRENT_INDEX_VERSION)
+            .setIndexPattern(".integration_knowledge*")
+            .setAliasName(".integration_knowledge")
+            .setDescription("Integration package knowledge base content storage")
+            .build();
+    }
+
     private static SystemDataStreamDescriptor fleetActionsResultsDescriptor() {
         final String source = loadTemplateSource("/fleet-actions-results.json", FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION);
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, source)) {

x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java

@@ -45,7 +45,8 @@ public void testFleetIndexNames() {
                 ".fleet-policies-leader*",
                 ".fleet-enrollment-api-keys*",
                 ".fleet-artifacts*",
-                ".fleet-secrets*"
+                ".fleet-secrets*",
+                ".integration_knowledge*"
             )
         );
 
@@ -60,6 +61,8 @@ public void testFleetIndexNames() {
         assertFalse(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results")));
 
         assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-secrets")));
+
+        assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".integration_knowledge")));
     }
 
     public void testFleetFeature() {

x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java

@@ -28,6 +28,8 @@ public FleetRestIT(final ClientYamlTestCandidate testCandidate) {
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("x-pack-fleet")
         .module("x-pack-ilm")
+        .module("mapper-version")
+        .module("x-pack-inference")
         .module("data-streams")
         .setting("xpack.license.self_generated.type", "basic")
         .setting("xpack.security.enabled", "true")