diff --git a/docs/changelog/132506.yaml b/docs/changelog/132506.yaml new file mode 100644 index 0000000000000..3fd90cb657b49 --- /dev/null +++ b/docs/changelog/132506.yaml @@ -0,0 +1,5 @@ +pr: 132506 +summary: Add .integration_knowledge system index for usage by AI assistants +area: Infra/Core +type: feature +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 7f507b11cc9a5..ac128fdc7a41c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -174,6 +174,14 @@ static RoleDescriptor kibanaSystem(String name) { .privileges("write", "delete", "create_index") .allowRestrictedIndices(true) .build(), + // Integrations knowledge base: Fleet creates, manages, and uses this index to store knowledge base documents to be consumed + // by AI assistants to support integrations + // assistants. + RoleDescriptor.IndicesPrivileges.builder() + .indices(".integration_knowledge*") + .privileges("read", "write", "create_index") + .allowRestrictedIndices(true) + .build(), // Other Fleet indices. Kibana reads and writes to these indices to manage // Elastic Agents. RoleDescriptor.IndicesPrivileges.builder() diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index 28fea1a7099ed..efd94f0d7eade 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -103,7 +103,8 @@ public class TestRestrictedIndices { SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-[0-9]+*", "fleet policies"), SystemIndexDescriptorUtils.createUnmanaged(".fleet-policies-leader*", "fleet policies leader"), SystemIndexDescriptorUtils.createUnmanaged(".fleet-servers*", "fleet servers"), - SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts") + SystemIndexDescriptorUtils.createUnmanaged(".fleet-artifacts*", "fleet artifacts"), + SystemIndexDescriptorUtils.createUnmanaged(".integration_knowledge*", "fleet integration knowledge base") ), List.of( new SystemDataStreamDescriptor( diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json new file mode 100644 index 0000000000000..648fc4385c7d4 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-integration-knowledge.json @@ -0,0 +1,38 @@ +{ + "settings": { + "auto_expand_replicas": "0-1", + "index.hidden": true + }, + "mappings": { + "_doc": { + "dynamic": false, + "_meta": { + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version}, + "description": "Integration package knowledge base content storage" + }, + "properties": { + "filename": { + "type": "keyword" + }, + "content": { + "type": "semantic_text", + "inference_id": ".elser-2-elasticsearch", + "model_settings": { + "service": "elasticsearch", + "task_type": "sparse_embedding" + } + }, + "version": { + "type": "version" + }, + "package_name": { + "type": "keyword" + }, + "installed_at": { + "type": "date" + } + } + } + } +} diff --git a/x-pack/plugin/fleet/build.gradle b/x-pack/plugin/fleet/build.gradle index 4b19ff849fad8..a9643d796dbf6 100644 --- a/x-pack/plugin/fleet/build.gradle +++ b/x-pack/plugin/fleet/build.gradle @@ -24,7 +24,10 @@ dependencies { javaRestTestImplementation(project(path: xpackModule('core'))) javaRestTestImplementation(testArtifact(project(xpackModule('core')))) compileOnly project(path: xpackModule('ilm')) + testImplementation project(path: xpackModule('mapper-version')) clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-version')) + clusterModules project(xpackModule('inference')) clusterModules project(':modules:data-streams') } diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java index a835b9caa7a65..bae9c80ea0f60 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/AbstractFleetIT.java @@ -17,6 +17,8 @@ public abstract class AbstractFleetIT extends ESRestTestCase { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("x-pack-fleet") .module("x-pack-ilm") + .module("mapper-version") + .module("x-pack-inference") .module("data-streams") .setting("xpack.security.enabled", "true") .setting("xpack.security.autoconfiguration.enabled", "false") diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java index 99f04d2997608..6d1a2f2a9eb3a 100644 --- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java +++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java @@ -293,4 +293,20 @@ public void verifyFileDeliveryDataILMPolicyExists() throws Exception { assertThat(policyMap.size(), equalTo(2)); }); } + + public void testCreationOfIntegrationKnowledge() throws Exception { + Request request = new Request("PUT", ".integration_knowledge"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + request = new Request("GET", ".integration_knowledge/_mapping"); + response = client().performRequest(request); + String responseBody = EntityUtils.toString(response.getEntity()); + assertThat(responseBody, containsString("content")); + + request = new Request("GET", ".integration_knowledge-7/_mapping"); + response = client().performRequest(request); + responseBody = EntityUtils.toString(response.getEntity()); + assertThat(responseBody, containsString("content")); + } } diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index e21d13524f1bf..018082cb27832 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -87,6 +87,7 @@ public class Fleet extends Plugin implements SystemIndexPlugin { private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1; private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1; private static final int FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION = 1; + private static final int FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION = 1; @Override public Collection createComponents(PluginServices services) { @@ -111,7 +112,8 @@ public Collection getSystemIndexDescriptors(Settings sett fleetPoliciesSystemIndexDescriptor(), fleetPoliciesLeaderSystemIndexDescriptor(), fleetServersSystemIndexDescriptors(), - fleetArtifactsSystemIndexDescriptors() + fleetArtifactsSystemIndexDescriptors(), + fleetIntegrationKnowledgeSystemIndexDescriptor() ); } @@ -267,6 +269,26 @@ private static SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { .build(); } + private static SystemIndexDescriptor fleetIntegrationKnowledgeSystemIndexDescriptor() { + PutIndexTemplateRequest request = new PutIndexTemplateRequest(); + request.source( + loadTemplateSource("/fleet-integration-knowledge.json", FLEET_INTEGRATION_KNOWLEDGE_MAPPINGS_VERSION), + XContentType.JSON + ); + + return SystemIndexDescriptor.builder() + .setType(Type.EXTERNAL_MANAGED) + .setAllowedElasticProductOrigins(ALLOWED_PRODUCTS) + .setOrigin(FLEET_ORIGIN) + .setMappings(request.mappings()) + .setSettings(request.settings()) + .setPrimaryIndex(".integration_knowledge-" + CURRENT_INDEX_VERSION) + .setIndexPattern(".integration_knowledge*") + .setAliasName(".integration_knowledge") + .setDescription("Integration package knowledge base content storage") + .build(); + } + private static SystemDataStreamDescriptor fleetActionsResultsDescriptor() { final String source = loadTemplateSource("/fleet-actions-results.json", FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, source)) { diff --git a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java index e8b2738da603f..b8bd493769c67 100644 --- a/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java +++ b/x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java @@ -45,7 +45,8 @@ public void testFleetIndexNames() { ".fleet-policies-leader*", ".fleet-enrollment-api-keys*", ".fleet-artifacts*", - ".fleet-secrets*" + ".fleet-secrets*", + ".integration_knowledge*" ) ); @@ -60,6 +61,8 @@ public void testFleetIndexNames() { assertFalse(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-actions-results"))); assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".fleet-secrets"))); + + assertTrue(fleetDescriptors.stream().anyMatch(d -> d.matchesIndexPattern(".integration_knowledge"))); } public void testFleetFeature() { diff --git a/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java b/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java index fd710373a95c4..81497d5f784f0 100644 --- a/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java +++ b/x-pack/plugin/fleet/src/yamlRestTest/java/org/elasticsearch/xpack/fleet/FleetRestIT.java @@ -28,6 +28,8 @@ public FleetRestIT(final ClientYamlTestCandidate testCandidate) { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("x-pack-fleet") .module("x-pack-ilm") + .module("mapper-version") + .module("x-pack-inference") .module("data-streams") .setting("xpack.license.self_generated.type", "basic") .setting("xpack.security.enabled", "true")