[Fleet] Add index and pipeline to allow fleet agents enrichment

Author: nchaulet

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java

@@ -189,6 +189,11 @@ static RoleDescriptor kibanaSystem(String name) {
                     .privileges("all")
                     .allowRestrictedIndices(true)
                     .build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".fleet-agent-policies-metadata*")
+                    .privileges("all")
+                    .allowRestrictedIndices(true)
+                    .build(),
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".fleet-policies-leader*")
                     .privileges("all")

x-pack/plugin/core/template-resources/src/main/resources/fleet-agent-policies-metadata.json

@@ -0,0 +1,23 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1"
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}",
+        "managed_index_mappings_version": ${fleet.managed.index.version}
+      },
+      "properties": {
+        "policy_id": {
+          "type": "keyword"
+        },
+        "agent_policy": {
+          "type": "object",
+          "enabled": false
+        }
+      }
+    }
+  }
+}

x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json

@@ -1,6 +1,9 @@
 {
   "settings": {
-    "auto_expand_replicas": "0-1"
+    "auto_expand_replicas": "0-1",
+    "index": {
+      "default_pipeline": "fleet-agents@default-pipeline"
+    } 
   },
   "mappings": {
     "_doc": {
@@ -30,6 +33,30 @@
             }
           }
         },
+        "agent_policy": {
+          "properties": {
+            "name": {
+              "type": "text"
+            },
+            "package_policies": {
+              "properties": {
+                "name": {
+                  "type": "text"
+                },
+                "package": {
+                  "properties": {
+                    "name": {
+                      "type": "keyword"
+                    },
+                    "version": {
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "default_api_key": {
           "type": "keyword"
         },

x-pack/plugin/core/template-resources/src/main/resources/[email protected]

@@ -0,0 +1,14 @@
+{
+  "processors": [
+    {
+      "pipeline": {
+        "name": "fleet-agents@enrich-agent-policies-pipeline",
+        "ignore_missing_pipeline": true
+      }
+    }
+  ],
+  "_meta": {
+    "description": "TODO",
+    "managed": true
+  }
+}

x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java

@@ -84,6 +84,7 @@ public class Fleet extends Plugin implements SystemIndexPlugin {
     private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 1;
     private static final int FLEET_SECRETS_MAPPINGS_VERSION = 1;
     private static final int FLEET_POLICIES_MAPPINGS_VERSION = 1;
+    private static final int FLEET_POLICIES_METADATA_MAPPINGS_VERSION = 1;
     private static final int FLEET_POLICIES_LEADER_MAPPINGS_VERSION = 1;
     private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1;
     private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1;
@@ -110,6 +111,7 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
             fleetEnrollmentApiKeysSystemIndexDescriptor(),
             fleetSecretsSystemIndexDescriptor(),
             fleetPoliciesSystemIndexDescriptor(),
+            fleetPoliciesMetadataSystemIndexDescriptor(),
             fleetPoliciesLeaderSystemIndexDescriptor(),
             fleetServersSystemIndexDescriptors(),
             fleetArtifactsSystemIndexDescriptors()
@@ -217,6 +219,23 @@ private static SystemIndexDescriptor fleetPoliciesSystemIndexDescriptor() {
             .build();
     }
 
+    private static SystemIndexDescriptor fleetPoliciesMetadataSystemIndexDescriptor() {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest();
+        request.source(loadTemplateSource("/fleet-agent-policies-metadata.json", FLEET_POLICIES_METADATA_MAPPINGS_VERSION), XContentType.JSON);
+
+        return SystemIndexDescriptor.builder()
+            .setType(Type.EXTERNAL_MANAGED)
+            .setAllowedElasticProductOrigins(ALLOWED_PRODUCTS)
+            .setOrigin(FLEET_ORIGIN)
+            .setMappings(request.mappings())
+            .setSettings(request.settings())
+            .setPrimaryIndex(".fleet-agent-policies-metadata-" + CURRENT_INDEX_VERSION)
+            .setIndexPattern(".fleet-agent-policies-metadata-[0-9]+*")
+            .setAliasName(".fleet-agent-policies-metadata")
+            .setDescription("Fleet Policies metadata for agent enrichment")
+            .build();
+    }
+
     private static SystemIndexDescriptor fleetPoliciesLeaderSystemIndexDescriptor() {
         PutIndexTemplateRequest request = new PutIndexTemplateRequest();
         request.source(loadTemplateSource("/fleet-policies-leader.json", FLEET_POLICIES_LEADER_MAPPINGS_VERSION), XContentType.JSON);

x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java

@@ -17,13 +17,15 @@
 import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
 import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
+import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
+import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig;
 
 import java.util.List;
 import java.util.Map;
 
 public class FleetTemplateRegistry extends IndexTemplateRegistry {
 
-    public static final int INDEX_TEMPLATE_VERSION = 1;
+    public static final int INDEX_TEMPLATE_VERSION = 2;
 
     public static final String TEMPLATE_VERSION_VARIABLE = "xpack.fleet.template.version";
 
@@ -86,4 +88,18 @@ protected List<LifecyclePolicyConfig> getLifecycleConfigs() {
     protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
         return COMPOSABLE_INDEX_TEMPLATE_CONFIGS;
     }
+
+    @Override
+    protected List<IngestPipelineConfig> getIngestPipelines() {
+        return INGEST_PIPELINE_CONFIGS;
+    }
+
+    private static final List<IngestPipelineConfig> INGEST_PIPELINE_CONFIGS = List.of(
+        new JsonIngestPipelineConfig(
+            "fleet-agents@default-pipeline",
+            "/[email protected]",
+            INDEX_TEMPLATE_VERSION,
+            TEMPLATE_VERSION_VARIABLE
+        )
+    );
 }