Revert "Revert "[Search] Add system index descriptors to Connector indices (#…"

This reverts commit ddb97cc.

Author: navarone-feekery

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java

@@ -196,6 +196,9 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion(
     public static final String APM_ORIGIN = "apm";
     public static final String OTEL_ORIGIN = "otel";
     public static final String REINDEX_DATA_STREAM_ORIGIN = "reindex_data_stream";
+    // TODO consolidate the Kibana origin with the one defined in org/elasticsearch/kibana/KibanaPlugin.java
+    public static final String KIBANA_ORIGIN = "kibana";
+    public static final String CLOUD_ORIGIN = "cloud";
 
     private ClientHelper() {}
 

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json renamed to x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json

@@ -1,12 +1,16 @@
 {
-  "template": {
-    "aliases": {
-      ".elastic-connectors-sync-jobs":  {}
-    },
-    "mappings": {
-      "dynamic": "false",
+  "settings": {
+    "index": {
+      "number_of_shards": "1",
+      "auto_expand_replicas": "0-1"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": "strict",
       "_meta": {
-        "version": ${xpack.application.connector.template.version}
+        "version": "${elastic-connectors-sync-jobs.version}",
+        "managed_index_mappings_version": ${elastic-connectors-sync-jobs.managed.index.version}
       },
       "properties": {
         "cancelation_requested_at": {
@@ -21,9 +25,11 @@
         "connector": {
           "properties": {
             "configuration": {
+              "dynamic": "false",
               "type": "object"
             },
             "filtering": {
+              "dynamic": "false",
               "properties": {
                 "advanced_snippet": {
                   "properties": {
@@ -91,6 +97,7 @@
               "type": "keyword"
             },
             "pipeline": {
+              "dynamic": "false",
               "properties": {
                 "extract_binary_content": {
                   "type": "boolean"
@@ -110,6 +117,7 @@
               "type": "keyword"
             },
             "sync_cursor": {
+              "dynamic": "false",
               "type": "object"
             }
           }
@@ -136,6 +144,7 @@
           "type": "date"
         },
         "metadata": {
+          "dynamic": "false",
           "type": "object"
         },
         "started_at": {
@@ -155,10 +164,5 @@
         }
       }
     }
-  },
-  "_meta": {
-    "description": "Built-in mappings applied by default to elastic-connectors indices",
-    "managed": true
-  },
-  "version": ${xpack.application.connector.template.version}
+  }
 }

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json renamed to x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json

@@ -1,36 +1,43 @@
 {
-  "template": {
-    "aliases": {
-      ".elastic-connectors":  {}
-    },
-    "mappings": {
-      "dynamic": "false",
+  "settings": {
+    "index": {
+      "number_of_shards": "1",
+      "auto_expand_replicas": "0-1"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": "strict",
       "_meta": {
-        "pipeline": {
-          "default_name": "search-default-ingestion",
-          "default_extract_binary_content": true,
-          "default_run_ml_inference": true,
-          "default_reduce_whitespace": true
-        },
-        "version": ${xpack.application.connector.template.version}
+        "version": "${elastic-connectors.version}",
+        "managed_index_mappings_version": ${elastic-connectors.managed.index.version}
       },
       "properties": {
         "api_key_id": {
           "type": "keyword"
         },
+        "api_key_secret_id": {
+          "type": "keyword"
+        },
         "configuration": {
+          "dynamic": "false",
           "type": "object"
         },
         "custom_scheduling": {
+          "dynamic": "false",
           "type": "object"
         },
+        "deleted": {
+          "type": "boolean"
+        },
         "description": {
           "type": "text"
         },
         "error": {
           "type": "keyword"
         },
         "features": {
+          "dynamic": "false",
           "properties": {
             "filtering_advanced_config": {
               "type": "boolean"
@@ -66,6 +73,7 @@
           }
         },
         "filtering": {
+          "dynamic": "false",
           "properties": {
             "active": {
               "properties": {
@@ -78,6 +86,7 @@
                       "type": "date"
                     },
                     "value": {
+                      "dynamic": "false",
                       "type": "object"
                     }
                   }
@@ -143,6 +152,7 @@
                       "type": "date"
                     },
                     "value": {
+                      "dynamic": "false",
                       "type": "object"
                     }
                   }
@@ -242,6 +252,7 @@
           "type": "keyword"
         },
         "pipeline": {
+          "dynamic": "false",
           "properties": {
             "extract_binary_content": {
               "type": "boolean"
@@ -258,6 +269,7 @@
           }
         },
         "scheduling": {
+          "dynamic": "false",
           "properties": {
             "access_control": {
               "properties": {
@@ -298,22 +310,13 @@
           "type": "keyword"
         },
         "sync_cursor": {
+          "dynamic": "false",
           "type": "object"
         },
         "sync_now": {
           "type": "boolean"
-        },
-        "deleted": {
-          "type": "boolean"
         }
       }
     }
-  },
-  "_meta": {
-    "description": "Built-in mappings applied by default to elastic-connectors indices",
-    "managed": true
-  },
-  "version": ${xpack.application.connector.template.version}
+  }
 }
-
-

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-settings.json

@@ -46,6 +46,7 @@
 import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction;
 import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig;
 import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature;
+import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
 import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction;
 import org.elasticsearch.xpack.application.connector.action.GetConnectorAction;
@@ -124,6 +125,7 @@
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportGetConnectorSecretAction;
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportPostConnectorSecretAction;
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportPutConnectorSecretAction;
+import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
 import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction;
@@ -477,7 +479,12 @@ public Collection<?> createComponents(PluginServices services) {
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         Collection<SystemIndexDescriptor> systemIndices = new ArrayList<>(
-            List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor())
+            List.of(
+                SearchApplicationIndexService.getSystemIndexDescriptor(),
+                QueryRulesIndexService.getSystemIndexDescriptor(),
+                ConnectorSyncJobIndexService.getSystemIndexDescriptor(),
+                ConnectorIndexService.getSystemIndexDescriptor()
+            )
         );
 
         if (ConnectorSecretsFeature.isEnabled()) {

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs.json

@@ -10,10 +10,12 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -33,6 +35,7 @@
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.query.WildcardQueryBuilder;
+import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -59,6 +62,7 @@
 import org.elasticsearch.xpack.application.connector.filtering.FilteringValidationState;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob;
 import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
 
 import java.time.Instant;
 import java.util.ArrayList;
@@ -76,6 +80,7 @@
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.CONNECTORS_ALLOWED_PRODUCT_ORIGINS;
 import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
 import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN;
 
@@ -87,7 +92,20 @@ public class ConnectorIndexService {
     // The client to interact with the system index (internal user).
     private final Client clientWithOrigin;
 
-    public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN;
+    // TODO use proper version IDs (see org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java)
+    // TODO if this version is updated, a test should be added to
+    // javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java
+    private static final int CONNECTORS_INDEX_VERSION = 1;
+    // TODO rename to CONNECTOR_ALIAS_NAME
+    public static final String CONNECTOR_INDEX_NAME = ".elastic-connectors";
+    public static final String CONNECTOR_INDEX_PREFIX = ".elastic-connectors-v";
+    public static final String CONNECTOR_CONCRETE_INDEX_NAME = CONNECTOR_INDEX_PREFIX + CONNECTORS_INDEX_VERSION;
+    // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs
+
+    public static final String CONNECTOR_INDEX_NAME_PATTERN = CONNECTOR_INDEX_PREFIX + "*";
+
+    private static final String CONNECTORS_MAPPING_VERSION_VARIABLE = "elastic-connectors.version";
+    private static final String CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE = "elastic-connectors.managed.index.version";
 
     /**
      * @param client A client for executing actions on the connector index
@@ -96,6 +114,36 @@ public ConnectorIndexService(Client client) {
         this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN);
     }
 
+    /**
+     * Returns the {@link SystemIndexDescriptor} for the Connector system index.
+     *
+     * @return The {@link SystemIndexDescriptor} for the Connector system index.
+     */
+    public static SystemIndexDescriptor getSystemIndexDescriptor() {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest();
+        String templateSource = TemplateUtils.loadTemplate(
+            "/elastic-connectors.json",
+            Version.CURRENT.toString(),
+            CONNECTORS_MAPPING_VERSION_VARIABLE,
+            Map.of(CONNECTORS_MAPPING_MANAGED_VERSION_VARIABLE, Integer.toString(CONNECTORS_INDEX_VERSION))
+        );
+        request.source(templateSource, XContentType.JSON);
+
+        // The index pattern needs a stricter regex to prevent conflicts with .elastic-connectors-sync-jobs
+        return SystemIndexDescriptor.builder()
+            .setIndexPattern(CONNECTOR_INDEX_NAME_PATTERN)
+            .setPrimaryIndex(CONNECTOR_CONCRETE_INDEX_NAME)
+            .setAliasName(CONNECTOR_INDEX_NAME)
+            .setDescription("Search connectors")
+            .setMappings(request.mappings())
+            .setSettings(request.settings())
+            .setOrigin(CONNECTORS_ORIGIN)
+            .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED)
+            .setAllowedElasticProductOrigins(CONNECTORS_ALLOWED_PRODUCT_ORIGINS)
+            .setNetNew()
+            .build();
+    }
+
     /**
      * Creates or updates the {@link Connector} in the underlying index with a specific doc ID
      * if connectorId is provided. Otherwise, the connector doc is indexed with auto-generated doc ID.