[Behavioral Analytics] Analytics pipeline to the index template registry (#96104)

* move the pipeline to the index template registry

* remove unneeded utils for pipeline install

* fix tests + added coverage for ingest pipeline

* fix cancellable search request

* disable search parameter tests

Author: joemcelroy

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/entsearch/EnterpriseSearchRestIT.java

@@ -9,14 +9,12 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/95917")
 public class EnterpriseSearchRestIT extends ESClientYamlSuiteTestCase {
 
     public EnterpriseSearchRestIT(final ClientYamlTestCandidate testCandidate) {

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/55_search_application_search.yml

@@ -92,6 +92,9 @@ teardown:
 
 ---
 "Query Search Application with default parameters":
+  - skip:
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/enterprise-search-team/issues/4540"
 
   - do:
       search_application.search:
@@ -103,6 +106,9 @@ teardown:
 
 ---
 "Query Search Application overriding part of the parameters":
+  - skip:
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/enterprise-search-team/issues/4540"
 
   - do:
       search_application.search:
@@ -117,6 +123,9 @@ teardown:
 
 ---
 "Query Search Application overriding all parameters":
+  - skip:
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/enterprise-search-team/issues/4540"
 
   - do:
       search_application.search:
@@ -132,6 +141,9 @@ teardown:
 
 ---
 "Query Search Application with invalid parameter validation":
+  - skip:
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/enterprise-search-team/issues/4540"
 
   - do:
       catch: "bad_request"
@@ -144,6 +156,9 @@ teardown:
 
 ---
 "Query Search Application without required parameter":
+  - skip:
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/enterprise-search-team/issues/4540"
 
   - do:
       catch: "bad_request"

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java

@@ -37,7 +37,6 @@
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xpack.application.analytics.AnalyticsIngestPipelineRegistry;
 import org.elasticsearch.xpack.application.analytics.AnalyticsTemplateRegistry;
 import org.elasticsearch.xpack.application.analytics.action.DeleteAnalyticsCollectionAction;
 import org.elasticsearch.xpack.application.analytics.action.GetAnalyticsCollectionAction;
@@ -183,13 +182,7 @@ public Collection<Object> createComponents(
         );
         analyticsTemplateRegistry.initialize();
 
-        final AnalyticsIngestPipelineRegistry analyticsPipelineRegistry = new AnalyticsIngestPipelineRegistry(
-            clusterService,
-            threadPool,
-            client
-        );
-
-        return Arrays.asList(analyticsTemplateRegistry, analyticsPipelineRegistry);
+        return Arrays.asList(analyticsTemplateRegistry);
     }
 
     @Override

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsIngestPipelineRegistry.java

@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
+import org.elasticsearch.xpack.core.template.IngestPipelineConfig;
 import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
 
 import java.io.IOException;
@@ -48,6 +49,8 @@ public class AnalyticsTemplateRegistry extends IndexTemplateRegistry {
     static final String EVENT_DATA_STREAM_SETTINGS_COMPONENT_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "settings";
     static final String EVENT_DATA_STREAM_MAPPINGS_COMPONENT_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "mappings";
 
+    static final String EVENT_DATA_STREAM_INGEST_PIPELINE_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "final_pipeline";
+
     static final Map<String, ComponentTemplate> COMPONENT_TEMPLATES;
 
     static {
@@ -78,6 +81,18 @@ public class AnalyticsTemplateRegistry extends IndexTemplateRegistry {
         COMPONENT_TEMPLATES = Map.copyOf(componentTemplates);
     }
 
+    @Override
+    protected List<IngestPipelineConfig> getIngestPipelines() {
+        return List.of(
+            new IngestPipelineConfig(
+                EVENT_DATA_STREAM_INGEST_PIPELINE_NAME,
+                ROOT_RESOURCE_PATH + EVENT_DATA_STREAM_INGEST_PIPELINE_NAME + ".json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE
+            )
+        );
+    }
+
     // Composable index templates configuration.
     static final String EVENT_DATA_STREAM_TEMPLATE_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "default";
     static final String EVENT_DATA_STREAM_TEMPLATE_FILENAME = EVENT_DATA_STREAM_INDEX_PREFIX + "template";

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java

@@ -13,6 +13,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -83,6 +86,11 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);