ESQL: Views REST API

docs/changelog/137818.yaml

@@ -0,0 +1,5 @@
+pr: 137818
+summary: ES|QL Views REST API
+area: "ES|QL"
+type: feature
+issues: []

rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json

@@ -0,0 +1,34 @@
+{
+  "esql.delete_view": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-delete",
+      "description": "Delete a non-materialized VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/view/{name}",
+          "methods": [
+            "DELETE"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view to delete"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json

@@ -0,0 +1,40 @@
+{
+  "esql.get_view": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-get",
+      "description": "Get a non-materialized VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/view/{name}",
+          "methods": [
+            "GET"
+          ],
+          "parts": {
+            "name": {
+              "type": "list",
+              "description": "A comma-separated list of view names"
+            }
+          }
+        },
+        {
+          "path": "/_query/view",
+          "methods": [
+            "GET"
+          ]
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json

@@ -0,0 +1,38 @@
+{
+  "esql.put_view": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-put",
+      "description": "Creates a non-materialized VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/view/{name}",
+          "methods": [
+            "PUT"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view to create or update"
+            }
+          }
+        }
+      ]
+    },
+    "body": {
+      "description": "Use the `query` element to define the ES|QL query to use as a non-materialized VIEW.",
+      "required": true
+    }
+  }
+}

server/src/main/resources/transport/definitions/referable/esql_views.csv

@@ -0,0 +1 @@
+9220000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-search_project_routing,9219000
+esql_views,9220000

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java

@@ -34,9 +34,15 @@ public class EsqlFeatures implements FeatureSpecification {
      */
     public static final NodeFeature METRICS_SYNTAX = new NodeFeature("esql.metrics_syntax");
 
+    /**
+     * Cluster support for view management (create, get, update, list)
+     * This marks that the cluster state has support for the ViewMetadata CustomProject
+     */
+    public static final NodeFeature ESQL_VIEWS = new NodeFeature("esql.views");
+
     private Set<NodeFeature> snapshotBuildFeatures() {
         assert Build.current().isSnapshot() : Build.current();
-        return Set.of(METRICS_SYNTAX);
+        return Set.of(METRICS_SYNTAX, ESQL_VIEWS);
     }
 
     @Override

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java

@@ -6,7 +6,9 @@
  */
 package org.elasticsearch.xpack.esql.plugin;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -48,6 +50,8 @@
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
@@ -74,6 +78,7 @@
 import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.expression.ExpressionWritables;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder;
 import org.elasticsearch.xpack.esql.plan.PlanWritables;
@@ -82,6 +87,18 @@
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
 import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
+import org.elasticsearch.xpack.esql.view.ClusterViewService;
+import org.elasticsearch.xpack.esql.view.DeleteViewAction;
+import org.elasticsearch.xpack.esql.view.GetViewAction;
+import org.elasticsearch.xpack.esql.view.PutViewAction;
+import org.elasticsearch.xpack.esql.view.RestDeleteViewAction;
+import org.elasticsearch.xpack.esql.view.RestGetViewAction;
+import org.elasticsearch.xpack.esql.view.RestPutViewAction;
+import org.elasticsearch.xpack.esql.view.TransportDeleteViewAction;
+import org.elasticsearch.xpack.esql.view.TransportGetViewAction;
+import org.elasticsearch.xpack.esql.view.TransportPutViewAction;
+import org.elasticsearch.xpack.esql.view.ViewMetadata;
+import org.elasticsearch.xpack.esql.view.ViewService;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -182,6 +199,14 @@ public Collection<?> createComponents(PluginServices services) {
         );
         BigArrays bigArrays = services.indicesService().getBigArrays().withCircuitBreaking();
         var blockFactoryProvider = blockFactoryProvider(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize);
+        EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
+        ViewService viewService = new ClusterViewService(
+            functionRegistry,
+            services.clusterService(),
+            services.featureService(),
+            services.projectResolver(),
+            ViewService.ViewServiceConfig.fromSettings(settings)
+        );
         setupSharedSecrets();
         List<BiConsumer<LogicalPlan, Failures>> extraCheckers = extraCheckerProviders.stream()
             .flatMap(p -> p.checkers(services.projectResolver(), services.clusterService()).stream())
@@ -201,7 +226,9 @@ public Collection<?> createComponents(PluginServices services) {
                 ThreadPool.Names.SEARCH,
                 blockFactoryProvider.blockFactory()
             ),
-            blockFactoryProvider
+            blockFactoryProvider,
+            functionRegistry,
+            viewService
         );
     }
 
@@ -254,7 +281,7 @@ public List<Setting<?>> getSettings() {
 
     @Override
     public List<ActionHandler> getActions() {
-        return List.of(
+        List<ActionHandler> releasedActions = List.of(
             new ActionHandler(EsqlQueryAction.INSTANCE, TransportEsqlQueryAction.class),
             new ActionHandler(EsqlAsyncGetResultAction.INSTANCE, TransportEsqlAsyncGetResultsAction.class),
             new ActionHandler(EsqlStatsAction.INSTANCE, TransportEsqlStatsAction.class),
@@ -266,6 +293,18 @@ public List<ActionHandler> getActions() {
             new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class),
             new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class)
         );
+        if (Build.current().isSnapshot()) {
+            List<ActionHandler> actions = new ArrayList<>(releasedActions);
+            actions.addAll(
+                List.of(
+                    new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class),
+                    new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class),
+                    new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class)
+                )
+            );
+            return actions;
+        }
+        return releasedActions;
     }
 
     @Override
@@ -280,14 +319,20 @@ public List<RestHandler> getRestHandlers(
         Supplier<DiscoveryNodes> nodesInCluster,
         Predicate<NodeFeature> clusterSupportsFeature
     ) {
-        return List.of(
+        List<RestHandler> releasedRestHandlers = List.of(
             new RestEsqlQueryAction(),
             new RestEsqlAsyncQueryAction(),
             new RestEsqlGetAsyncResultAction(),
             new RestEsqlStopAsyncAction(),
             new RestEsqlDeleteAsyncResultAction(),
             new RestEsqlListQueriesAction()
         );
+        if (Build.current().isSnapshot()) {
+            List<RestHandler> restHandlers = new ArrayList<>(releasedRestHandlers);
+            restHandlers.addAll(List.of(new RestPutViewAction(), new RestDeleteViewAction(), new RestGetViewAction()));
+            return restHandlers;
+        }
+        return releasedRestHandlers;
     }
 
     @Override
@@ -315,12 +360,26 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
 
         entries.add(ExpressionQueryBuilder.ENTRY);
         entries.add(PlanStreamWrapperQueryBuilder.ENTRY);
+        if (Build.current().isSnapshot()) {
+            entries.addAll(ViewMetadata.ENTRIES);
+        }
 
         entries.addAll(ExpressionWritables.getNamedWriteables());
         entries.addAll(PlanWritables.getNamedWriteables());
         return entries;
     }
 
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContent() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        if (Build.current().isSnapshot()) {
+            namedXContent.add(
+                new NamedXContentRegistry.Entry(Metadata.ProjectCustom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent)
+            );
+        }
+        return namedXContent;
+    }
+
     public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
         final int allocatedProcessors = EsExecutors.allocatedProcessors(settings);
         return List.of(