ESQL: Views REST API

This is extracted from the Views Prototype at #134995
and is originally based on the "ESQL: View" prototype by Nik but with the following changes:
* No ES|QL language integration, only REST API for adding/removing and managing Views in cluster state
* ViewMetadata now inside ProjectMetadata
* NodeFeature to ensure metadata only exists in clusters fully upgraded
* Wide CRUD support (add, update, delete, list)
* View name validation
* View count, length and depth restrictions

Co-authored-by: Craig Taverner <[email protected]>

Author: nik9000

rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json

@@ -0,0 +1,34 @@
+{
+  "esql.delete_view": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-delete",
+      "description": "Delete a non-materialized VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/view/{name}",
+          "methods": [
+            "DELETE"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view to delete"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json

@@ -0,0 +1,34 @@
+{
+  "esql.get_view": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-get",
+      "description": "Get a non-materialized VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/view/{name}",
+          "methods": [
+            "GET"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view to get"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json

@@ -0,0 +1,28 @@
+{
+  "esql.list_views": {
+    "documentation": {
+      "url": null,
+      "description": "List all defined non-materialized ES|QL views"
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/views",
+          "methods": [
+            "GET"
+          ]
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json

@@ -0,0 +1,38 @@
+{
+  "esql.put_view": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-put",
+      "description": "Creates a non-materialized VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/view/{name}",
+          "methods": [
+            "PUT"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view to create or update"
+            }
+          }
+        }
+      ]
+    },
+    "body": {
+      "description": "Use the `query` element to define the ES|QL query to use as a non-materialized VIEW.",
+      "required": true
+    }
+  }
+}

server/src/main/resources/transport/definitions/referable/esql_views.csv

@@ -0,0 +1 @@
+9216000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-inference_api_eis_authorization_persistent_task,9215000
+esql_views,9216000

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java

@@ -34,9 +34,15 @@ public class EsqlFeatures implements FeatureSpecification {
      */
     public static final NodeFeature METRICS_SYNTAX = new NodeFeature("esql.metrics_syntax");
 
+    /**
+     * Cluster support for view management (create, get, update, list)
+     * This marks that the cluster state has support for the ViewMetadata CustomProject
+     */
+    public static final NodeFeature ESQL_VIEWS = new NodeFeature("esql.views");
+
     private Set<NodeFeature> snapshotBuildFeatures() {
         assert Build.current().isSnapshot() : Build.current();
-        return Set.of(METRICS_SYNTAX);
+        return Set.of(METRICS_SYNTAX, ESQL_VIEWS);
     }
 
     @Override

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.esql.plugin;
 
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -48,6 +49,8 @@
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
@@ -74,6 +77,7 @@
 import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.expression.ExpressionWritables;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder;
 import org.elasticsearch.xpack.esql.plan.PlanWritables;
@@ -82,6 +86,21 @@
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
 import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
+import org.elasticsearch.xpack.esql.view.ClusterViewService;
+import org.elasticsearch.xpack.esql.view.DeleteViewAction;
+import org.elasticsearch.xpack.esql.view.GetViewAction;
+import org.elasticsearch.xpack.esql.view.ListViewsAction;
+import org.elasticsearch.xpack.esql.view.PutViewAction;
+import org.elasticsearch.xpack.esql.view.RestDeleteViewAction;
+import org.elasticsearch.xpack.esql.view.RestGetViewAction;
+import org.elasticsearch.xpack.esql.view.RestListViewsAction;
+import org.elasticsearch.xpack.esql.view.RestPutViewAction;
+import org.elasticsearch.xpack.esql.view.TransportDeleteViewAction;
+import org.elasticsearch.xpack.esql.view.TransportGetViewAction;
+import org.elasticsearch.xpack.esql.view.TransportListViewsAction;
+import org.elasticsearch.xpack.esql.view.TransportPutViewAction;
+import org.elasticsearch.xpack.esql.view.ViewMetadata;
+import org.elasticsearch.xpack.esql.view.ViewService;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -182,6 +201,14 @@ public Collection<?> createComponents(PluginServices services) {
         );
         BigArrays bigArrays = services.indicesService().getBigArrays().withCircuitBreaking();
         var blockFactoryProvider = blockFactoryProvider(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize);
+        EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
+        ViewService viewService = new ClusterViewService(
+            functionRegistry,
+            services.clusterService(),
+            services.featureService(),
+            services.projectResolver(),
+            ViewService.ViewServiceConfig.fromSettings(settings)
+        );
         setupSharedSecrets();
         List<BiConsumer<LogicalPlan, Failures>> extraCheckers = extraCheckerProviders.stream()
             .flatMap(p -> p.checkers(services.projectResolver(), services.clusterService()).stream())
@@ -201,7 +228,9 @@ public Collection<?> createComponents(PluginServices services) {
                 ThreadPool.Names.SEARCH,
                 blockFactoryProvider.blockFactory()
             ),
-            blockFactoryProvider
+            blockFactoryProvider,
+            functionRegistry,
+            viewService
         );
     }
 
@@ -264,7 +293,11 @@ public List<ActionHandler> getActions() {
             new ActionHandler(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class),
             new ActionHandler(EsqlAsyncStopAction.INSTANCE, TransportEsqlAsyncStopAction.class),
             new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class),
-            new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class)
+            new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class),
+            new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class),
+            new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class),
+            new ActionHandler(GetViewAction.INSTANCE, TransportGetViewAction.class),
+            new ActionHandler(ListViewsAction.INSTANCE, TransportListViewsAction.class)
         );
     }
 
@@ -286,7 +319,11 @@ public List<RestHandler> getRestHandlers(
             new RestEsqlGetAsyncResultAction(),
             new RestEsqlStopAsyncAction(),
             new RestEsqlDeleteAsyncResultAction(),
-            new RestEsqlListQueriesAction()
+            new RestEsqlListQueriesAction(),
+            new RestPutViewAction(),
+            new RestDeleteViewAction(),
+            new RestGetViewAction(),
+            new RestListViewsAction()
         );
     }
 
@@ -315,12 +352,22 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
 
         entries.add(ExpressionQueryBuilder.ENTRY);
         entries.add(PlanStreamWrapperQueryBuilder.ENTRY);
+        entries.add(ViewMetadata.ENTRY);
 
         entries.addAll(ExpressionWritables.getNamedWriteables());
         entries.addAll(PlanWritables.getNamedWriteables());
         return entries;
     }
 
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContent() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.add(
+            new NamedXContentRegistry.Entry(Metadata.ProjectCustom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent)
+        );
+        return namedXContent;
+    }
+
     public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
         final int allocatedProcessors = EsExecutors.allocatedProcessors(settings);
         return List.of(

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ClusterViewService.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.view;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.project.ProjectResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
+
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Implementation of {@link ViewService} that keeps the views in the cluster state.
+ */
+public class ClusterViewService extends ViewService {
+    private final ClusterService clusterService;
+    private final FeatureService featureService;
+    private final ProjectResolver projectResolver;
+
+    public ClusterViewService(
+        EsqlFunctionRegistry functionRegistry,
+        ClusterService clusterService,
+        FeatureService featureService,
+        ProjectResolver projectResolver,
+        ViewServiceConfig config
+    ) {
+        super(functionRegistry, config);
+        this.clusterService = clusterService;
+        this.featureService = featureService;
+        this.projectResolver = projectResolver;
+    }
+
+    @Override
+    protected ViewMetadata getMetadata() {
+        return getMetadata(clusterService.state());
+    }
+
+    protected ViewMetadata getMetadata(ClusterState clusterState) {
+        return getProjectMetadata(clusterState).custom(ViewMetadata.TYPE, ViewMetadata.EMPTY);
+    }
+
+    protected ProjectMetadata getProjectMetadata(ClusterState clusterState) {
+        return projectResolver.getProjectMetadata(clusterService.state());
+    }
+
+    @Override
+    protected void updateViewMetadata(ActionListener<Void> callback, Function<ViewMetadata, Map<String, View>> function) {
+        submitUnbatchedTask("update-esql-view-metadata", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                var project = getProjectMetadata(currentState);
+                var views = project.custom(ViewMetadata.TYPE, ViewMetadata.EMPTY);
+                Map<String, View> policies = function.apply(views);
+                var metadata = ProjectMetadata.builder(project).putCustom(ViewMetadata.TYPE, new ViewMetadata(policies));
+                return ClusterState.builder(currentState).putProjectMetadata(metadata).build();
+            }
+
+            @Override
+            public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
+                callback.onResponse(null);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                callback.onFailure(e);
+            }
+        });
+    }
+
+    @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here
+    private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
+        clusterService.submitUnbatchedStateUpdateTask(source, task);
+    }
+
+    @Override
+    protected void assertMasterNode() {
+        assert clusterService.localNode().isMasterNode();
+    }
+
+    @Override
+    protected boolean viewsFeatureEnabled() {
+        return featureService.clusterHasFeature(clusterService.state(), EsqlFeatures.ESQL_VIEWS);
+    }
+}