ESQL: Views prototype

This is based on the origional "ESQL: View" prototype by Nik but with the following changes:
* Removal of support for FROM VIEW command (ie. all parser/grammar changes reverted)
* Removal of UnresolvedView and View resolution from that
* Port to latest origin/main as of mid-September

The goal was to keep the following parts:
* REST API for view CRUD
* The initial design concept of ViewService and ViewMetadata

Co-authored-by: Craig Taverner <[email protected]>

Author: nik9000

rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json

@@ -0,0 +1,30 @@
+{
+  "esql.delete_view":{
+    "documentation":{
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-delete",
+      "description":"Creates a VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_query/view/{name}",
+          "methods":[
+            "DELETE"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json

@@ -0,0 +1,34 @@
+{
+  "esql.put_view":{
+    "documentation":{
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-put",
+      "description":"Creates a VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_query/view/{name}",
+          "methods":[
+            "POST"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view"
+            }
+          }
+        }
+      ]
+    },
+    "body":{
+      "description":"Use the `query` element to start a query.",
+      "required":true
+    }
+  }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -1081,6 +1081,11 @@ public enum Cap {
          */
         FORK_UNION_TYPES,
 
+        /**
+         * Views.
+         */
+        VIEW_V1(Build.current().isSnapshot()),
+
         /**
          * Support for the {@code leading_zeros} named parameter.
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
 import org.elasticsearch.xpack.esql.telemetry.PlanTelemetryManager;
 import org.elasticsearch.xpack.esql.telemetry.QueryMetric;
+import org.elasticsearch.xpack.esql.view.ViewService;
 
 import java.util.List;
 import java.util.function.BiConsumer;
@@ -49,14 +50,15 @@ public class PlanExecutor {
 
     public PlanExecutor(
         IndexResolver indexResolver,
+        EsqlFunctionRegistry functionRegistry,
         MeterRegistry meterRegistry,
         XPackLicenseState licenseState,
         EsqlQueryLog queryLog,
         List<BiConsumer<LogicalPlan, Failures>> extraCheckers
     ) {
         this.indexResolver = indexResolver;
         this.preAnalyzer = new PreAnalyzer();
-        this.functionRegistry = new EsqlFunctionRegistry();
+        this.functionRegistry = functionRegistry;
         this.mapper = new Mapper();
         this.metrics = new Metrics(functionRegistry);
         this.verifier = new Verifier(metrics, licenseState, extraCheckers);
@@ -69,6 +71,7 @@ public void esql(
         String sessionId,
         AnalyzerSettings analyzerSettings,
         EnrichPolicyResolver enrichPolicyResolver,
+        ViewService viewService,
         EsqlExecutionInfo executionInfo,
         IndicesExpressionGrouper indicesExpressionGrouper,
         EsqlSession.PlanRunner planRunner,
@@ -81,6 +84,7 @@ public void esql(
             analyzerSettings,
             indexResolver,
             enrichPolicyResolver,
+            viewService,
             preAnalyzer,
             functionRegistry,
             mapper,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/UnresolvedView.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.plan.logical;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.capabilities.TelemetryAware;
+import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class UnresolvedView extends LeafPlan implements Unresolvable, TelemetryAware {
+    private final String name;
+
+    public UnresolvedView(Source source, String name) {
+        super(source);
+        this.name = name;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    public String getWriteableName() {
+        throw new UnsupportedOperationException("not serialized");
+    }
+
+    @Override
+    protected NodeInfo<UnresolvedView> info() {
+        return NodeInfo.create(this, UnresolvedView::new, name);
+    }
+
+    @Override
+    public boolean resolved() {
+        return false;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    /**
+     *
+     * This is used by {@link PlanTelemetry} to collect query statistics
+     * It can return
+     * <ul>
+     *     <li>"FROM" if this a <code>|FROM idx</code> command</li>
+     *     <li>"FROM TS" if it is the result of a <code>| METRICS idx some_aggs() BY fields</code> command</li>
+     *     <li>"METRICS" if it is the result of a <code>| METRICS idx</code> (no aggs, no groupings)</li>
+     * </ul>
+     */
+    @Override
+    public String telemetryLabel() {
+        return "FROM VIEW";
+    }
+
+    @Override
+    public boolean expressionsResolved() {
+        return false;
+    }
+
+    @Override
+    public List<Attribute> output() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public String unresolvedMessage() {
+        return "Unknown view [" + name + "]";
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), name);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        UnresolvedView other = (UnresolvedView) obj;
+        return name.equals(other.name);
+    }
+
+    @Override
+    public String toString() {
+        return UNRESOLVED_PREFIX + name;
+    }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java

@@ -74,6 +74,7 @@
 import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator;
 import org.elasticsearch.xpack.esql.execution.PlanExecutor;
 import org.elasticsearch.xpack.esql.expression.ExpressionWritables;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
 import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder;
 import org.elasticsearch.xpack.esql.plan.PlanWritables;
@@ -82,6 +83,13 @@
 import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery;
 import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog;
 import org.elasticsearch.xpack.esql.session.IndexResolver;
+import org.elasticsearch.xpack.esql.view.DeleteViewAction;
+import org.elasticsearch.xpack.esql.view.PutViewAction;
+import org.elasticsearch.xpack.esql.view.RestDeleteViewAction;
+import org.elasticsearch.xpack.esql.view.RestPutViewAction;
+import org.elasticsearch.xpack.esql.view.TransportDeleteViewAction;
+import org.elasticsearch.xpack.esql.view.TransportPutViewAction;
+import org.elasticsearch.xpack.esql.view.ViewService;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -182,6 +190,8 @@ public Collection<?> createComponents(PluginServices services) {
         );
         BigArrays bigArrays = services.indicesService().getBigArrays().withCircuitBreaking();
         var blockFactoryProvider = blockFactoryProvider(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize);
+        EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
+        ViewService viewService = new ViewService(services.clusterService(), functionRegistry);
         setupSharedSecrets();
         List<BiConsumer<LogicalPlan, Failures>> extraCheckers = extraCheckerProviders.stream()
             .flatMap(p -> p.checkers(services.projectResolver(), services.clusterService()).stream())
@@ -190,6 +200,7 @@ public Collection<?> createComponents(PluginServices services) {
         return List.of(
             new PlanExecutor(
                 new IndexResolver(services.client()),
+                functionRegistry,
                 services.telemetryProvider().getMeterRegistry(),
                 getLicenseState(),
                 new EsqlQueryLog(services.clusterService().getClusterSettings(), services.slowLogFieldProvider()),
@@ -201,7 +212,9 @@ public Collection<?> createComponents(PluginServices services) {
                 ThreadPool.Names.SEARCH,
                 blockFactoryProvider.blockFactory()
             ),
-            blockFactoryProvider
+            blockFactoryProvider,
+            functionRegistry,
+            viewService
         );
     }
 
@@ -264,7 +277,9 @@ public List<ActionHandler> getActions() {
             new ActionHandler(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class),
             new ActionHandler(EsqlAsyncStopAction.INSTANCE, TransportEsqlAsyncStopAction.class),
             new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class),
-            new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class)
+            new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class),
+            new ActionHandler(PutViewAction.INSTANCE, TransportPutViewAction.class),
+            new ActionHandler(DeleteViewAction.INSTANCE, TransportDeleteViewAction.class)
         );
     }
 
@@ -286,7 +301,9 @@ public List<RestHandler> getRestHandlers(
             new RestEsqlGetAsyncResultAction(),
             new RestEsqlStopAsyncAction(),
             new RestEsqlDeleteAsyncResultAction(),
-            new RestEsqlListQueriesAction()
+            new RestEsqlListQueriesAction(),
+            new RestPutViewAction(),
+            new RestDeleteViewAction()
         );
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java

@@ -55,6 +55,7 @@
 import org.elasticsearch.xpack.esql.planner.PlannerSettings;
 import org.elasticsearch.xpack.esql.session.EsqlSession.PlanRunner;
 import org.elasticsearch.xpack.esql.session.Result;
+import org.elasticsearch.xpack.esql.view.ViewService;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -77,6 +78,7 @@ public class TransportEsqlQueryAction extends HandledTransportAction<EsqlQueryRe
     private final ClusterService clusterService;
     private final Executor requestExecutor;
     private final EnrichPolicyResolver enrichPolicyResolver;
+    private final ViewService viewService;
     private final EnrichLookupService enrichLookupService;
     private final LookupFromIndexService lookupFromIndexService;
     private final AsyncTaskManagementService<EsqlQueryRequest, EsqlQueryResponse, EsqlQueryTask> asyncTaskManagementService;
@@ -98,6 +100,7 @@ public TransportEsqlQueryAction(
         SearchService searchService,
         ExchangeService exchangeService,
         ClusterService clusterService,
+        ViewService viewService,
         ProjectResolver projectResolver,
         ThreadPool threadPool,
         BigArrays bigArrays,
@@ -112,6 +115,7 @@ public TransportEsqlQueryAction(
         this.threadPool = threadPool;
         this.planExecutor = planExecutor;
         this.clusterService = clusterService;
+        this.viewService = viewService;
         this.requestExecutor = threadPool.executor(ThreadPool.Names.SEARCH);
         exchangeService.registerTransportHandler(transportService);
         this.exchangeService = exchangeService;
@@ -262,6 +266,7 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener<Es
                 timeseriesResultTruncationDefaultSize
             ),
             enrichPolicyResolver,
+            viewService,
             executionInfo,
             remoteClusterService,
             planRunner,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java

@@ -83,6 +83,7 @@
 import org.elasticsearch.xpack.esql.planner.premapper.PreMapper;
 import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
 import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
+import org.elasticsearch.xpack.esql.view.ViewService;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -120,6 +121,7 @@ public interface PlanRunner {
     private final AnalyzerSettings clusterSettings;
     private final IndexResolver indexResolver;
     private final EnrichPolicyResolver enrichPolicyResolver;
+    private final ViewService viewService;
 
     private final PreAnalyzer preAnalyzer;
     private final Verifier verifier;
@@ -144,6 +146,7 @@ public EsqlSession(
         AnalyzerSettings clusterSettings,
         IndexResolver indexResolver,
         EnrichPolicyResolver enrichPolicyResolver,
+        ViewService viewService,
         PreAnalyzer preAnalyzer,
         EsqlFunctionRegistry functionRegistry,
         Mapper mapper,
@@ -156,6 +159,7 @@ public EsqlSession(
         this.clusterSettings = clusterSettings;
         this.indexResolver = indexResolver;
         this.enrichPolicyResolver = enrichPolicyResolver;
+        this.viewService = viewService;
         this.preAnalyzer = preAnalyzer;
         this.verifier = verifier;
         this.functionRegistry = functionRegistry;
@@ -181,7 +185,6 @@ public void execute(EsqlQueryRequest request, EsqlExecutionInfo executionInfo, P
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH);
         assert executionInfo != null : "Null EsqlExecutionInfo";
         LOGGER.debug("ESQL query:\n{}", request.query());
-        EsqlStatement statement = parse(request.query(), request.params());
         Configuration configuration = new Configuration(
             request.timeZone() == null
                 ? statement.setting(QuerySettings.TIME_ZONE)
@@ -203,7 +206,7 @@ public void execute(EsqlQueryRequest request, EsqlExecutionInfo executionInfo, P
         );
         FoldContext foldContext = configuration.newFoldContext();
 
-        LogicalPlan plan = statement.plan();
+        LogicalPlan plan = viewService.replaceViews(parse(request.query(), request.params()).plan(), planTelemetry, configuration);
         if (plan instanceof Explain explain) {
             explainMode = true;
             plan = explain.query();