ES|QL Views prototype

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.esql.analysis.*;
 import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.DateUtils;
 import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
@@ -26,6 +27,7 @@
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.QueryParams;
+import org.elasticsearch.xpack.esql.plan.IndexPattern;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 import org.elasticsearch.xpack.esql.session.Configuration;
@@ -108,7 +110,7 @@ public void setup() {
             new AnalyzerContext(
                 config,
                 functionRegistry,
-                IndexResolution.valid(esIndex),
+                Map.of(new IndexPattern(Source.EMPTY, esIndex.name()), IndexResolution.valid(esIndex)),
                 Map.of(),
                 new EnrichResolution(),
                 InferenceResolution.EMPTY,

docs/changelog/134995.yaml

@@ -0,0 +1,5 @@
+pr: 134995
+summary: ES|QL Views prototype
+area: ES|QL
+type: feature
+issues: []

rest-api-spec/src/main/resources/rest-api-spec/api/esql.delete_view.json

@@ -0,0 +1,30 @@
+{
+  "esql.delete_view":{
+    "documentation":{
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-delete",
+      "description":"Creates a VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_query/view/{name}",
+          "methods":[
+            "DELETE"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.get_view.json

@@ -0,0 +1,34 @@
+{
+  "esql.get_view":{
+    "documentation":{
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-get",
+      "description":"Get a VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_query/view/{name}",
+          "methods":[
+            "GET"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.list_views.json

@@ -0,0 +1,28 @@
+{
+  "esql.list_views": {
+    "documentation": {
+      "url": null,
+      "description": "List all defined ES|QL views"
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_query/views",
+          "methods": [
+            "GET"
+          ]
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/main/resources/rest-api-spec/api/esql.put_view.json

@@ -0,0 +1,34 @@
+{
+  "esql.put_view":{
+    "documentation":{
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-esql-view-put",
+      "description":"Creates a VIEW for ESQL."
+    },
+    "stability": "experimental",
+    "visibility": "public",
+    "headers": {
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_query/view/{name}",
+          "methods":[
+            "PUT"
+          ],
+          "parts": {
+            "name": {
+              "type": "string",
+              "description": "The name of the view"
+            }
+          }
+        }
+      ]
+    },
+    "body":{
+      "description":"Use the `query` element to start a query.",
+      "required":true
+    }
+  }
+}

server/src/main/resources/transport/definitions/referable/esql_views.csv

@@ -0,0 +1 @@
+9198000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-dimension_values,9197000
+esql_views,9198000

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java

@@ -35,7 +35,9 @@ public abstract class Attribute extends NamedExpression {
     /**
      * Changing this will break bwc with 8.15, see {@link FieldAttribute#fieldName()}.
      */
-    protected static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
+    public static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
+
+    public static final String SYNTHETIC_ATTRIBUTE_NAME_SEPARATOR = "$";
 
     private static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = TransportVersion.fromName("esql_qualifiers_in_attributes");
 
@@ -77,7 +79,7 @@ public Attribute(
     }
 
     public static String rawTemporaryName(String... parts) {
-        var name = String.join("$", parts);
+        var name = String.join(SYNTHETIC_ATTRIBUTE_NAME_SEPARATOR, parts);
         return name.isEmpty() || name.startsWith(SYNTHETIC_ATTRIBUTE_NAME_PREFIX) ? name : SYNTHETIC_ATTRIBUTE_NAME_PREFIX + name;
     }
 

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

@@ -56,6 +56,7 @@
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_V1;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.hasCapabilities;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
@@ -157,6 +158,8 @@ protected void shouldSkipTest(String testName) throws IOException {
         }
         // Unmapped fields require a correct capability response from every cluster, which isn't currently implemented.
         assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()));
+        // Views require a correct capability response from every cluster, which isn't currently implemented.
+        assumeFalse("VIEWS not yet supported in CCS", testCase.requiredCapabilities.contains(VIEWS_V1.capabilityName()));
         // Tests that use capabilities not supported in CCS
         assumeFalse(
             "This syntax is not supported with remote LOOKUP JOIN",

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -51,6 +51,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.LongStream;
@@ -70,7 +71,9 @@
 import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.createInferenceEndpoints;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteInferenceEndpoints;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteViews;
 import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs;
+import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadViewsIntoEs;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.COMPLETION;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.KNN_FUNCTION_V5;
@@ -97,6 +100,7 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase {
     protected final String instructions;
     protected final Mode mode;
     protected static Boolean supportsTook;
+    protected static Boolean supportsViews;
 
     @ParametersFactory(argumentFormatting = "csv-spec:%2$s.%3$s")
     public static List<Object[]> readScriptSpec() throws Exception {
@@ -122,23 +126,67 @@ protected EsqlSpecTestCase(
         this.mode = randomFrom(Mode.values());
     }
 
-    private static boolean dataLoaded = false;
+    private static class Protected {
+        private volatile boolean completed = false;
+        private volatile boolean started = false;
+        private volatile Throwable failure = null;
+
+        private void protectedBlock(Callable<Void> callable) {
+            if (completed) {
+                return;
+            }
+            // In case tests get run in parallel, we ensure only one setup is run, and other tests wait for this
+            synchronized (this) {
+                if (completed) {
+                    return;
+                }
+                if (started) {
+                    // Should only happen if a previous test setup failed, possibly with partial setup, let's fail fast the current test
+                    if (failure != null) {
+                        fail(failure, "Previous test setup failed: " + failure.getMessage());
+                    }
+                    fail("Previous test setup failed with unknown error");
+                }
+                started = true;
+                try {
+                    callable.call();
+                    completed = true;
+                } catch (Throwable t) {
+                    failure = t;
+                    fail(failure, "Current test setup failed: " + failure.getMessage());
+                }
+            }
+        }
+
+        private synchronized void reset() {
+            completed = false;
+            started = false;
+            failure = null;
+        }
+    }
+
+    private static final Protected INGEST = new Protected();
+    private static final Protected VIEWS = new Protected();
     protected static boolean testClustersOk = true;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() {
         assumeTrue("test clusters were broken", testClustersOk);
-        boolean supportsLookup = supportsIndexModeLookup();
-        boolean supportsSourceMapping = supportsSourceFieldMapping();
-        boolean supportsInferenceTestService = supportsInferenceTestService();
-        if (dataLoaded == false) {
-            if (supportsInferenceTestService) {
+        INGEST.protectedBlock(() -> {
+            // Inference endpoints must be created before ingesting any datasets that rely on them (mapping of inference_id)
+            if (supportsInferenceTestService()) {
                 createInferenceEndpoints(adminClient());
             }
-
-            loadDataSetIntoEs(client(), supportsLookup, supportsSourceMapping, supportsInferenceTestService);
-            dataLoaded = true;
-        }
+            loadDataSetIntoEs(client(), supportsIndexModeLookup(), supportsSourceFieldMapping(), supportsInferenceTestService());
+            return null;
+        });
+        // Views can be created before or after ingest, since index resolution is currently only done on the combined query
+        VIEWS.protectedBlock(() -> {
+            if (supportsViews()) {
+                loadViewsIntoEs(adminClient());
+            }
+            return null;
+        });
     }
 
     @AfterClass
@@ -147,15 +195,16 @@ public static void wipeTestData() throws IOException {
             return;
         }
         try {
-            dataLoaded = false;
             adminClient().performRequest(new Request("DELETE", "/*"));
         } catch (ResponseException e) {
             // 404 here just means we had no indexes
             if (e.getResponse().getStatusLine().getStatusCode() != 404) {
                 throw e;
             }
         }
-
+        INGEST.reset();
+        deleteViews(adminClient());
+        VIEWS.reset();
         deleteInferenceEndpoints(adminClient());
     }
 
@@ -481,6 +530,13 @@ protected boolean supportsTook() throws IOException {
         return supportsTook;
     }
 
+    protected boolean supportsViews() {
+        if (supportsViews == null) {
+            supportsViews = hasCapabilities(adminClient(), List.of("views_v1"));
+        }
+        return supportsViews;
+    }
+
     private String tookKey(long took) {
         if (took < 10) {
             return "lt_10ms";

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java

@@ -13,7 +13,10 @@
 import java.io.IOException;
 import java.util.List;
 
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.*;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_V1;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.hasCapabilities;
 
 /**
@@ -55,6 +58,16 @@ protected void shouldSkipTest(String testName) throws IOException {
             testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName())
         );
 
+        assumeFalse(
+            "Tests using VIEWS not supported for now (until we merge VIEWS and Subqueries/FORK including branch merging)",
+            testCase.requiredCapabilities.contains(VIEWS_V1.capabilityName())
+        );
+
+        assumeFalse(
+            "Tests using subqueries are skipped since we don't support nested subqueries",
+            testCase.requiredCapabilities.contains(SUBQUERY_IN_FROM_COMMAND.capabilityName())
+        );
+
         assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName())));
     }
 }