Support index pattern selector syntax in ES|QL

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -59,6 +59,7 @@
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 /**
  * This class main focus is to resolve multi-syntax target expressions to resources or concrete indices. This resolution is influenced
@@ -2148,23 +2149,7 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
             int lastDoubleColon = expression.lastIndexOf(SELECTOR_SEPARATOR);
             if (lastDoubleColon >= 0) {
                 String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length());
-                IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
-                if (selector == null) {
-                    // Do some work to surface a helpful error message for likely errors
-                    if (Regex.isSimpleMatchPattern(suffix)) {
-                        throw new InvalidIndexNameException(
-                            expression,
-                            "Invalid usage of :: separator, ["
-                                + suffix
-                                + "] contains a wildcard, but only the match all wildcard [*] is supported in a selector"
-                        );
-                    } else {
-                        throw new InvalidIndexNameException(
-                            expression,
-                            "Invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
-                        );
-                    }
-                }
+                doValidateSelectorString(() -> expression, suffix);
                 String expressionBase = expression.substring(0, lastDoubleColon);
                 ensureNoMoreSelectorSeparators(expressionBase, expression);
                 return bindFunction.apply(expressionBase, suffix);
@@ -2173,6 +2158,30 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
             return bindFunction.apply(expression, null);
         }
 
+        public static void validateIndexSelectorString(String indexName, String suffix) {
+            doValidateSelectorString(() -> indexName + SELECTOR_SEPARATOR + suffix, suffix);
+        }
+
+        private static void doValidateSelectorString(Supplier<String> expression, String suffix) {
+            IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
+            if (selector == null) {
+                // Do some work to surface a helpful error message for likely errors
+                if (Regex.isSimpleMatchPattern(suffix)) {
+                    throw new InvalidIndexNameException(
+                        expression.get(),
+                        "Invalid usage of :: separator, ["
+                            + suffix
+                            + "] contains a wildcard, but only the match all wildcard [*] is supported in a selector"
+                    );
+                } else {
+                    throw new InvalidIndexNameException(
+                        expression.get(),
+                        "Invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
+                    );
+                }
+            }
+        }
+
         /**
          * Checks the selectors that have been returned from splitting an expression and throws an exception if any were present.
          * @param expression Original expression

x-pack/plugin/esql/build.gradle

@@ -58,6 +58,8 @@ dependencies {
   testImplementation project(path: ':modules:analysis-common')
   testImplementation project(path: ':modules:ingest-common')
   testImplementation project(path: ':modules:legacy-geo')
+  testImplementation project(path: ':modules:data-streams')
+  testImplementation project(path: ':modules:mapper-extras')
   testImplementation project(xpackModule('esql:compute:test'))
   testImplementation('net.nextencia:rrdiagram:0.9.4')
   testImplementation('org.webjars.npm:fontsource__roboto-mono:4.5.7')

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java

@@ -8,26 +8,43 @@
 package org.elasticsearch.xpack.esql.action;
 
 import org.elasticsearch.Build;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.ClusterAdminClient;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.ResettableValue;
+import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ListMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,11 +55,13 @@
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.junit.Assume;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -60,6 +79,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.IntStream;
 import java.util.stream.LongStream;
+import java.util.stream.Stream;
 
 import static java.util.Comparator.comparing;
 import static java.util.Comparator.naturalOrder;
@@ -100,6 +120,11 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             .build();
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Stream.concat(super.nodePlugins().stream(), Stream.of(DataStreamsPlugin.class, MapperExtrasPlugin.class)).toList();
+    }
+
     public void testProjectConstant() {
         try (EsqlQueryResponse results = run("from test | eval x = 1 | keep x")) {
             assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "integer"))));
@@ -970,6 +995,152 @@ public void testIndexPatterns() throws Exception {
         }
     }
 
+    public void testDataStreamPatterns() throws Exception {
+        Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
+
+        Map<String, Long> testCases = new HashMap<>();
+        testCases.put("test_ds_patterns*", 15L);
+        testCases.put("test_ds_patterns*::data", 15L);
+        testCases.put("test_ds_patterns*::failures", 9L);
+        testCases.put("test_ds_patterns*::*", 24L);
+
+        testCases.put("test_ds_patterns_1,test_ds_patterns_2", 10L);
+        testCases.put("test_ds_patterns_1::data,test_ds_patterns_2::data", 10L);
+        testCases.put("test_ds_patterns_1::failures,test_ds_patterns_2::failures", 6L);
+        testCases.put("test_ds_patterns_1::*,test_ds_patterns_2::*", 16L);
+
+        testCases.put("test_ds_patterns_1*,test_ds_patterns_2*", 10L);
+        testCases.put("test_ds_patterns_1*::data,test_ds_patterns_2*::data", 10L);
+        testCases.put("test_ds_patterns_1*::failures,test_ds_patterns_2*::failures", 6L);
+        testCases.put("test_ds_patterns_1*::*,test_ds_patterns_2*::*", 16L);
+
+        testCases.put("*", 15L);
+        testCases.put("*::data", 15L);
+        testCases.put("*::failures", 9L);
+        testCases.put("*::*", 24L);
+
+        testCases.put("test_ds_patterns_2", 5L);
+        testCases.put("test_ds_patterns_2::data", 5L);
+        testCases.put("test_ds_patterns_2::failures", 3L);
+        testCases.put("test_ds_patterns_2::*", 8L);
+
+        boolean deleteTemplate = false;
+        List<String> deleteDataStreams = new ArrayList<>();
+        try {
+            assertAcked(
+                client().execute(
+                    TransportPutComposableIndexTemplateAction.TYPE,
+                    new TransportPutComposableIndexTemplateAction.Request("test_ds_template").indexTemplate(
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of("test_ds_patterns_*"))
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                            .template(
+                                Template.builder()
+                                    .mappings(new CompressedXContent("""
+                                        {
+                                          "dynamic": false,
+                                          "properties": {
+                                            "@timestamp": {
+                                              "type": "date"
+                                            },
+                                            "count": {
+                                                "type": "long"
+                                            }
+                                          }
+                                        }"""))
+                                    .dataStreamOptions(
+                                        ResettableValue.create(
+                                            new DataStreamOptions.Template(
+                                                ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true)))
+                                            )
+                                        )
+                                    )
+                                    .build()
+                            )
+                            .build()
+                    )
+                )
+            );
+            deleteTemplate = true;
+
+            String[] dsNames = { "test_ds_patterns_1", "test_ds_patterns_2", "test_ds_patterns_3" };
+            String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
+            int i = 0;
+            for (String dsName : dsNames) {
+                BulkResponse bulkItemResponses = client().prepareBulk()
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("1")
+                            .source("@timestamp", time, "count", ++i * 1000)
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("2")
+                            .source("@timestamp", time, "count", ++i * 1000)
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("3")
+                            .source("@timestamp", time, "count", ++i * 1000)
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("4")
+                            .source("@timestamp", time, "count", ++i * 1000)
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("5")
+                            .source("@timestamp", time, "count", ++i * 1000)
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("6")
+                            .source("@timestamp", time, "count", "garbage")
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("7")
+                            .source("@timestamp", time, "count", "garbage")
+                    )
+                    .add(
+                        new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE)
+                            .id("8")
+                            .source("@timestamp", time, "count", "garbage")
+                    )
+                    .get();
+                assertThat(bulkItemResponses.hasFailures(), is(false));
+                ensureYellow(dsName);
+                deleteDataStreams.add(dsName);
+            }
+
+            for (Map.Entry<String, Long> testCase : testCases.entrySet()) {
+                try (var results = run("from " + testCase.getKey() + " | stats count(@timestamp)")) {
+                    assertEquals(testCase.getKey(), 1, getValuesList(results).size());
+                    assertEquals(testCase.getKey(), testCase.getValue(), getValuesList(results).get(0).get(0));
+                }
+            }
+        } finally {
+            if (deleteDataStreams.isEmpty() == false) {
+                assertAcked(
+                    client().execute(
+                        DeleteDataStreamAction.INSTANCE,
+                        new DeleteDataStreamAction.Request(new TimeValue(30, TimeUnit.SECONDS), deleteDataStreams.toArray(String[]::new))
+                    )
+                );
+            }
+            if (deleteTemplate) {
+                assertAcked(
+                    client().execute(
+                        TransportDeleteComposableIndexTemplateAction.TYPE,
+                        new TransportDeleteComposableIndexTemplateAction.Request("test_ds_template")
+                    )
+                );
+            }
+        }
+    }
+
     public void testOverlappingIndexPatterns() throws Exception {
         String[] indexNames = { "test_overlapping_index_patterns_1", "test_overlapping_index_patterns_2" };
 

x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4

@@ -279,6 +279,7 @@ FROM_PIPE : PIPE -> type(PIPE), popMode;
 FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET);
 FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET);
 FROM_COLON : COLON -> type(COLON);
+FROM_SELECTOR : {this.isDevVersion()}? CAST_OP -> type(CAST_OP);
 FROM_COMMA : COMMA -> type(COMMA);
 FROM_ASSIGN : ASSIGN -> type(ASSIGN);
 METADATA : 'metadata';
@@ -608,6 +609,10 @@ CLOSING_METRICS_COLON
     : COLON -> type(COLON), popMode, pushMode(METRICS_MODE)
     ;
 
+CLOSING_METRICS_SELECTOR
+    : {this.isDevVersion()}? CAST_OP -> type(CAST_OP), popMode, pushMode(METRICS_MODE)
+    ;
+
 CLOSING_METRICS_COMMA
     : COMMA -> type(COMMA), popMode, pushMode(METRICS_MODE)
     ;

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -139,12 +139,17 @@ fromCommand
 
 indexPattern
     : (clusterString COLON)? indexString
+    | {this.isDevVersion()}? (clusterString COLON)? indexString (CAST_OP selectorString)?
     ;
 
 clusterString
     : UNQUOTED_SOURCE
     ;
 
+selectorString
+    : UNQUOTED_SOURCE
+    ;
+
 indexString
     : UNQUOTED_SOURCE
     | QUOTED_STRING