Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -42,7 +42,7 @@ The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} ind
 
 ## Example
 
-`LOOKUP JOIN` has left-join behavior. If no rows match in the looked index, `LOOKUP JOIN` retains the incoming row and adds `null`s. If many rows in the lookedup index match, `LOOKUP JOIN` adds one row per match.
+`LOOKUP JOIN` has left-join behavior. If no rows match in the lookup index, `LOOKUP JOIN` retains the incoming row and adds `null`s. If many rows in the lookup index match, `LOOKUP JOIN` adds one row per match.
 
 In this example, we have two sample tables:
 
@@ -112,7 +112,9 @@ To use `LOOKUP JOIN`, the following requirements must be met:
   * Numeric types follow these compatibility rules:
     * `short` and `byte` are compatible with `integer` (all represented as `int`)
     * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`)
-  * For text fields: You can use text fields on the left-hand side of the join only if they have a `.keyword` subfield
+  * For text fields: You can only use text fields as the join key on the left-hand side of the join and only if they have a `.keyword` subfield
+
+To obtain a join key with a compatible type, use a [conversion function](/reference/query-languages/esql/esql-functions-operators.md#esql-type-conversion-functions) if needed.
 
 For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types).
 
@@ -121,7 +123,7 @@ For a complete list of supported data types and their internal representations,
 The following are the current limitations with `LOOKUP JOIN`
 
 * Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded.
-* Cross cluster search is unsupported. Both source and lookup indices must be local.
+* Cross cluster search is unsupported initially. Both source and lookup indices must be local.
 * Currently, only matching on equality is supported.
 * `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.
 * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve.

muted-tests.yml

@@ -380,9 +380,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.spatial.SpatialExtentAggregationNoLicenseIT
   method: testStExtentAggregationWithPoints
   issue: https://github.com/elastic/elasticsearch/issues/125735
-- class: org.elasticsearch.repositories.blobstore.testkit.analyze.GCSRepositoryAnalysisRestIT
-  method: testRepositoryAnalysis
-  issue: https://github.com/elastic/elasticsearch/issues/125668
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test schedule_now on an already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/120720

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2InstanceProfileIT.java

@@ -10,14 +10,13 @@
 package org.elasticsearch.discovery.ec2;
 
 import fixture.aws.DynamicAwsCredentials;
+import fixture.aws.DynamicRegionSupplier;
 import fixture.aws.ec2.AwsEc2HttpFixture;
 import fixture.aws.imds.Ec2ImdsHttpFixture;
 import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
 
-import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
@@ -29,7 +28,7 @@
 public class DiscoveryEc2InstanceProfileIT extends DiscoveryEc2ClusterFormationTestCase {
 
     // Lazy-initialized so we can generate it randomly, which is not possible in static context.
-    private static final Supplier<String> regionSupplier = new LazyInitializable<>(ESTestCase::randomIdentifier)::getOrCompute;
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
 
     private static final DynamicAwsCredentials dynamicCredentials = new DynamicAwsCredentials(regionSupplier, "ec2");
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -2365,27 +2365,29 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
             int lastDoubleColon = expression.lastIndexOf(SELECTOR_SEPARATOR);
             if (lastDoubleColon >= 0) {
                 String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length());
-                doValidateSelectorString(() -> expression, suffix);
+                IndexComponentSelector selector = resolveAndValidateSelectorString(() -> expression, suffix);
                 String expressionBase = expression.substring(0, lastDoubleColon);
                 ensureNoMoreSelectorSeparators(expressionBase, expression);
+                ensureNotMixingRemoteClusterExpressionWithSelectorSeparator(expressionBase, selector, expression);
                 return bindFunction.apply(expressionBase, suffix);
             }
             // Otherwise accept the default
             return bindFunction.apply(expression, null);
         }
 
         public static void validateIndexSelectorString(String indexName, String suffix) {
-            doValidateSelectorString(() -> indexName + SELECTOR_SEPARATOR + suffix, suffix);
+            resolveAndValidateSelectorString(() -> indexName + SELECTOR_SEPARATOR + suffix, suffix);
         }
 
-        private static void doValidateSelectorString(Supplier<String> expression, String suffix) {
+        private static IndexComponentSelector resolveAndValidateSelectorString(Supplier<String> expression, String suffix) {
             IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
             if (selector == null) {
                 throw new InvalidIndexNameException(
                     expression.get(),
                     "invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
                 );
             }
+            return selector;
         }
 
         /**
@@ -2416,6 +2418,22 @@ private static void ensureNoMoreSelectorSeparators(String remainingExpression, S
                 );
             }
         }
+
+        /**
+         * Checks the expression for remote cluster pattern and throws an exception if it is combined with :: selectors.
+         * @throws InvalidIndexNameException if remote cluster pattern is detected after parsing the selector expression
+         */
+        private static void ensureNotMixingRemoteClusterExpressionWithSelectorSeparator(
+            String expressionWithoutSelector,
+            IndexComponentSelector selector,
+            String originalExpression
+        ) {
+            if (selector != null) {
+                if (RemoteClusterAware.isRemoteIndexName(expressionWithoutSelector)) {
+                    throw new InvalidIndexNameException(originalExpression, "Selectors are not yet supported on remote cluster patterns");
+                }
+            }
+        }
     }
 
     /**

server/src/test/java/org/elasticsearch/cluster/metadata/SelectorResolverTests.java

@@ -17,10 +17,13 @@
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Set;
+
 import static org.elasticsearch.action.support.IndexComponentSelector.DATA;
 import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES;
 import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
 import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -72,16 +75,49 @@ public void testResolveExpression() {
         // === Corner Cases
         // Empty index name is not necessarily disallowed, but will be filtered out in the next steps of resolution
         assertThat(resolve(selectorsAllowed, "::data"), equalTo(new ResolvedExpression("", DATA)));
-        // Remote cluster syntax is respected, even if code higher up the call stack is likely to already have handled it already
-        assertThat(resolve(selectorsAllowed, "cluster:index::data"), equalTo(new ResolvedExpression("cluster:index", DATA)));
-        // CCS with an empty index name is not necessarily disallowed, though other code in the resolution logic will likely throw
-        assertThat(resolve(selectorsAllowed, "cluster:::data"), equalTo(new ResolvedExpression("cluster:", DATA)));
-        // Same for empty cluster and index names
+        assertThat(resolve(selectorsAllowed, "::failures"), equalTo(new ResolvedExpression("", FAILURES)));
+        // CCS with an empty index and cluster name is not necessarily disallowed, though other code in the resolution logic will likely
+        // throw
         assertThat(resolve(selectorsAllowed, ":::data"), equalTo(new ResolvedExpression(":", DATA)));
+        assertThat(resolve(selectorsAllowed, ":::failures"), equalTo(new ResolvedExpression(":", FAILURES)));
         // Any more prefix colon characters will trigger the multiple separators error logic
         expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "::::data"));
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "::::failures"));
+        expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, ":::::failures"));
         // Suffix case is not supported because there is no component named with the empty string
         expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "index::"));
+
+        // remote cluster syntax is not allowed with :: selectors
+        final Set<String> remoteClusterExpressionsWithSelectors = Set.of(
+            "cluster:index::failures",
+            "cluster-*:index::failures",
+            "cluster-*:index-*::failures",
+            "cluster-*:*::failures",
+            "*:index-*::failures",
+            "*:*::failures",
+            "*:-test*,*::failures",
+            "cluster:::failures",
+            "failures:index::failures",
+            "data:index::failures",
+            "failures:failures::failures",
+            "data:data::failures",
+            "cluster:index::data",
+            "cluster-*:index::data",
+            "cluster-*:index-*::data",
+            "cluster-*:*::data",
+            "*:index-*::data",
+            "*:*::data",
+            "cluster:::data",
+            "failures:index::data",
+            "data:index::data",
+            "failures:failures::data",
+            "data:data::data",
+            "*:-test*,*::data"
+        );
+        for (String expression : remoteClusterExpressionsWithSelectors) {
+            var e = expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, expression));
+            assertThat(e.getMessage(), containsString("Selectors are not yet supported on remote cluster patterns"));
+        }
     }
 
     public void testResolveMatchAllToSelectors() {

test/fixtures/aws-fixture-utils/src/main/java/fixture/aws/DynamicRegionSupplier.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package fixture.aws;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+/**
+ * Lazy supplier for a region name. We cannot use randomness like {@link ESTestCase#randomIdentifier()} when creating the test fixtures in
+ * the first place because this happens in static context, so instead we create one of these and defer the creation of the region name
+ * itself until the test actually starts running.
+ */
+public class DynamicRegionSupplier implements Supplier<String> {
+    private final AtomicReference<String> generatedRegion = new AtomicReference<>();
+
+    @Override
+    public String get() {
+        return Objects.requireNonNullElseGet(generatedRegion.get(), this::generateAndGet);
+    }
+
+    private String generateAndGet() {
+        final var newRegion = "DynamicRegionSupplier-" + ESTestCase.randomIdentifier();
+        return Objects.requireNonNullElse(generatedRegion.compareAndExchange(null, newRegion), newRegion);
+    }
+}

test/fixtures/aws-sts-fixture/src/main/java/fixture/aws/sts/AwsStsHttpHandler.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
+import java.time.Clock;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
@@ -73,7 +74,7 @@ public void handle(final HttpExchange exchange) throws IOException {
                     exchange.close();
                     return;
                 }
-                final var accessKey = randomIdentifier();
+                final var accessKey = "test_key_STS_" + randomIdentifier();
                 final var sessionToken = randomIdentifier();
                 newCredentialsConsumer.accept(accessKey, sessionToken);
                 final byte[] response = String.format(
@@ -104,7 +105,7 @@ public void handle(final HttpExchange exchange) throws IOException {
                     ROLE_NAME,
                     sessionToken,
                     randomSecretKey(),
-                    ZonedDateTime.now().plusDays(1L).format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ")),
+                    ZonedDateTime.now(Clock.systemUTC()).plusDays(1L).format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ")),
                     accessKey
                 ).getBytes(StandardCharsets.UTF_8);
                 exchange.getResponseHeaders().add("Content-Type", "text/xml; charset=UTF-8");

test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java

@@ -121,7 +121,7 @@ public void handle(final HttpExchange exchange) throws IOException {
 
             if ("GET".equals(requestMethod)) {
                 if (path.equals(IMDS_SECURITY_CREDENTIALS_PATH) && dynamicProfileNames) {
-                    final var profileName = randomIdentifier();
+                    final var profileName = "imds_profile_" + randomIdentifier();
                     validCredentialsEndpoints.add(IMDS_SECURITY_CREDENTIALS_PATH + profileName);
                     sendStringResponse(exchange, profileName);
                     return;
@@ -133,7 +133,7 @@ public void handle(final HttpExchange exchange) throws IOException {
                     sendStringResponse(exchange, Strings.toString(instanceIdentityDocument));
                     return;
                 } else if (validCredentialsEndpoints.contains(path)) {
-                    final String accessKey = randomIdentifier();
+                    final String accessKey = "test_key_imds_" + randomIdentifier();
                     final String sessionToken = randomIdentifier();
                     newCredentialsConsumer.accept(accessKey, sessionToken);
                     final byte[] response = Strings.format(

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java

@@ -637,7 +637,13 @@ public void testInvalidCharacterInIndexPattern() {
                 expectDoubleColonErrorWithLineNumber(command, "*:*::failures", parseLineNumber + 3);
 
                 // Too many colons
-                expectInvalidIndexNameErrorWithLineNumber(command, "\"index:::data\"", lineNumber, "index:", "must not contain ':'");
+                expectInvalidIndexNameErrorWithLineNumber(
+                    command,
+                    "\"index:::data\"",
+                    lineNumber,
+                    "index:::data",
+                    "Selectors are not yet supported on remote cluster patterns"
+                );
                 expectInvalidIndexNameErrorWithLineNumber(
                     command,
                     "\"index::::data\"",