Merge branch 'main' into charlotte-connector-RN-update

Author: charlotte-hoblik

docs/changelog/126843.yaml

@@ -42,8 +42,7 @@ breaking:
       `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
 
     * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
-      the `s3.client.${CLIENT_NAME}.protocol` setting is deprecated and no longer
-      has any effect.
+      the `s3.client.${CLIENT_NAME}.protocol` setting is deprecated.
 
     * AWS SDK v2 does not permit control over throttling for retries, so the
       the `s3.client.${CLIENT_NAME}.use_throttle_retries` setting is deprecated
@@ -81,9 +80,9 @@ breaking:
     * If applicable, discontinue use of the
       `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
 
-    * If applicable, specify that you wish to use the insecure HTTP protocol to
-      access the S3 API by setting `s3.client.${CLIENT_NAME}.endpoint` to a URL
-      which starts with `http://`.
+    * If applicable, specify the protocol to use to access the S3 API by
+      setting `s3.client.${CLIENT_NAME}.endpoint` to a URL which starts with
+      `http://` or `https://`.
 
     * If applicable, discontinue use of the `log-delivery-write` canned ACL.
 

docs/changelog/127582.yaml

@@ -0,0 +1,5 @@
+pr: 127582
+summary: Specialize ags `AddInput` for each block type
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/elasticsearch/configuration-reference/security-settings.md

@@ -1486,6 +1486,15 @@ $$$jwt-claim-pattern-principal$$$
 `client_authentication.rotation_grace_period`
 :   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Sets the grace period for how long after rotating the `client_authentication.shared_secret` is valid. `client_authentication.shared_secret` can be rotated by updating the keystore then calling the [reload API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-nodes-reload-secure-settings). Defaults to `1m`.
 
+`http.proxy.host`
+:   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Specifies the address of the proxy server for the HTTP client that is used for fetching the JSON Web Key Set from a remote URL.
+
+`http.proxy.scheme`
+:   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Specifies the protocol to use to connect to the proxy server for the HTTP client that is used for fetching the JSON Web Key Set from a remote URL. Must be `http`.
+
+`http.proxy.port`
+:   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Specifies the port of the proxy server for the HTTP client that is used for fetching the JSON Web Key Set from a remote URL. Defaults to `80`.
+
 `http.connect_timeout` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
 :   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Sets the timeout for the HTTP client that is used for fetching the JSON Web Key Set from a remote URL. A value of zero means the timeout is not used. Defaults to `5s`.
 

docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md

@@ -42,7 +42,7 @@ If multiple documents in the lookup index match a single row in your
 results, the output will contain one row for each matching combination.
 
 ::::{tip}
-In case of name collisions, the newly created columns will override existing columns.
+For important information about using `LOOKUP JOIN`, refer to [Usage notes](../../../../esql/esql-lookup-join.md#usage-notes).
 ::::
 
 **Examples**

docs/reference/query-languages/esql/_snippets/commands/layout/mv_expand.md

@@ -22,6 +22,12 @@ MV_EXPAND column
 `column`
 :   The multivalued column to expand.
 
+::::{warning}
+The output rows produced by `MV_EXPAND` can be in any order and may not respect
+preceding `SORT`s. To guarantee a certain ordering, place a `SORT` after any
+`MV_EXPAND`s.
+::::
+
 **Example**
 
 :::{include} ../examples/mv_expand.csv-spec/simple.md

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -156,9 +156,27 @@ To obtain a join key with a compatible type, use a [conversion function](/refere
 
 For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types).
 
+## Usage notes
+
+This section covers important details about `LOOKUP JOIN` that impact query behavior and results. Review these details to ensure your queries work as expected and to troubleshoot unexpected results.
+
+### Handling name collisions
+
+When fields from the lookup index match existing column names, the new columns override the existing ones.
+Before the `LOOKUP JOIN` command, preserve columns by either:
+
+* Using `RENAME` to assign non-conflicting names
+* Using `EVAL` to create new columns with different names
+
+### Sorting behavior
+
+The output rows produced by `LOOKUP JOIN` can be in any order and may not
+respect preceding `SORT`s. To guarantee a certain ordering, place a `SORT` after
+any `LOOKUP JOIN`s.
+
 ## Limitations
 
-The following are the current limitations with `LOOKUP JOIN`
+The following are the current limitations with `LOOKUP JOIN`:
 
 * Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded.
 * Cross cluster search is unsupported initially. Both source and lookup indices must be local.

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ExplicitProtocolRestIT.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.aws.DynamicRegionSupplier;
+import fixture.s3.S3HttpFixture;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.util.function.Supplier;
+
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+import static org.hamcrest.Matchers.startsWith;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
+public class RepositoryS3ExplicitProtocolRestIT extends AbstractRepositoryS3RestTestCase {
+
+    private static final String PREFIX = getIdentifierPrefix("RepositoryS3ExplicitProtocolRestIT");
+    private static final String BUCKET = PREFIX + "bucket";
+    private static final String BASE_PATH = PREFIX + "base_path";
+    private static final String ACCESS_KEY = PREFIX + "access-key";
+    private static final String SECRET_KEY = PREFIX + "secret-key";
+    private static final String CLIENT = "explicit_protocol_client";
+
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        fixedAccessKey(ACCESS_KEY, regionSupplier, "s3")
+    );
+
+    private static String getEndpoint() {
+        final var s3FixtureAddress = s3Fixture.getAddress();
+        assertThat(s3FixtureAddress, startsWith("http://"));
+        return s3FixtureAddress.substring("http://".length());
+    }
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .systemProperty("aws.region", regionSupplier)
+        .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY)
+        .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
+        .setting("s3.client." + CLIENT + ".endpoint", RepositoryS3ExplicitProtocolRestIT::getEndpoint)
+        .setting("s3.client." + CLIENT + ".protocol", () -> "http")
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String getBucketName() {
+        return BUCKET;
+    }
+
+    @Override
+    protected String getBasePath() {
+        return BASE_PATH;
+    }
+
+    @Override
+    protected String getClientName() {
+        return CLIENT;
+    }
+}

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java

@@ -77,9 +77,10 @@ final class S3ClientSettings {
         key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope)
     );
 
-    /** Formerly the protocol to use to connect to s3, now unused. V2 AWS SDK can infer the protocol from {@link #endpoint}. */
+    /** The protocol to use to connect to s3, now only used if {@link #endpoint} is not a proper URI that starts with {@code http://} or
+     * {@code https://}. */
     @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // no longer used, should be removed in v10
-    static final Setting.AffixSetting<HttpScheme> UNUSED_PROTOCOL_SETTING = Setting.affixKeySetting(
+    static final Setting.AffixSetting<HttpScheme> PROTOCOL_SETTING = Setting.affixKeySetting(
         PREFIX,
         "protocol",
         key -> new Setting<>(key, "https", s -> HttpScheme.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated)
@@ -181,6 +182,9 @@ final class S3ClientSettings {
     /** Credentials to authenticate with s3. */
     final AwsCredentials credentials;
 
+    /** The scheme (HTTP or HTTPS) for talking to the endpoint, for use only if the endpoint doesn't contain an explicit scheme */
+    final HttpScheme protocol;
+
     /** The s3 endpoint the client should talk to, or empty string to use the default. */
     final String endpoint;
 
@@ -221,6 +225,7 @@ final class S3ClientSettings {
 
     private S3ClientSettings(
         AwsCredentials credentials,
+        HttpScheme protocol,
         String endpoint,
         String proxyHost,
         int proxyPort,
@@ -235,6 +240,7 @@ private S3ClientSettings(
         String region
     ) {
         this.credentials = credentials;
+        this.protocol = protocol;
         this.endpoint = endpoint;
         this.proxyHost = proxyHost;
         this.proxyPort = proxyPort;
@@ -261,6 +267,7 @@ S3ClientSettings refine(Settings repositorySettings) {
             .put(repositorySettings)
             .normalizePrefix(PREFIX + PLACEHOLDER_CLIENT + '.')
             .build();
+        final HttpScheme newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol);
         final String newEndpoint = getRepoSettingOrDefault(ENDPOINT_SETTING, normalizedSettings, endpoint);
 
         final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxyHost);
@@ -284,7 +291,8 @@ S3ClientSettings refine(Settings repositorySettings) {
             newCredentials = credentials;
         }
         final String newRegion = getRepoSettingOrDefault(REGION, normalizedSettings, region);
-        if (Objects.equals(endpoint, newEndpoint)
+        if (Objects.equals(protocol, newProtocol)
+            && Objects.equals(endpoint, newEndpoint)
             && Objects.equals(proxyHost, newProxyHost)
             && proxyPort == newProxyPort
             && proxyScheme == newProxyScheme
@@ -299,6 +307,7 @@ S3ClientSettings refine(Settings repositorySettings) {
         }
         return new S3ClientSettings(
             newCredentials,
+            newProtocol,
             newEndpoint,
             newProxyHost,
             newProxyPort,
@@ -405,6 +414,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String
         ) {
             return new S3ClientSettings(
                 S3ClientSettings.loadCredentials(settings, clientName),
+                getConfigValue(settings, clientName, PROTOCOL_SETTING),
                 getConfigValue(settings, clientName, ENDPOINT_SETTING),
                 getConfigValue(settings, clientName, PROXY_HOST_SETTING),
                 getConfigValue(settings, clientName, PROXY_PORT_SETTING),
@@ -435,6 +445,7 @@ public boolean equals(final Object o) {
             && maxConnections == that.maxConnections
             && maxRetries == that.maxRetries
             && Objects.equals(credentials, that.credentials)
+            && Objects.equals(protocol, that.protocol)
             && Objects.equals(endpoint, that.endpoint)
             && Objects.equals(proxyHost, that.proxyHost)
             && proxyScheme == that.proxyScheme
@@ -448,6 +459,7 @@ public boolean equals(final Object o) {
     public int hashCode() {
         return Objects.hash(
             credentials,
+            protocol,
             endpoint,
             proxyHost,
             proxyPort,

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java

@@ -131,7 +131,7 @@ public List<Setting<?>> getSettings() {
             S3ClientSettings.SECRET_KEY_SETTING,
             S3ClientSettings.SESSION_TOKEN_SETTING,
             S3ClientSettings.ENDPOINT_SETTING,
-            S3ClientSettings.UNUSED_PROTOCOL_SETTING,
+            S3ClientSettings.PROTOCOL_SETTING,
             S3ClientSettings.PROXY_HOST_SETTING,
             S3ClientSettings.PROXY_PORT_SETTING,
             S3ClientSettings.PROXY_SCHEME_SETTING,

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -257,14 +257,18 @@ protected S3ClientBuilder buildClientBuilder(S3ClientSettings clientSettings, Sd
             String endpoint = clientSettings.endpoint;
             if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) {
                 // The SDK does not know how to interpret endpoints without a scheme prefix and will error. Therefore, when the scheme is
-                // absent, we'll supply HTTPS as a default to avoid errors.
+                // absent, we'll look at the deprecated .protocol setting
                 // See https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/client-configuration.html#client-config-other-diffs
-                endpoint = "https://" + endpoint;
+                endpoint = switch (clientSettings.protocol) {
+                    case HTTP -> "http://" + endpoint;
+                    case HTTPS -> "https://" + endpoint;
+                };
                 LOGGER.warn(
                     """
-                        found S3 client with endpoint [{}] that is missing a scheme, guessing it should use 'https://'; \
+                        found S3 client with endpoint [{}] that is missing a scheme, guessing it should be [{}]; \
                         to suppress this warning, add a scheme prefix to the [{}] setting on this node""",
                     clientSettings.endpoint,
+                    endpoint,
                     S3ClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace("CLIENT_NAME").getKey()
                 );
             }