From 068a2bd4db521ac4b9f0fb1119d7f65c0bf6656f Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Wed, 3 Dec 2025 23:53:11 +0530 Subject: [PATCH 1/8] Register s3.client.*.disable_chunked_encoding in repository-s3 plugin settings Signed-off-by: Aman Gautam --- .../org/opensearch/repositories/s3/S3ClientSettings.java | 6 +++++- .../org/opensearch/repositories/s3/S3RepositoryPlugin.java | 2 ++ .../opensearch/repositories/s3/S3RepositoryPluginTests.java | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index ee4a7ac09cca1..5fd6b296a1731 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -673,6 +673,7 @@ public boolean equals(final Object o) { && connectionTimeoutMillis == that.connectionTimeoutMillis && connectionTTLMillis == that.connectionTTLMillis && maxConnections == that.maxConnections + && maxSyncConnections == that.maxSyncConnections // fixed here && connectionAcquisitionTimeoutMillis == that.connectionAcquisitionTimeoutMillis && maxRetries == that.maxRetries && throttleRetries == that.throttleRetries @@ -680,12 +681,13 @@ public boolean equals(final Object o) { && Objects.equals(endpoint, that.endpoint) && protocol == that.protocol && proxySettings.equals(that.proxySettings) - && Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding) + && disableChunkedEncoding == that.disableChunkedEncoding && Objects.equals(region, that.region) && Objects.equals(signerOverride, that.signerOverride) && Objects.equals(irsaCredentials, that.irsaCredentials); } + @Override public int hashCode() { return Objects.hash( @@ -698,6 +700,7 @@ public int hashCode() { connectionTimeoutMillis, connectionTTLMillis, maxConnections, + maxSyncConnections, // fixed here connectionAcquisitionTimeoutMillis, maxRetries, throttleRetries, @@ -707,6 +710,7 @@ public int hashCode() { ); } + private static T getConfigValue(Settings settings, String clientName, Setting.AffixSetting clientSetting) { final Setting concreteSetting = clientSetting.getConcreteSettingForNamespace(clientName); return concreteSetting.get(settings); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 4b5510ad70186..100241a4909a2 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -363,6 +363,8 @@ public List> getSettings() { S3Repository.ACCESS_KEY_SETTING, S3Repository.SECRET_KEY_SETTING, S3ClientSettings.SIGNER_OVERRIDE, + // Fixed the bug in this line + S3ClientSettings.DISABLE_CHUNKED_ENCODING, S3ClientSettings.REGION, S3ClientSettings.ROLE_ARN_SETTING, S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING, diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java index 799cbe90103e5..7301b2b90e86c 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java @@ -75,6 +75,9 @@ public void testGetExecutorBuilders() throws IOException { ); } assertTrue(plugin.getSettings().contains(S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE)); + // New assertion for your bug fix + assertTrue(plugin.getSettings().contains(S3ClientSettings.DISABLE_CHUNKED_ENCODING)); + } finally { if (threadPool != null) { terminate(threadPool); From 7251e940a06615b26cc0811dbe199d6e32777a96 Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Thu, 4 Dec 2025 20:27:05 +0530 Subject: [PATCH 2/8] Expose all missing S3 client settings in repository-s3 plugin Signed-off-by: Aman Gautam --- .../repositories/s3/S3ClientSettings.java | 2 -- .../repositories/s3/S3RepositoryPlugin.java | 13 +++++++++++-- .../repositories/s3/S3RepositoryPluginTests.java | 8 ++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 5fd6b296a1731..fbfdcdc095695 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -687,7 +687,6 @@ public boolean equals(final Object o) { && Objects.equals(irsaCredentials, that.irsaCredentials); } - @Override public int hashCode() { return Objects.hash( @@ -710,7 +709,6 @@ public int hashCode() { ); } - private static T getConfigValue(Settings settings, String clientName, Setting.AffixSetting clientSetting) { final Setting concreteSetting = clientSetting.getConcreteSettingForNamespace(clientName); return concreteSetting.get(settings); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 100241a4909a2..bc1a3b489abfb 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -345,7 +345,7 @@ public Map getRepositories( @Override public List> getSettings() { return Arrays.asList( - // named s3 client configuration settings + // s3 client configuration settings S3ClientSettings.ACCESS_KEY_SETTING, S3ClientSettings.SECRET_KEY_SETTING, S3ClientSettings.SESSION_TOKEN_SETTING, @@ -363,8 +363,17 @@ public List> getSettings() { S3Repository.ACCESS_KEY_SETTING, S3Repository.SECRET_KEY_SETTING, S3ClientSettings.SIGNER_OVERRIDE, - // Fixed the bug in this line + + // required new settings + S3ClientSettings.REQUEST_TIMEOUT_SETTING, + S3ClientSettings.CONNECTION_TIMEOUT_SETTING, + S3ClientSettings.CONNECTION_TTL_SETTING, + S3ClientSettings.MAX_CONNECTIONS_SETTING, + S3ClientSettings.MAX_SYNC_CONNECTIONS_SETTING, + S3ClientSettings.CONNECTION_ACQUISITION_TIMEOUT, + S3ClientSettings.MAX_PENDING_CONNECTION_ACQUIRES, S3ClientSettings.DISABLE_CHUNKED_ENCODING, + S3ClientSettings.REGION, S3ClientSettings.ROLE_ARN_SETTING, S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING, diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java index 7301b2b90e86c..91e8bc5ef36f5 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java @@ -77,6 +77,14 @@ public void testGetExecutorBuilders() throws IOException { assertTrue(plugin.getSettings().contains(S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE)); // New assertion for your bug fix assertTrue(plugin.getSettings().contains(S3ClientSettings.DISABLE_CHUNKED_ENCODING)); + // settings added to expose missing s3.client.* configs + assertTrue(plugin.getSettings().contains(S3ClientSettings.REQUEST_TIMEOUT_SETTING)); + assertTrue(plugin.getSettings().contains(S3ClientSettings.CONNECTION_TIMEOUT_SETTING)); + assertTrue(plugin.getSettings().contains(S3ClientSettings.CONNECTION_TTL_SETTING)); + assertTrue(plugin.getSettings().contains(S3ClientSettings.MAX_CONNECTIONS_SETTING)); + assertTrue(plugin.getSettings().contains(S3ClientSettings.MAX_SYNC_CONNECTIONS_SETTING)); + assertTrue(plugin.getSettings().contains(S3ClientSettings.CONNECTION_ACQUISITION_TIMEOUT)); + assertTrue(plugin.getSettings().contains(S3ClientSettings.MAX_PENDING_CONNECTION_ACQUIRES)); } finally { if (threadPool != null) { From aeceb9a63826073e965f3bbedc3f7fe1a38c75ef Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Thu, 4 Dec 2025 20:37:00 +0530 Subject: [PATCH 3/8] Update comment for clarity Signed-off-by: Aman Gautam --- .../java/org/opensearch/repositories/s3/S3RepositoryPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index bc1a3b489abfb..700f41e590ef4 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -364,7 +364,7 @@ public List> getSettings() { S3Repository.SECRET_KEY_SETTING, S3ClientSettings.SIGNER_OVERRIDE, - // required new settings + // additional s3 client configuration settings S3ClientSettings.REQUEST_TIMEOUT_SETTING, S3ClientSettings.CONNECTION_TIMEOUT_SETTING, S3ClientSettings.CONNECTION_TTL_SETTING, From 12ac7d1b6883c02b262c5a58908c1e9f7570f560 Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Thu, 4 Dec 2025 23:08:42 +0530 Subject: [PATCH 4/8] Include max_sync_connections in S3 client refine logic Signed-off-by: Aman Gautam --- .../java/org/opensearch/repositories/s3/S3ClientSettings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index fbfdcdc095695..c3f410fefeaff 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -443,6 +443,7 @@ S3ClientSettings refine(Settings repositorySettings) { && newConnectionTTLMillis == connectionTTLMillis && newMaxConnections == maxConnections && newConnectionAcquisitionTimeoutMillis == connectionAcquisitionTimeoutMillis + && newMaxSyncConnections == maxSyncConnections // added maxSyncConnections && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials) From 07f99f13f094d1f5a3b5f81315cf5b58823cf4f1 Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Fri, 5 Dec 2025 00:09:43 +0530 Subject: [PATCH 5/8] Fix S3 settings order to prevent CI timeout on docker fixtures Signed-off-by: Aman Gautam --- .../java/org/opensearch/repositories/s3/S3RepositoryPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 700f41e590ef4..92c494cabec4b 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -373,7 +373,7 @@ public List> getSettings() { S3ClientSettings.CONNECTION_ACQUISITION_TIMEOUT, S3ClientSettings.MAX_PENDING_CONNECTION_ACQUIRES, S3ClientSettings.DISABLE_CHUNKED_ENCODING, - + S3ClientSettings.MAX_RETRIES_SETTING, S3ClientSettings.REGION, S3ClientSettings.ROLE_ARN_SETTING, S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING, From 1d6707907cc7558dac8158d49f7127213600d780 Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Fri, 5 Dec 2025 00:56:17 +0530 Subject: [PATCH 6/8] Fix duplicate S3 client settings registration by removing already registered settings Signed-off-by: Aman Gautam --- .../repositories/s3/S3RepositoryPlugin.java | 25 +++---------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 92c494cabec4b..5c8db3b59662d 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -356,37 +356,18 @@ public List> getSettings() { S3ClientSettings.PROXY_PORT_SETTING, S3ClientSettings.PROXY_USERNAME_SETTING, S3ClientSettings.PROXY_PASSWORD_SETTING, - S3ClientSettings.READ_TIMEOUT_SETTING, - S3ClientSettings.MAX_RETRIES_SETTING, S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, S3ClientSettings.USE_PATH_STYLE_ACCESS, S3Repository.ACCESS_KEY_SETTING, S3Repository.SECRET_KEY_SETTING, S3ClientSettings.SIGNER_OVERRIDE, - // additional s3 client configuration settings - S3ClientSettings.REQUEST_TIMEOUT_SETTING, - S3ClientSettings.CONNECTION_TIMEOUT_SETTING, - S3ClientSettings.CONNECTION_TTL_SETTING, - S3ClientSettings.MAX_CONNECTIONS_SETTING, +// additional s3 client configuration settings added in this PR S3ClientSettings.MAX_SYNC_CONNECTIONS_SETTING, S3ClientSettings.CONNECTION_ACQUISITION_TIMEOUT, S3ClientSettings.MAX_PENDING_CONNECTION_ACQUIRES, - S3ClientSettings.DISABLE_CHUNKED_ENCODING, - S3ClientSettings.MAX_RETRIES_SETTING, - S3ClientSettings.REGION, - S3ClientSettings.ROLE_ARN_SETTING, - S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING, - S3ClientSettings.ROLE_SESSION_NAME_SETTING, - S3ClientSettings.LEGACY_MD5_CHECKSUM_CALCULATION, - S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING, - S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING, - S3Repository.REDIRECT_LARGE_S3_UPLOAD, - S3Repository.UPLOAD_RETRY_ENABLED, - S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT, - S3Repository.PERMIT_BACKED_TRANSFER_ENABLED, - S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE - ); + S3ClientSettings.DISABLE_CHUNKED_ENCODING + ); } @Override From 673752607e38e0c55e761bfeaada5c286516bda9 Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Fri, 5 Dec 2025 17:39:57 +0530 Subject: [PATCH 7/8] Register missing S3 client settings (REQUEST_TIMEOUT, CONNECTION_TIMEOUT, TTL, MAX_CONNECTIONS) --- .../repositories/s3/S3RepositoryPlugin.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 5c8db3b59662d..d78f9c33256d4 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -356,20 +356,40 @@ public List> getSettings() { S3ClientSettings.PROXY_PORT_SETTING, S3ClientSettings.PROXY_USERNAME_SETTING, S3ClientSettings.PROXY_PASSWORD_SETTING, + S3ClientSettings.READ_TIMEOUT_SETTING, + S3ClientSettings.MAX_RETRIES_SETTING, S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, S3ClientSettings.USE_PATH_STYLE_ACCESS, S3Repository.ACCESS_KEY_SETTING, S3Repository.SECRET_KEY_SETTING, S3ClientSettings.SIGNER_OVERRIDE, -// additional s3 client configuration settings added in this PR + // additional s3 client configuration settings added in this PR + S3ClientSettings.REQUEST_TIMEOUT_SETTING, + S3ClientSettings.CONNECTION_TIMEOUT_SETTING, + S3ClientSettings.CONNECTION_TTL_SETTING, + S3ClientSettings.MAX_CONNECTIONS_SETTING, S3ClientSettings.MAX_SYNC_CONNECTIONS_SETTING, S3ClientSettings.CONNECTION_ACQUISITION_TIMEOUT, S3ClientSettings.MAX_PENDING_CONNECTION_ACQUIRES, - S3ClientSettings.DISABLE_CHUNKED_ENCODING - ); + S3ClientSettings.DISABLE_CHUNKED_ENCODING, + + S3ClientSettings.REGION, + S3ClientSettings.ROLE_ARN_SETTING, + S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING, + S3ClientSettings.ROLE_SESSION_NAME_SETTING, + S3ClientSettings.LEGACY_MD5_CHECKSUM_CALCULATION, + S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING, + S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING, + S3Repository.REDIRECT_LARGE_S3_UPLOAD, + S3Repository.UPLOAD_RETRY_ENABLED, + S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT, + S3Repository.PERMIT_BACKED_TRANSFER_ENABLED, + S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE + ); } + @Override public void reload(Settings settings) { // secure settings should be readable From 92d39feadb3352e2218f49687284263e675cf172 Mon Sep 17 00:00:00 2001 From: Aman Gautam Date: Fri, 5 Dec 2025 17:54:26 +0530 Subject: [PATCH 8/8] CI trigger: retrigger check-result --- plugins/repository-s3/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/repository-s3/README.md b/plugins/repository-s3/README.md index 9bd2fb4bed2e1..a8e0882164ae1 100644 --- a/plugins/repository-s3/README.md +++ b/plugins/repository-s3/README.md @@ -27,3 +27,4 @@ Optional environment variables: - `amazon_s3_path_style_access`: Possible values true or false. Default is false. - `amazon_s3_endpoint`: s3 custom endpoint url if aws s3 default endpoint is not being used. +