diff --git a/build.gradle b/build.gradle index ecc794f5a2..4d9549cba8 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ buildscript { ext { opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") - opensearch_version = System.getProperty("opensearch.version", "2.19.3-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.19.4-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") // 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT @@ -71,6 +71,7 @@ allprojects { } + subprojects { configurations { testImplementation.extendsFrom compileOnly @@ -80,6 +81,18 @@ subprojects { // Force spotless depending on newer version of guava due to CVE-2023-2976. Remove after spotless upgrades. resolutionStrategy.force "com.google.guava:guava:32.1.3-jre" resolutionStrategy.force 'org.apache.commons:commons-compress:1.26.0' + resolutionStrategy.force "org.apache.commons:commons-lang3:${versions.commonslang}" + resolutionStrategy.force 'software.amazon.awssdk:bom:2.32.29' + + resolutionStrategy.force 'io.netty:netty-buffer:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-codec:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-codec-http:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-codec-http2:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-common:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-handler:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-resolver:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-transport:4.1.125.Final' + resolutionStrategy.force 'io.netty:netty-transport-native-unix-common:4.1.125.Final' } } diff --git a/common/build.gradle b/common/build.gradle index b84024b2b4..660df0b1cb 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -44,6 +44,10 @@ dependencies { compileOnly group: 'com.networknt' , name: 'json-schema-validator', version: '1.4.0' // Multi-tenant SDK Client compileOnly "org.opensearch:opensearch-remote-metadata-sdk:${opensearch_build}" + compileOnly (group: 'software.amazon.awssdk', name: 'netty-nio-client', version: "2.32.29") { + exclude(group: 'org.reactivestreams', module: 'reactive-streams') + exclude(group: 'org.slf4j', module: 'slf4j-api') + } } lombok { diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/httpclient/MLHttpClientFactory.java b/common/src/main/java/org/opensearch/ml/common/httpclient/MLHttpClientFactory.java similarity index 58% rename from ml-algorithms/src/main/java/org/opensearch/ml/engine/httpclient/MLHttpClientFactory.java rename to common/src/main/java/org/opensearch/ml/common/httpclient/MLHttpClientFactory.java index ffc95c30de..109a5de5f8 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/httpclient/MLHttpClientFactory.java +++ b/common/src/main/java/org/opensearch/ml/common/httpclient/MLHttpClientFactory.java @@ -3,19 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.ml.engine.httpclient; +package org.opensearch.ml.common.httpclient; import java.net.Inet4Address; import java.net.InetAddress; import java.net.UnknownHostException; -import java.security.AccessController; -import java.security.PrivilegedActionException; -import java.security.PrivilegedExceptionAction; import java.time.Duration; import java.util.Arrays; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; +import org.opensearch.common.util.concurrent.ThreadContextAccess; + import lombok.extern.log4j.Log4j2; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; @@ -24,19 +23,15 @@ public class MLHttpClientFactory { public static SdkAsyncHttpClient getAsyncHttpClient(Duration connectionTimeout, Duration readTimeout, int maxConnections) { - try { - return AccessController - .doPrivileged( - (PrivilegedExceptionAction) () -> NettyNioAsyncHttpClient - .builder() - .connectionTimeout(connectionTimeout) - .readTimeout(readTimeout) - .maxConcurrency(maxConnections) - .build() - ); - } catch (PrivilegedActionException e) { - return null; - } + return ThreadContextAccess + .doPrivileged( + () -> NettyNioAsyncHttpClient + .builder() + .connectionTimeout(connectionTimeout) + .readTimeout(readTimeout) + .maxConcurrency(maxConnections) + .build() + ); } /** @@ -50,7 +45,7 @@ public static SdkAsyncHttpClient getAsyncHttpClient(Duration connectionTimeout, public static void validate(String protocol, String host, int port, AtomicBoolean connectorPrivateIpEnabled) throws UnknownHostException { if (protocol != null && !"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) { - log.error("Remote inference protocol is not http or https: " + protocol); + log.error("Remote inference protocol is not http or https: {}", protocol); throw new IllegalArgumentException("Protocol is not http or https: " + protocol); } // When port is not specified, the default port is -1, and we need to set it to 80 or 443 based on protocol. @@ -62,7 +57,7 @@ public static void validate(String protocol, String host, int port, AtomicBoolea } } if (port < 0 || port > 65536) { - log.error("Remote inference port out of range: " + port); + log.error("Remote inference port out of range: {}", port); throw new IllegalArgumentException("Port out of range: " + port); } validateIp(host, connectorPrivateIpEnabled); @@ -71,7 +66,7 @@ public static void validate(String protocol, String host, int port, AtomicBoolea private static void validateIp(String hostName, AtomicBoolean connectorPrivateIpEnabled) throws UnknownHostException { InetAddress[] addresses = InetAddress.getAllByName(hostName); if ((connectorPrivateIpEnabled == null || !connectorPrivateIpEnabled.get()) && hasPrivateIpAddress(addresses)) { - log.error("Remote inference host name has private ip address: " + hostName); + log.error("Remote inference host name has private ip address: {}", hostName); throw new IllegalArgumentException("Remote inference host name has private ip address: " + hostName); } } @@ -83,23 +78,8 @@ private static boolean hasPrivateIpAddress(InetAddress[] ipAddress) { if (bytes.length != 4) { return true; } else { - int firstOctets = bytes[0] & 0xff; - int firstInOctal = parseWithOctal(String.valueOf(firstOctets)); - int firstInHex = Integer.parseInt(String.valueOf(firstOctets), 16); - if (firstInOctal == 127 || firstInHex == 127) { - return bytes[1] == 0 && bytes[2] == 0 && bytes[3] == 1; - } else if (firstInOctal == 10 || firstInHex == 10) { + if (isPrivateIPv4(bytes)) { return true; - } else if (firstInOctal == 172 || firstInHex == 172) { - int secondOctets = bytes[1] & 0xff; - int secondInOctal = parseWithOctal(String.valueOf(secondOctets)); - int secondInHex = Integer.parseInt(String.valueOf(secondOctets), 16); - return (secondInOctal >= 16 && secondInOctal <= 32) || (secondInHex >= 16 && secondInHex <= 32); - } else if (firstInOctal == 192 || firstInHex == 192) { - int secondOctets = bytes[1] & 0xff; - int secondInOctal = parseWithOctal(String.valueOf(secondOctets)); - int secondInHex = Integer.parseInt(String.valueOf(secondOctets), 16); - return secondInOctal == 168 || secondInHex == 168; } } } @@ -107,11 +87,14 @@ private static boolean hasPrivateIpAddress(InetAddress[] ipAddress) { return Arrays.stream(ipAddress).anyMatch(x -> x.isSiteLocalAddress() || x.isLoopbackAddress() || x.isAnyLocalAddress()); } - private static int parseWithOctal(String input) { - try { - return Integer.parseInt(input, 8); - } catch (NumberFormatException e) { - return Integer.parseInt(input); - } + private static boolean isPrivateIPv4(byte[] bytes) { + int first = bytes[0] & 0xff; + int second = bytes[1] & 0xff; + + // 127.0.0.1, 10.x.x.x, 172.16-31.x.x, 192.168.x.x, 169.254.x.x + return (first == 10) + || (first == 172 && second >= 16 && second <= 31) + || (first == 192 && second == 168) + || (first == 169 && second == 254); } } diff --git a/common/src/test/java/org/opensearch/ml/common/httpclient/MLHttpClientFactoryTests.java b/common/src/test/java/org/opensearch/ml/common/httpclient/MLHttpClientFactoryTests.java new file mode 100644 index 0000000000..1c01172344 --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/httpclient/MLHttpClientFactoryTests.java @@ -0,0 +1,191 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.httpclient; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; + +public class MLHttpClientFactoryTests { + + private static final String TEST_HOST = "api.openai.com"; + private static final String HTTP = "http"; + private static final String HTTPS = "https"; + private static final AtomicBoolean PRIVATE_IP_DISABLED = new AtomicBoolean(false); + private static final AtomicBoolean PRIVATE_IP_ENABLED = new AtomicBoolean(true); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void test_getSdkAsyncHttpClient_success() { + SdkAsyncHttpClient client = MLHttpClientFactory.getAsyncHttpClient(Duration.ofSeconds(100), Duration.ofSeconds(100), 100); + assertNotNull(client); + } + + @Test + public void test_invalidIP_localHost_privateIPDisabled() { + IllegalArgumentException e1 = assertThrows( + IllegalArgumentException.class, + () -> MLHttpClientFactory.validate(HTTP, "127.0.0.1", 80, PRIVATE_IP_DISABLED) + ); + assertEquals("Remote inference host name has private ip address: 127.0.0.1", e1.getMessage()); + + IllegalArgumentException e2 = assertThrows( + IllegalArgumentException.class, + () -> MLHttpClientFactory.validate(HTTP, "192.168.0.1", 80, PRIVATE_IP_DISABLED) + ); + assertEquals("Remote inference host name has private ip address: 192.168.0.1", e2.getMessage()); + + IllegalArgumentException e3 = assertThrows( + IllegalArgumentException.class, + () -> MLHttpClientFactory.validate(HTTP, "169.254.0.1", 80, PRIVATE_IP_DISABLED) + ); + assertEquals("Remote inference host name has private ip address: 169.254.0.1", e3.getMessage()); + + IllegalArgumentException e4 = assertThrows( + IllegalArgumentException.class, + () -> MLHttpClientFactory.validate(HTTP, "172.16.0.1", 80, PRIVATE_IP_DISABLED) + ); + assertEquals("Remote inference host name has private ip address: 172.16.0.1", e4.getMessage()); + + IllegalArgumentException e5 = assertThrows( + IllegalArgumentException.class, + () -> MLHttpClientFactory.validate(HTTP, "172.31.0.1", 80, PRIVATE_IP_DISABLED) + ); + assertEquals("Remote inference host name has private ip address: 172.31.0.1", e5.getMessage()); + } + + @Test + public void test_validateIp_validIp_noException() throws Exception { + MLHttpClientFactory.validate(HTTP, TEST_HOST, 80, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTPS, TEST_HOST, 443, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, "127.0.0.1", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTPS, "127.0.0.1", 443, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "177.16.0.1", 80, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, "177.0.1.1", 80, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, "177.0.0.2", 80, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, "::ffff", 80, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, "172.32.0.1", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "172.2097152", 80, PRIVATE_IP_ENABLED); + } + + @Test + public void test_validateIp_rarePrivateIp_throwException() throws Exception { + try { + MLHttpClientFactory.validate(HTTP, "0254.020.00.01", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "172.1048577", 80, PRIVATE_IP_DISABLED); + } catch (Exception e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "2886729729", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "192.11010049", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "3232300545", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "0:0:0:0:0:ffff:127.0.0.1", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "153.24.76.232", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "177.0.0.1", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + + try { + MLHttpClientFactory.validate(HTTP, "12.16.2.3", 80, PRIVATE_IP_DISABLED); + } catch (IllegalArgumentException e) { + assertNotNull(e); + } + } + + @Test + public void test_validateIp_rarePrivateIp_NotThrowException() throws Exception { + MLHttpClientFactory.validate(HTTP, "0254.020.00.01", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTPS, "0254.020.00.01", 443, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "172.1048577", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "2886729729", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "192.11010049", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "3232300545", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "0:0:0:0:0:ffff:127.0.0.1", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTPS, "0:0:0:0:0:ffff:127.0.0.1", 443, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "153.24.76.232", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTP, "10.24.76.186", 80, PRIVATE_IP_ENABLED); + MLHttpClientFactory.validate(HTTPS, "10.24.76.186", 443, PRIVATE_IP_ENABLED); + } + + @Test + public void test_validateSchemaAndPort_success() throws Exception { + MLHttpClientFactory.validate(HTTP, TEST_HOST, 80, PRIVATE_IP_DISABLED); + } + + @Test + public void test_validateSchemaAndPort_notAllowedSchema_throwException() throws Exception { + expectedException.expect(IllegalArgumentException.class); + MLHttpClientFactory.validate("ftp", TEST_HOST, 80, PRIVATE_IP_DISABLED); + } + + @Test + public void test_validateSchemaAndPort_portNotInRange1_throwException() throws Exception { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Port out of range: 65537"); + MLHttpClientFactory.validate(HTTPS, TEST_HOST, 65537, PRIVATE_IP_DISABLED); + } + + @Test + public void test_validateSchemaAndPort_portNotInRange2_throwException() throws Exception { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Port out of range: -10"); + MLHttpClientFactory.validate(HTTP, TEST_HOST, -10, PRIVATE_IP_DISABLED); + } + + @Test + public void test_validatePort_boundaries_success() throws Exception { + MLHttpClientFactory.validate(HTTP, TEST_HOST, 65536, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, TEST_HOST, 0, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTP, TEST_HOST, -1, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(HTTPS, TEST_HOST, -1, PRIVATE_IP_DISABLED); + MLHttpClientFactory.validate(null, TEST_HOST, -1, PRIVATE_IP_DISABLED); + } + +} diff --git a/ml-algorithms/build.gradle b/ml-algorithms/build.gradle index 9405f6a9ee..9800a8bb55 100644 --- a/ml-algorithms/build.gradle +++ b/ml-algorithms/build.gradle @@ -58,6 +58,7 @@ dependencies { // Multi-tenant SDK Client implementation "org.opensearch:opensearch-remote-metadata-sdk:${opensearch_build}" implementation 'commons-beanutils:commons-beanutils:1.11.0' + implementation "org.opensearch:opensearch-remote-metadata-sdk-ddb-client:${opensearch_build}" def os = DefaultNativePlatform.currentOperatingSystem //arm/macos doesn't support GPU @@ -71,24 +72,27 @@ dependencies { } } - implementation platform('software.amazon.awssdk:bom:2.30.18') - api 'software.amazon.awssdk:auth:2.30.18' - implementation 'software.amazon.awssdk:apache-client' + implementation platform(group: 'software.amazon.awssdk', name: 'bom', version:"2.32.29") + api "software.amazon.awssdk:auth:2.32.29" + implementation group: 'software.amazon.awssdk', name:'apache-client', version: "2.32.29" + implementation (group: 'software.amazon.awssdk', name: 'bedrockruntime', version: "2.32.29") { + exclude group: 'io.netty' + } implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') { exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on' } - implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1' - - compileOnly group: 'software.amazon.awssdk', name: 'aws-core', version: "2.30.18" - compileOnly group: 'software.amazon.awssdk', name: 's3', version: "2.30.18" - compileOnly group: 'software.amazon.awssdk', name: 'regions', version: "2.30.18" + // needed by aws-encryption-sdk-java + implementation "org.bouncycastle:bc-fips:2.1.1" + compileOnly group: 'software.amazon.awssdk', name: 'aws-core', version: "2.32.29" + compileOnly group: 'software.amazon.awssdk', name: 's3', version: "2.32.29" + compileOnly group: 'software.amazon.awssdk', name: 'regions', version: "2.32.29" implementation ('com.jayway.jsonpath:json-path:2.9.0') { exclude group: 'net.minidev', module: 'json-smart' } implementation('net.minidev:json-smart:2.5.2') implementation group: 'org.json', name: 'json', version: '20231013' - implementation group: 'software.amazon.awssdk', name: 'netty-nio-client', version: "2.30.18" + implementation group: 'software.amazon.awssdk', name: 'netty-nio-client', version: "2.32.29" testImplementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") testImplementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") testImplementation group: 'com.networknt' , name: 'json-schema-validator', version: '1.4.0' @@ -101,7 +105,7 @@ lombok { configurations.all { resolutionStrategy.force 'com.google.protobuf:protobuf-java:3.25.5' resolutionStrategy.force 'org.apache.commons:commons-compress:1.26.0' - resolutionStrategy.force 'software.amazon.awssdk:bom:2.30.18' + resolutionStrategy.force group: 'software.amazon.awssdk', name:'bom', version:"2.32.29" resolutionStrategy.force 'commons-beanutils:commons-beanutils:1.11.0' } diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java index e0bcd1bc73..e7e5b7c71f 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java @@ -24,11 +24,11 @@ import org.opensearch.ml.common.connector.AwsConnector; import org.opensearch.ml.common.connector.Connector; import org.opensearch.ml.common.exception.MLException; +import org.opensearch.ml.common.httpclient.MLHttpClientFactory; import org.opensearch.ml.common.input.MLInput; import org.opensearch.ml.common.model.MLGuard; import org.opensearch.ml.common.output.model.ModelTensors; import org.opensearch.ml.engine.annotation.ConnectorExecutor; -import org.opensearch.ml.engine.httpclient.MLHttpClientFactory; import org.opensearch.script.ScriptService; import lombok.Getter; diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/HttpJsonConnectorExecutor.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/HttpJsonConnectorExecutor.java index 5ac0245701..57f2a7019c 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/HttpJsonConnectorExecutor.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/HttpJsonConnectorExecutor.java @@ -26,11 +26,11 @@ import org.opensearch.ml.common.connector.Connector; import org.opensearch.ml.common.connector.HttpConnector; import org.opensearch.ml.common.exception.MLException; +import org.opensearch.ml.common.httpclient.MLHttpClientFactory; import org.opensearch.ml.common.input.MLInput; import org.opensearch.ml.common.model.MLGuard; import org.opensearch.ml.common.output.model.ModelTensors; import org.opensearch.ml.engine.annotation.ConnectorExecutor; -import org.opensearch.ml.engine.httpclient.MLHttpClientFactory; import org.opensearch.script.ScriptService; import lombok.Getter; diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/httpclient/MLHttpClientFactoryTests.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/httpclient/MLHttpClientFactoryTests.java deleted file mode 100644 index 1d79ac995e..0000000000 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/httpclient/MLHttpClientFactoryTests.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.ml.engine.httpclient; - -import static org.junit.Assert.assertNotNull; - -import java.time.Duration; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import software.amazon.awssdk.http.async.SdkAsyncHttpClient; - -public class MLHttpClientFactoryTests { - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Test - public void test_getSdkAsyncHttpClient_success() { - SdkAsyncHttpClient client = MLHttpClientFactory.getAsyncHttpClient(Duration.ofSeconds(100), Duration.ofSeconds(100), 100); - assertNotNull(client); - } - - @Test - public void test_validateIp_validIp_noException() throws Exception { - AtomicBoolean privateIpEnabled = new AtomicBoolean(false); - MLHttpClientFactory.validate("http", "api.openai.com", 80, privateIpEnabled); - } - - @Test - public void test_validateIp_rarePrivateIp_throwException() throws Exception { - AtomicBoolean privateIpEnabled = new AtomicBoolean(false); - try { - MLHttpClientFactory.validate("http", "0254.020.00.01", 80, privateIpEnabled); - } catch (IllegalArgumentException e) { - assertNotNull(e); - } - - try { - MLHttpClientFactory.validate("http", "172.1048577", 80, privateIpEnabled); - } catch (Exception e) { - assertNotNull(e); - } - - try { - MLHttpClientFactory.validate("http", "2886729729", 80, privateIpEnabled); - } catch (IllegalArgumentException e) { - assertNotNull(e); - } - - try { - MLHttpClientFactory.validate("http", "192.11010049", 80, privateIpEnabled); - } catch (IllegalArgumentException e) { - assertNotNull(e); - } - - try { - MLHttpClientFactory.validate("http", "3232300545", 80, privateIpEnabled); - } catch (IllegalArgumentException e) { - assertNotNull(e); - } - - try { - MLHttpClientFactory.validate("http", "0:0:0:0:0:ffff:127.0.0.1", 80, privateIpEnabled); - } catch (IllegalArgumentException e) { - assertNotNull(e); - } - - try { - MLHttpClientFactory.validate("http", "153.24.76.232", 80, privateIpEnabled); - } catch (IllegalArgumentException e) { - assertNotNull(e); - } - } - - @Test - public void test_validateIp_rarePrivateIp_NotThrowException() throws Exception { - AtomicBoolean privateIpEnabled = new AtomicBoolean(true); - MLHttpClientFactory.validate("http", "0254.020.00.01", 80, privateIpEnabled); - MLHttpClientFactory.validate("http", "172.1048577", 80, privateIpEnabled); - MLHttpClientFactory.validate("http", "2886729729", 80, privateIpEnabled); - MLHttpClientFactory.validate("http", "192.11010049", 80, privateIpEnabled); - MLHttpClientFactory.validate("http", "3232300545", 80, privateIpEnabled); - MLHttpClientFactory.validate("http", "0:0:0:0:0:ffff:127.0.0.1", 80, privateIpEnabled); - MLHttpClientFactory.validate("http", "153.24.76.232", 80, privateIpEnabled); - } - - @Test - public void test_validateSchemaAndPort_success() throws Exception { - MLHttpClientFactory.validate("http", "api.openai.com", 80, new AtomicBoolean(false)); - } - - @Test - public void test_validateSchemaAndPort_notAllowedSchema_throwException() throws Exception { - expectedException.expect(IllegalArgumentException.class); - MLHttpClientFactory.validate("ftp", "api.openai.com", 80, new AtomicBoolean(false)); - } - - @Test - public void test_validateSchemaAndPort_portNotInRange_throwException() throws Exception { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Port out of range: 65537"); - MLHttpClientFactory.validate("https", "api.openai.com", 65537, new AtomicBoolean(false)); - } - -} diff --git a/plugin/build.gradle b/plugin/build.gradle index a352702c2e..b1a0b05a27 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -54,15 +54,15 @@ dependencies { implementation project(':opensearch-ml-memory') compileOnly "com.google.guava:guava:32.1.3-jre" - implementation group: 'software.amazon.awssdk', name: 'aws-core', version: "2.30.18" - implementation group: 'software.amazon.awssdk', name: 's3', version: "2.30.18" - implementation group: 'software.amazon.awssdk', name: 'regions', version: "2.30.18" + implementation group: 'software.amazon.awssdk', name: 'aws-core', version: "2.32.29" + implementation group: 'software.amazon.awssdk', name: 's3', version: "2.32.29" + implementation group: 'software.amazon.awssdk', name: 'regions', version: "2.32.29" - implementation group: 'software.amazon.awssdk', name: 'aws-xml-protocol', version: "2.30.18" + implementation group: 'software.amazon.awssdk', name: 'aws-xml-protocol', version: "2.32.29" - implementation group: 'software.amazon.awssdk', name: 'aws-query-protocol', version: "2.30.18" + implementation group: 'software.amazon.awssdk', name: 'aws-query-protocol', version: "2.32.29" - implementation group: 'software.amazon.awssdk', name: 'protocol-core', version: "2.30.18" + implementation group: 'software.amazon.awssdk', name: 'protocol-core', version: "2.32.29" zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" diff --git a/search-processors/build.gradle b/search-processors/build.gradle index e9fbc9a585..2f9f8bb380 100644 --- a/search-processors/build.gradle +++ b/search-processors/build.gradle @@ -27,6 +27,7 @@ repositories { mavenLocal() } + dependencies { implementation project(path: ":${rootProject.name}-common", configuration: 'shadow') compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"