SOLR-15599: Upgrade AWS SDK from v1 to v2 (#271)

Also removed woodstox-core-asl dependency, and replaced with com.fasterxml.woodstox:woodstox-core:6.2.4, the newer version of the dependency.

Author: HoustonPutman

solr/CHANGES.txt

@@ -396,6 +396,8 @@ Improvements
 
 * SOLR-9853: Project multi-valued fields in SQL query results (Timothy Potter)
 
+* SOLR-15599: Upgrade AWS SDK from v1 to v2 for S3 Repository (Houston Putman)
+
 Optimizations
 ---------------------
 * SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)
@@ -437,6 +439,8 @@ Other Changes
 
 * SOLR-15324: Upgrade Jaeger dependency from 1.1.0 to 1.6.0 and thus also libthrift to 0.14.1 (janhoy)
 
+* SOLR-15599: woodstox-core-asl:4.4.1 (org.codehaus) replaced with woodstox-core:6.2.4 (com.fasterxml) (Houston Putman)
+
 ==================  8.9.0 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

solr/contrib/s3-repository/README.md

@@ -7,14 +7,16 @@ This S3 repository is a backup repository implementation designed to provide bac
 
 Add this to your `solr.xml`:
 
+```xml
     <backup>
         <repository name="s3" class="org.apache.solr.s3.S3BackupRepository" default="false">
             <str name="s3.bucket.name">BUCKET_NAME</str>
             <str name="s3.region">us-west-2</str>
         </repository>
     </backup>
+```
 
-This plugin uses the [default AWS credentials provider chain](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html), so ensure that your credentials are set appropriately (e.g., via env var, or in `~/.aws/credentials`, etc.).
+This plugin uses the [default AWS credentials provider chain](https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html), so ensure that your credentials are set appropriately (e.g., via env var, or in `~/.aws/credentials`, etc.).
 
 ## Testing locally
 
@@ -25,13 +27,15 @@ To run / test locally, first spin up S3Mock:
 
 Add this to your `solr.xml`:
 
+```xml
     <backup>
         <repository name="s3" class="org.apache.solr.s3.S3BackupRepository" default="false">
             <str name="s3.endpoint">http://localhost:9090</str>
             <str name="s3.bucket.name">TEST_BUCKET</str>
             <str name="s3.region">us-east-1</str>
         </repository>
     </backup>
+```
 
 Start Solr, and create a collection (e.g., "foo"). Then hit the following URL, which will take a backup and persist it in S3Mock under the name `test`:
 
@@ -45,13 +49,15 @@ http://localhost:8983/solr/admin/collections?action=RESTORE&repository=s3&locati
 
 If you are also running Solr in a docker image, and need to set the endpoint of S3Mock to be different than `localhost`, then add the following under `<repository>`:
 
+```xml
     <backup>
         <repository name="s3" class="org.apache.solr.s3.S3BackupRepository" default="false">
             <str name="s3.bucket.name">TEST_BUCKET</str>
             <str name="s3.endpoint">http://host.docker.internal:9090</str>
             <str name="s3.region">us-east-1</str>
         </repository>
     </backup>
+```
 
 This works for the regular S3 backup repository as well (not mock).
 But the plugin only provides official support for AWS S3, not _S3 compatible_ products.

solr/contrib/s3-repository/build.gradle

@@ -23,14 +23,17 @@ description = 'S3 Repository'
 dependencies {
     api project(':solr:core')
 
-    implementation platform('com.amazonaws:aws-java-sdk-bom')
-
-    implementation ('com.amazonaws:aws-java-sdk-core') { exclude group: 'commons-codec', module: 'commons-codec' }
-    implementation ('com.amazonaws:aws-java-sdk-s3') { exclude group: 'commons-codec', module: 'commons-codec' }
-
-    // Not strictly needed, but provides better performance according to AWS SDK
-    // Can remove this when we upgrade to AWS SDK v2
-    implementation ('javax.xml.bind:jaxb-api') { transitive = false }
+    implementation platform(group: 'software.amazon.awssdk', name: 'bom')
+    implementation (group: 'software.amazon.awssdk', name: 'auth')
+    implementation (group: 'software.amazon.awssdk', name: 'apache-client')
+    implementation (group: 'software.amazon.awssdk', name: 'aws-core')
+    implementation (group: 'software.amazon.awssdk', name: 's3') {
+        exclude group: 'software.amazon.awssdk', module: 'netty-nio-client'
+    }
+    implementation (group: 'software.amazon.awssdk', name: 'sdk-core')
+    implementation (group: 'software.amazon.awssdk', name: 'protocol-core')
+    runtimeOnly (group: 'com.fasterxml.woodstox', name: 'woodstox-core')
+    runtimeOnly (group: 'org.codehaus.woodstox', name: 'stax2-api')
 
     testImplementation('com.adobe.testing:s3mock-junit4') {
         // Don't pull in separate versions of these libs, just use what Solr already has
@@ -41,6 +44,8 @@ dependencies {
         exclude group: 'commons-io', module: 'commons-io'
         exclude group: 'commons-codec', module: 'commons-codec'
         exclude group: 'org.apache.commons', module: 'commons-lang3'
+        exclude group: 'software.amazon.awssdk', module: 'netty-nio-client'
+        exclude group: 'org.codehaus.woodstox', module: 'stax2-api'
     }
 
     testImplementation project(':solr:test-framework')
@@ -52,9 +57,11 @@ test {
     environment "spring.jmx.enabled", "false"
 
     // Without this, randomizedtesting will think our tests leak threads (b/c S3Mock's Jetty hasn't bothered to clean up yet)
-    environment "server.jetty.threads.idle-timeout", "3s"
+    environment "server.jetty.threads.idle-timeout", "1s"
 
     // Reduce logging noise during the tests
     environment "aws.accessKeyId", "foo"
-    environment "aws.secretKey", "bar"
+    environment "aws.secretAccessKey", "bar"
+    environment "aws.sharedCredentialsFile", "conf/credentials"
+    environment "aws.configFile", "conf/config"
 }

solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3BackupRepository.java

@@ -87,9 +87,9 @@ public URI createURI(String location) {
       if (location.startsWith(S3_SCHEME + ":")) {
         result = new URI(location);
       } else if (location.startsWith("/")) {
-        result = new URI(S3_SCHEME, null, location, null);
+        result = new URI(S3_SCHEME, "", location, null);
       } else {
-        result = new URI(S3_SCHEME, null, "/" + location, null);
+        result = new URI(S3_SCHEME, "", "/" + location, null);
       }
       return result;
     } catch (URISyntaxException ex) {
@@ -124,9 +124,11 @@ public URI resolve(URI baseUri, String... pathComponents) {
   @Override
   public URI resolveDirectory(URI baseUri, String... pathComponents) {
     if (pathComponents.length > 0) {
-      pathComponents[pathComponents.length - 1] = pathComponents[pathComponents.length - 1] + "/";
+      if (!pathComponents[pathComponents.length - 1].endsWith("/")) {
+        pathComponents[pathComponents.length - 1] = pathComponents[pathComponents.length - 1] + "/";
+      }
     } else {
-      if (!baseUri.getPath().endsWith("/")) {
+      if (!baseUri.toString().endsWith("/")) {
         baseUri = URI.create(baseUri + "/");
       }
     }

solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3BackupRepositoryConfig.java

@@ -29,26 +29,26 @@ public class S3BackupRepositoryConfig {
   public static final String BUCKET_NAME = "s3.bucket.name";
   public static final String REGION = "s3.region";
   public static final String ENDPOINT = "s3.endpoint";
-  public static final String PROXY_HOST = "s3.proxy.host";
-  public static final String PROXY_PORT = "s3.proxy.port";
+  public static final String PROXY_URL = "s3.proxy.url";
+  public static final String PROXY_USE_SYSTEM_SETTINGS = "s3.proxy.useSystemSettings";
 
   private final String bucketName;
   private final String region;
-  private final String proxyHost;
-  private final int proxyPort;
+  private final String proxyURL;
+  private final boolean proxyUseSystemSettings;
   private final String endpoint;
 
   public S3BackupRepositoryConfig(NamedList<?> config) {
     region = getStringConfig(config, REGION);
     bucketName = getStringConfig(config, BUCKET_NAME);
-    proxyHost = getStringConfig(config, PROXY_HOST);
-    proxyPort = getIntConfig(config, PROXY_PORT);
+    proxyURL = getStringConfig(config, PROXY_URL);
+    proxyUseSystemSettings = getBooleanConfig(config, PROXY_USE_SYSTEM_SETTINGS, true);
     endpoint = getStringConfig(config, ENDPOINT);
   }
 
   /** Construct a {@link S3StorageClient} from the provided config. */
   public S3StorageClient buildClient() {
-    return new S3StorageClient(bucketName, region, proxyHost, proxyPort, endpoint);
+    return new S3StorageClient(bucketName, region, proxyURL, proxyUseSystemSettings, endpoint);
   }
 
   private static String getStringConfig(NamedList<?> config, String property) {
@@ -73,10 +73,14 @@ private static int getIntConfig(NamedList<?> config, String property) {
 
   /** If the property as any other value than 'true' or 'TRUE', this will default to false. */
   private static boolean getBooleanConfig(NamedList<?> config, String property) {
+    return getBooleanConfig(config, property, false);
+  }
+
+  private static boolean getBooleanConfig(NamedList<?> config, String property, boolean def) {
     String envProp = System.getenv().get(toEnvVar(property));
     if (envProp == null) {
       Boolean configProp = config.getBooleanArg(property);
-      return configProp != null && configProp;
+      return configProp == null ? def : configProp;
     } else {
       return Boolean.parseBoolean(envProp);
     }