Use Iceberg AWS bundle instead of standalone AWS SDK JARs (#243)

Author: ferenc-csaky

flink-sql-runner/pom.xml

@@ -72,6 +72,14 @@
       <scope>provided</scope>
     </dependency>
 
+    <!-- Will be shipped as Flink lib -->
+    <dependency>
+      <groupId>org.apache.iceberg</groupId>
+      <artifactId>iceberg-aws-bundle</artifactId>
+      <version>${iceberg.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
     <!-- Required for Iceberg to work -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -90,28 +98,6 @@
       </exclusions>
     </dependency>
 
-    <!-- AWS SDK v2 required for Iceberg S3FileIO -->
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>s3</artifactId>
-      <version>${aws.sdk.version}</version>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>sts</artifactId>
-      <version>${aws.sdk.version}</version>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>software.amazon.awssdk</groupId>
-      <artifactId>url-connection-client</artifactId>
-      <version>${aws.sdk.version}</version>
-      <scope>provided</scope>
-    </dependency>
-
     <!-- Add logging framework, to produce console output when running in the IDE. -->
     <!-- These dependencies are excluded from the application JAR by default. -->
     <dependency>
@@ -462,7 +448,7 @@
         </executions>
       </plugin>
 
-      <!-- Copying UDF/SystemFunction JARs -->
+      <!-- Copying S3, Iceberg, UDF JARs -->
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
@@ -493,26 +479,27 @@
             </configuration>
           </execution>
           <execution>
-            <id>copy-hdfs-client-for-iceberg</id>
+            <id>copy-iceberg-aws-bundle</id>
             <goals>
               <goal>copy-dependencies</goal>
             </goals>
             <phase>process-resources</phase>
             <configuration>
-              <includeGroupIds>org.apache.hadoop</includeGroupIds>
-              <includeArtifactIds>hadoop-hdfs-client</includeArtifactIds>
+              <includeGroupIds>org.apache.iceberg</includeGroupIds>
+              <includeArtifactIds>iceberg-aws-bundle</includeArtifactIds>
               <outputDirectory>${project.build.directory}</outputDirectory>
             </configuration>
           </execution>
           <execution>
-            <id>copy-aws-sdk-for-iceberg</id>
+            <id>copy-hdfs-client-for-iceberg</id>
             <goals>
               <goal>copy-dependencies</goal>
             </goals>
             <phase>process-resources</phase>
             <configuration>
-              <includeGroupIds>software.amazon.awssdk</includeGroupIds>
-              <outputDirectory>${project.build.directory}/aws-sdk</outputDirectory>
+              <includeGroupIds>org.apache.hadoop</includeGroupIds>
+              <includeArtifactIds>hadoop-hdfs-client</includeArtifactIds>
+              <outputDirectory>${project.build.directory}</outputDirectory>
             </configuration>
           </execution>
           <execution>

flink-sql-runner/src/main/docker/Dockerfile

@@ -21,7 +21,7 @@ RUN mkdir -p /opt/flink/plugins/flink-sql-runner
 COPY flink-s3-fs-hadoop-*.jar /opt/flink/lib
 COPY hadoop-hdfs-client-*.jar /opt/flink/lib
 COPY iceberg-flink-runtime-*.jar /opt/flink/lib
-COPY aws-sdk/*.jar /opt/flink/lib/
+COPY iceberg-aws-bundle-*.jar /opt/flink/lib
 COPY stdlib-utils-*.jar /opt/flink/lib
 COPY flink-sql-runner.uber.jar /opt/flink/plugins/flink-sql-runner
 COPY entrypoint.sh /entrypoint.sh

pom.xml

@@ -94,7 +94,6 @@
     <assertj.version>3.27.4</assertj.version>
     <auto.service.version>1.1.1</auto.service.version>
     <awaitility.version>4.3.0</awaitility.version>
-    <aws.sdk.version>2.20.160</aws.sdk.version>
     <commons-exec.version>1.5.0</commons-exec.version>
     <commons-math3.version>3.6.1</commons-math3.version>
     <feign.version>13.5</feign.version>