3.3.x < hadoop <= 3.4.1

Author: kevinjqliu

dev/hive/Dockerfile

@@ -17,15 +17,15 @@ FROM openjdk:8-jre-slim AS build
 
 RUN apt-get update -qq && apt-get -qq -y install curl
 
-ENV HADOOP_VERSION=3.3.6
+ENV HADOOP_VERSION=3.4.1
 ENV AWS_SDK_BUNDLE=1.12.753
 
 RUN curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /tmp/hadoop-aws-${HADOOP_VERSION}.jar
 RUN curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar -Lo /tmp/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar
 
 FROM apache/hive:4.0.0
 
-ENV HADOOP_VERSION=3.3.6
+ENV HADOOP_VERSION=3.4.1
 ENV AWS_SDK_BUNDLE=1.12.753
 
 COPY --from=build /tmp/hadoop-aws-${HADOOP_VERSION}.jar /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar

tests/conftest.py

@@ -2535,7 +2535,7 @@ def spark() -> "SparkSession":
     spark_version = ".".join(importlib.metadata.version("pyspark").split(".")[:2])
     scala_version = "2.13"
     iceberg_version = "1.10.0"
-    hadoop_version = "3.3.6"
+    hadoop_version = "3.4.1"
     aws_sdk_version = "1.12.753"
 
     os.environ["PYSPARK_SUBMIT_ARGS"] = (
@@ -2577,11 +2577,6 @@ def spark() -> "SparkSession":
         .config("spark.sql.catalog.spark_catalog.warehouse", "s3://warehouse/hive/")
         .config("spark.hadoop.fs.s3a.endpoint", "http://localhost:9000")
         .config("spark.hadoop.fs.s3a.path.style.access", "true")
-        .config("spark.hadoop.fs.s3a.threads.keepalivetime", "60000")
-        .config("spark.hadoop.fs.s3a.connection.establish.timeout", "30000")
-        .config("spark.hadoop.fs.s3a.connection.timeout", "200000")
-        .config("spark.hadoop.fs.s3a.multipart.purge.age", str(24 * 60 * 60))
-        .config("spark.hadoop.fs.s3a.aws.credentials.provider", "com.amazonaws.auth.EnvironmentVariableCredentialsProvider")
         .config("spark.sql.catalogImplementation", "hive")
         .config("spark.sql.defaultCatalog", "integration")
         .config("spark.sql.execution.arrow.pyspark.enabled", "true")