[LIVY-1010] Add support for Spark 3.5.6

## What changes were proposed in this pull request?

Keep Spark 3 support up to date with the latest Spark 3 release, upgrade dependencies.

## How was this patch tested?

Unit and integration tests.

Author: gyogal

.github/workflows/integration-tests.yaml

@@ -48,7 +48,8 @@ jobs:
     -
       name: Set Python 3 as default for Spark 3 builds
       if: ${{ contains(matrix.maven_profile, 'spark3') }}
-      run: pyenv global 3
+      # This can be removed once support for Python 2 and Spark 2 is removed and the default python executable is python3
+      run: pyenv global 3 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV"
     - 
       name: Build with Maven
       run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipTests -Dmaven.javadoc.skip=true -B -V -e verify

integration-test/pom.xml

@@ -111,6 +111,11 @@
       </exclusions>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>

pom.xml

@@ -81,19 +81,21 @@
     <asynchttpclient.version>2.10.1</asynchttpclient.version>
     <hadoop.scope>compile</hadoop.scope>
     <slf4j.version>1.7.36</slf4j.version>
-    <reload4j.version>1.2.25</reload4j.version>
+    <reload4j.version>1.2.26</reload4j.version>
     <spark.scala-2.11.version>2.4.5</spark.scala-2.11.version>
     <spark.scala-2.12.version>2.4.5</spark.scala-2.12.version>
     <spark.version>${spark.scala-2.11.version}</spark.version>
     <kubernetes.client.version>5.6.0</kubernetes.client.version>
     <hive.version>3.0.0</hive.version>
-    <commons-codec.version>1.9</commons-codec.version>
-    <httpclient.version>4.5.13</httpclient.version>
-    <httpcore.version>4.4.4</httpcore.version>
+    <commons-codec.version>1.15</commons-codec.version>
+    <commons-lang3.version>3.17.0</commons-lang3.version>
+    <httpclient.version>4.5.14</httpclient.version>
+    <httpcore.version>4.4.16</httpcore.version>
     <jackson.version>2.12.7</jackson.version>
     <jackson-databind.version>2.12.7.1</jackson-databind.version>
+    <jacoco.version>0.8.13</jacoco.version>
     <javax.servlet-api.version>3.1.0</javax.servlet-api.version>
-    <jetty.version>9.4.50.v20221201</jetty.version>
+    <jetty.version>9.4.56.v20240826</jetty.version>
     <junit.version>4.13.1</junit.version>
     <libthrift.version>0.9.3</libthrift.version>
     <kryo.version>4.0.2</kryo.version>
@@ -130,6 +132,8 @@
     <!-- Set this to "true" to skip R tests. -->
     <skipRTests>false</skipRTests>
 
+    <!-- Set this to "true" to skip PySpark2 tests. -->
+    <skipPySpark2Tests>false</skipPySpark2Tests>
     <!-- Set this to "true" to skip PySpark3 tests. -->
     <skipPySpark3Tests>false</skipPySpark3Tests>
 
@@ -307,6 +311,12 @@
         <version>${commons-codec.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-lang3</artifactId>
+        <version>${commons-lang3.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>io.dropwizard.metrics</groupId>
         <artifactId>metrics-core</artifactId>
@@ -412,6 +422,18 @@
         </exclusions>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <version>${hadoop.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdfs</artifactId>
@@ -789,7 +811,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-shade-plugin</artifactId>
-          <version>3.2.1</version>
+          <version>3.5.0</version>
         </plugin>
 
         <plugin>
@@ -810,6 +832,7 @@
               <spark.ui.enabled>false</spark.ui.enabled>
               <project.version>${project.version}</project.version>
               <skipRTests>${skipRTests}</skipRTests>
+              <skipPySpark2Tests>${skipPySpark2Tests}</skipPySpark2Tests>
               <skipPySpark3Tests>${skipPySpark3Tests}</skipPySpark3Tests>
             </systemProperties>
             <redirectTestOutputToFile>${test.redirectToFile}</redirectTestOutputToFile>
@@ -839,6 +862,7 @@
               <spark.ui.enabled>false</spark.ui.enabled>
               <project.version>${project.version}</project.version>
               <skipRTests>${skipRTests}</skipRTests>
+              <skipPySpark2Tests>${skipPySpark2Tests}</skipPySpark2Tests>
               <skipPySpark3Tests>${skipPySpark3Tests}</skipPySpark3Tests>
             </systemProperties>
             <stdout>D</stdout>
@@ -1081,7 +1105,7 @@
       <plugin>
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
-        <version>0.8.2</version>
+        <version>${jacoco.version}</version>
         <executions>
           <execution>
             <goals>
@@ -1163,7 +1187,7 @@
       <id>scala-2.12</id>
       <properties>
         <scala.binary.version>2.12</scala.binary.version>
-        <scala.version>2.12.15</scala.version>
+        <scala.version>2.12.18</scala.version>
       </properties>
     </profile>
 
@@ -1192,15 +1216,20 @@
     <profile>
       <id>spark3</id>
       <properties>
-        <spark.version>3.2.3</spark.version>
+        <spark.version>3.5.6</spark.version>
+        <hadoop.major-minor.version>3</hadoop.major-minor.version>
+        <hadoop.version>3.3.4</hadoop.version>
         <java.version>1.8</java.version>
         <py4j.version>0.10.9.7</py4j.version>
         <json4s.version>3.7.0-M11</json4s.version>
-        <netty.version>4.1.92.Final</netty.version>
+        <netty.version>4.1.96.Final</netty.version>
+        <jackson.version>2.15.2</jackson.version>
+        <jackson-databind.version>2.15.2</jackson-databind.version>
         <spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
         <spark.bin.download.url>
           https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz
         </spark.bin.download.url>
+        <skipPySpark2Tests>true</skipPySpark2Tests>
       </properties>
     </profile>
 

python-api/setup.py

@@ -31,7 +31,6 @@
     'cloudpickle>=0.2.1',
     'configparser>=3.5.0',
     'future>=0.15.2',
-    'futures>=3.0.5',
     'mock~=3.0.5',
     'requests>=2.10.0',
     'responses>=0.5.1',
@@ -54,6 +53,9 @@
     keywords='livy pyspark development',
     classifiers=CLASSIFIERS,
     install_requires=requirements,
+    extras_require={
+        ':python_version == "2.7"': ['futures']
+    },
     setup_requires=['pytest-runner', 'flake8'],
     tests_require=['pytest']
 )

repl/src/test/scala/org/apache/livy/repl/PythonInterpreterSpec.scala

@@ -282,6 +282,11 @@ class Python2InterpreterSpec extends PythonBaseInterpreterSpec {
 
   implicit val formats = DefaultFormats
 
+  override protected def withFixture(test: NoArgTest): Outcome = {
+    assume(!sys.props.getOrElse("skipPySpark2Tests", "false").toBoolean, "Skipping PySpark2 tests.")
+    test()
+  }
+
   override def createInterpreter(): Interpreter = {
     val sparkConf = new SparkConf()
     PythonInterpreter(sparkConf, new SparkEntries(sparkConf))

repl/src/test/scala/org/apache/livy/repl/PythonSessionSpec.scala

@@ -170,7 +170,13 @@ abstract class PythonSessionSpec extends BaseSessionSpec(PySpark) {
   }
 }
 
-class Python2SessionSpec extends PythonSessionSpec
+class Python2SessionSpec extends PythonSessionSpec {
+
+  override protected def withFixture(test: NoArgTest): Outcome = {
+    assume(!sys.props.getOrElse("skipPySpark2Tests", "false").toBoolean, "Skipping PySpark2 tests.")
+    test()
+  }
+}
 
 class Python3SessionSpec extends PythonSessionSpec with BeforeAndAfterAll {
 

repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala

@@ -193,7 +193,7 @@ class SQLInterpreterSpec extends BaseInterpreterSpec {
     assert(resp.isInstanceOf[Interpreter.ExecuteError])
     val error = resp.asInstanceOf[Interpreter.ExecuteError]
     error.ename should be ("Error")
-    assert(error.evalue.contains("not found"))
+    assert(error.evalue.contains("not found") || error.evalue.contains("cannot be found"))
   }
 
   it should "fail if submitting multiple queries" in withInterpreter { interpreter =>

server/pom.xml

@@ -94,6 +94,11 @@
       <artifactId>javax.servlet-api</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-auth</artifactId>

server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala

@@ -30,6 +30,16 @@ object LivySparkUtils extends Logging {
   // For each Spark version we supported, we need to add this mapping relation in case Scala
   // version cannot be detected from "spark-submit --version".
   private val _defaultSparkScalaVersion = SortedMap(
+    // Spark 3.5 + Scala 2.12
+    (3, 5) -> "2.12",
+    // Spark 3.4 + Scala 2.12
+    (3, 4) -> "2.12",
+    // Spark 3.3 + Scala 2.12
+    (3, 3) -> "2.12",
+    // Spark 3.2 + Scala 2.12
+    (3, 2) -> "2.12",
+    // Spark 3.1 + Scala 2.12
+    (3, 1) -> "2.12",
     // Spark 3.0 + Scala 2.12
     (3, 0) -> "2.12",
     // Spark 2.4 + Scala 2.11
@@ -42,7 +52,7 @@ object LivySparkUtils extends Logging {
 
   // Supported Spark version
   private val MIN_VERSION = (2, 2)
-  private val MAX_VERSION = (3, 1)
+  private val MAX_VERSION = (3, 6)
 
   private val sparkVersionRegex = """version (.*)""".r.unanchored
   private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored

server/src/main/scala/org/apache/livy/utils/SparkKubernetesApp.scala

@@ -31,7 +31,7 @@ import scala.util.control.NonFatal
 import io.fabric8.kubernetes.api.model._
 import io.fabric8.kubernetes.api.model.networking.v1.{Ingress, IngressBuilder}
 import io.fabric8.kubernetes.client.{Config, ConfigBuilder, _}
-import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang3.StringUtils
 
 import org.apache.livy.{LivyConf, Logging}