MLE-24717 Bumping to Spark 4.1.0-preview3

And bumped a few others too, including langchain4j

Author: rjrudin

build.gradle

@@ -46,15 +46,15 @@ subprojects {
     }
   }
 
-  configurations.configureEach {
-    resolutionStrategy.eachDependency { DependencyResolveDetails details ->
-      // Defining this rule for io.netty here so that it affects both flux-cli and flux-embedding-model-azure-open-ai.
-      if (details.requested.group.equals("io.netty") and details.requested.version.startsWith("4.1.1")) {
-        details.useVersion "4.1.128.Final"
-        details.because "Bumping from 4.1.127 (what Spark SQL 4.1.0-preview2 depends on) to eliminate CVEs."
-      }
-    }
-  }
+ configurations.configureEach {
+   resolutionStrategy.eachDependency { DependencyResolveDetails details ->
+     // Defining this rule for io.netty here so that it affects both flux-cli and flux-embedding-model-azure-open-ai.
+     if (details.requested.group.equals("io.netty") and details.requested.version.startsWith("4.1.")) {
+       details.useVersion "4.2.7.Final"
+       details.because "Spark 4.1.0-preview3 depends on 4.2.7.Final, and we need azure-open-ai to do as well to avoid conflicts."
+     }
+   }
+ }
 
   test {
     useJUnitPlatform()

flux-cli/build.gradle

@@ -26,15 +26,9 @@ dependencies {
     implementation("org.eclipse.jetty:jetty-util-ajax:9.4.58.v20250814") {
       because "Bumping to latest 9.x to eliminate CVEs; this is brought in by hadoop-azure."
     }
-    implementation("org.apache.zookeeper:zookeeper:3.9.4") {
-      because "Addresses CVE-2025-58457; Spark depends on 3.9.3, and the CVE is fixed in 3.9.4."
-    }
-    // Not yet defining constraints for io.netty, as doing so would require several separate modules to be defined
-    // here. The hope is that the 4.1.0 final release will use the latest patch version of io.netty and thus no
-    // constraints or top-level resolution strategies will be needed.
   }
 
-  implementation ("org.apache.spark:spark-sql_2.13:${sparkVersion}") {
+  implementation("org.apache.spark:spark-sql_2.13:${sparkVersion}") {
     // The rocksdbjni dependency weighs in at 50mb and so far does not appear necessary for our use of Spark.
     exclude module: "rocksdbjni"
   }
@@ -68,7 +62,7 @@ dependencies {
   // For Blob Storage
   implementation "com.microsoft.azure:azure-storage:8.6.6"
   // For Data Lake Storage Gen2
-  implementation ("com.azure:azure-storage-file-datalake:12.23.1") {
+  implementation("com.azure:azure-storage-file-datalake:12.25.0") {
     // Exclude the Jackson dependencies that Spark already provides, and we need the version required by Spark.
     // This exclusion can be tested by publishing flux-api locally and running the test programs in
     // examples/client-project.
@@ -88,7 +82,7 @@ dependencies {
   }
 
   // For configuring two-way SSL in tests.
-  testImplementation ("com.marklogic:ml-app-deployer:6.2-SNAPSHOT") {
+  testImplementation("com.marklogic:ml-app-deployer:6.2-SNAPSHOT") {
     // Prefer Spark's versions of Jackson.
     exclude group: "com.fasterxml.jackson.core"
     exclude group: "com.fasterxml.jackson.dataformat"
@@ -107,7 +101,7 @@ dependencies {
   testImplementation "org.postgresql:postgresql:42.7.8"
 
   // Required for compile-time references in the tests to langchain4j classes.
-  testImplementation ("dev.langchain4j:langchain4j:${langchain4jVersion}") {
+  testImplementation("dev.langchain4j:langchain4j:${langchain4jVersion}") {
     // Prefer Spark's versions of Jackson.
     exclude group: "com.fasterxml.jackson.core"
   }
@@ -120,10 +114,10 @@ dependencies {
   testImplementation project(path: ":flux-embedding-model-ollama", configuration: "shadow")
 
   // For deep equals on JSON with diff.
-  testImplementation "net.javacrumbs.json-unit:json-unit-assertj:4.1.1"
+  testImplementation "net.javacrumbs.json-unit:json-unit-assertj:5.0.0"
 
   // For deep equals on XML with diff.
-  testImplementation "org.xmlunit:xmlunit-core:2.10.4"
+  testImplementation "org.xmlunit:xmlunit-core:2.11.0"
 }
 
 javadoc {

flux-embedding-model-azure-open-ai/build.gradle

@@ -19,8 +19,9 @@ dependencies {
   // Repeating this here so that the Jackson dependencies are included for running tests in this subproject.
   testImplementation "dev.langchain4j:langchain4j-azure-open-ai:${langchain4jVersion}"
 
-  testImplementation "org.junit.jupiter:junit-jupiter:5.11.2"
-  testImplementation "ch.qos.logback:logback-classic:1.5.19"
+  testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0"
+  testImplementation "org.junit.jupiter:junit-jupiter:5.14.0"
+  testImplementation "ch.qos.logback:logback-classic:1.5.20"
 }
 
 shadowJar {

flux-embedding-model-minilm/build.gradle

@@ -9,7 +9,7 @@ java {
 }
 
 dependencies {
-  implementation ("dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2:1.5.0-beta11") {
+  implementation ("dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2:1.8.0-beta15") {
     // Ensures that the version of Jackson preferred by Spark is used instead. Otherwise, this library's pom
     // will bring in a later version of Jackson that causes Spark to fail.
     exclude group: "com.fasterxml.jackson.core"

flux-embedding-model-ollama/build.gradle

@@ -18,8 +18,9 @@ dependencies {
   // Repeating this here so that the Jackson dependencies are included for running tests in this subproject.
   testImplementation "dev.langchain4j:langchain4j-ollama:${langchain4jVersion}"
 
-  testImplementation "org.junit.jupiter:junit-jupiter:5.11.2"
-  testImplementation "ch.qos.logback:logback-classic:1.5.19"
+  testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0"
+  testImplementation "org.junit.jupiter:junit-jupiter:5.14.0"
+  testImplementation "ch.qos.logback:logback-classic:1.5.20"
 }
 
 shadowJar {

gradle.properties

@@ -6,10 +6,10 @@ connectorVersion=3.0-SNAPSHOT
 picocliVersion=4.7.7
 
 # Aligned with our Spark connector.
-sparkVersion=4.1.0-preview2
+sparkVersion=4.1.0-preview3
 hadoopVersion=3.4.2
 
-langchain4jVersion=1.5.0
+langchain4jVersion=1.8.0
 tikaVersion=3.2.3
 
 # Define these on the command line to publish to OSSRH