Fix hadoop version tests. (#34155)

* Fix hadoop version tests.

* Trigger post-commit tests.

Author: shunping

.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 3
+  "modification": 4
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json

@@ -6,5 +6,6 @@
   "https://github.com/apache/beam/pull/33267": "noting that PR #33267 should run this test",
   "https://github.com/apache/beam/pull/33322": "noting that PR #33322 should run this test",
   "https://github.com/apache/beam/pull/34123": "noting that PR #34123 should run this test",
-  "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test"
+  "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test",
+  "https://github.com/apache/beam/pull/34155": "noting that PR #34155 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json

@@ -5,5 +5,6 @@
   "https://github.com/apache/beam/pull/32546": "noting that PR #32546 should run this test",
   "https://github.com/apache/beam/pull/33267": "noting that PR #33267 should run this test",
   "https://github.com/apache/beam/pull/34123": "noting that PR #34123 should run this test",
-  "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test"
+  "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test",
+  "https://github.com/apache/beam/pull/34155": "noting that PR #34155 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json

@@ -5,5 +5,6 @@
   "https://github.com/apache/beam/pull/32546": "noting that PR #32546 should run this test",
   "https://github.com/apache/beam/pull/33267": "noting that PR #33267 should run this test",
   "https://github.com/apache/beam/pull/33322": "noting that PR #33322 should run this test",
-  "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test"
+  "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test",
+  "https://github.com/apache/beam/pull/34155": "noting that PR #34155 should run this test"
 }

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -836,6 +836,8 @@ class BeamModulePlugin implements Plugin<Project> {
         log4j2_core                                 : "org.apache.logging.log4j:log4j-core:$log4j2_version",
         log4j2_to_slf4j                             : "org.apache.logging.log4j:log4j-to-slf4j:$log4j2_version",
         log4j2_slf4j_impl                           : "org.apache.logging.log4j:log4j-slf4j-impl:$log4j2_version",
+        log4j2_slf4j2_impl                          : "org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2_version",
+        log4j2_log4j12_api                          : "org.apache.logging.log4j:log4j-1.2-api:$log4j2_version",
         mockito_core                                : "org.mockito:mockito-core:4.11.0",
         mockito_inline                              : "org.mockito:mockito-inline:4.11.0",
         mongo_java_driver                           : "org.mongodb:mongo-java-driver:3.12.11",

runners/spark/spark_runner.gradle

@@ -216,12 +216,17 @@ dependencies {
   validatesRunner project(":sdks:java:io:hadoop-format").sourceSets.test.output
   validatesRunner project(path: ":examples:java", configuration: "testRuntimeMigration")
   validatesRunner project(path: project.path, configuration: "testRuntimeMigration")
-  validatesRunner library.java.log4j2_api
-  validatesRunner library.java.log4j2_core
   hadoopVersions.each { kv ->
     "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value"
     // Force paranamer 2.8 to avoid issues when using Scala 2.12
     "hadoopVersion$kv.key" "com.thoughtworks.paranamer:paranamer:2.8"
+    if ("$spark_version" >= "3.5.0") {
+      // Add log4j 2.x dependencies as Spark 3.5+ uses slf4j with log4j 2.x backend
+      "hadoopVersion$kv.key" library.java.log4j2_api
+      "hadoopVersion$kv.key" library.java.log4j2_core
+      "hadoopVersion$kv.key" library.java.log4j2_slf4j2_impl
+      "hadoopVersion$kv.key" library.java.log4j2_log4j12_api
+    }
   }
 }
 
@@ -240,13 +245,22 @@ configurations.all {
 configurations.validatesRunner {
   // Exclude to make sure log4j binding is used
   exclude group: "org.slf4j", module: "slf4j-simple"
+
+  if ("$spark_version" >= "3.5.0") {
+    // Exclude log4j 1.x dependencies to prevent conflict with log4j 2.x used by spark 3.5+
+    exclude group: "log4j", module: "log4j"
+  }
 }
 
 hadoopVersions.each { kv ->
   configurations."hadoopVersion$kv.key" {
     resolutionStrategy {
       force "org.apache.hadoop:hadoop-common:$kv.value"
     }
+    if ("$spark_version" >= "3.5.0") {
+      // Exclude log4j 1.x dependencies to prevent conflict with log4j 2.x used by spark 3.5+
+      exclude group: "log4j", module: "log4j"
+    }
   }
 }