[SNAPPYDATA] remaining changes for log4j 2.x transition

- updating remaining log4j 1.x usages
- changing gradle build to log4j 2.x
- update gradle builds to incorporate the pom.xml changes
- refactored gradle builds to move common sections to base build.gradle
- upgraded to log4j 2.17.2
- changed jetty version to 9.4.43.v20210629
- porting remaining changes for kafka 2.2.2 upgrade
- increased timeout in JavaTestUtils for parallel test runs
- fix YarnClusterSuite failures by forcing usage of old jetty version

Author: Sumedh Wale

build.gradle

@@ -63,9 +63,10 @@ allprojects {
     protobufVersion = '3.6.1'
     jerseyVersion = '2.22.2'
     sunJerseyVersion = '1.19.4'
-    jettyVersion = '9.4.44.v20210927'
+    jettyVersion = '9.4.43.v20210629'
+    yarnTestJettyVersion = '9.3.24.v20180605' // only used by yarn tests
     jettyOldVersion = '6.1.26'
-    log4jVersion = '1.2.17'
+    log4j2Version = '2.17.2'
     slf4jVersion = '1.7.32'
     junitVersion = '4.12'
     mockitoVersion = '1.10.19'
@@ -148,6 +149,110 @@ allprojects {
     pegdownVersion = '1.6.0'
 
     shadePackageName = 'org.spark_project'
+
+    libraries = [
+      hadoop_client: dependencies.create("org.apache.hadoop:hadoop-client:${hadoopVersion}") {
+        exclude(group: 'asm', module: 'asm')
+        exclude(group: 'org.codehaus.jackson', module: 'jackson-core-asl')
+        exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl')
+        exclude(group: 'org.ow2.asm', module: 'asm')
+        exclude(group: 'org.jboss.netty', module: 'netty')
+        exclude(group: 'commons-logging', module: 'commons-logging')
+        exclude(group: 'org.mockito', module: 'mockito-all')
+        exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5')
+        exclude(group: 'javax.servlet', module: 'servlet-api')
+        exclude(group: 'junit', module: 'junit')
+        exclude(group: 'com.google.guava', module: 'guava')
+        exclude(group: 'com.sun.jersey')
+        exclude(group: 'com.sun.jersey.jersey-test-framework')
+        exclude(group: 'com.sun.jersey.contribs')
+        exclude(group: 'io.netty', module: 'netty')
+        exclude(group: 'io.netty', module: 'netty-all')
+        exclude(group: 'org.apache.directory.server', module: 'apacheds-kerberos-codec')
+        exclude(group: 'log4j', module: 'log4j')
+        exclude(group: 'org.slf4j', module: 'slf4j-logj12')
+      },
+      hive_exec: dependencies.create("org.spark-project.hive:hive-exec:${hiveVersion}") {
+        exclude(group: 'org.datanucleus', module: 'datanucleus-core')
+        exclude(group: 'org.spark-project.hive', module: 'hive-metastore')
+        exclude(group: 'org.spark-project.hive', module: 'hive-shims')
+        exclude(group: 'org.spark-project.hive', module: 'hive-ant')
+        exclude(group: 'org.spark-project.hive', module: 'spark-client')
+        exclude(group: 'org.apache.ant', module: 'ant')
+        exclude(group: 'com.esotericsoftware.kryo', module: 'kryo')
+        exclude(group: 'commons-codec', module: 'commons-codec')
+        exclude(group: 'commons-httpclient', module: 'commons-httpclient')
+        exclude(group: 'org.apache.avro', module: 'avro-mapred')
+        exclude(group: 'org.apache.calcite', module: 'calcite-core')
+        exclude(group: 'org.apache.curator', module: 'apache-curator')
+        exclude(group: 'org.apache.curator', module: 'curator-client')
+        exclude(group: 'org.apache.curator', module: 'curator-framework')
+        exclude(group: 'org.apache.thrift', module: 'libthrift')
+        exclude(group: 'org.apache.thrift', module: 'libfb303')
+        exclude(group: 'org.apache.zookeeper', module: 'zookeeper')
+        exclude(group: 'org.slf4j', module: 'slf4j-api')
+        exclude(group: 'org.slf4j', module: 'slf4j-log4j12')
+        exclude(group: 'log4j', module: 'log4j')
+        exclude(group: 'log4j', module: 'apache-log4j-extras')
+        exclude(group: 'commons-logging', module: 'commons-logging')
+        exclude(group: 'org.codehaus.groovy', module: 'groovy-all')
+        exclude(group: 'jline', module: 'jline')
+        exclude(group: 'org.json', module: 'json')
+      },
+      hive_meta: dependencies.create("org.spark-project.hive:hive-metastore:${hiveVersion}") {
+        exclude(group: 'org.datanucleus', module: 'datanucleus-core')
+        exclude(group: 'org.datanucleus', module: 'datanucleus-api-jdo')
+        exclude(group: 'org.datanucleus', module: 'datanucleus-rdbms')
+        exclude(group: 'org.spark-project.hive', module: 'hive-serde')
+        exclude(group: 'org.spark-project.hive', module: 'hive-shims')
+        exclude(group: 'org.apache.thrift', module: 'libfb303')
+        exclude(group: 'org.apache.thrift', module: 'libthrift')
+        exclude(group: 'javax.servlet', module: 'servlet-api')
+        exclude(group: 'com.google.guava', module: 'guava')
+        exclude(group: 'org.slf4j', module: 'slf4j-api')
+        exclude(group: 'org.slf4j', module: 'slf4j-log4j12')
+        exclude(group: 'log4j', module: 'log4j')
+        exclude(group: 'log4j', module: 'apache-log4j-extras')
+        exclude(group: 'org.apache.derby', module: 'derby')
+      },
+      avro_ipc: dependencies.create("org.apache.avro:avro-ipc:${avroVersion}") {
+        exclude(group: 'io.netty', module: 'netty')
+        exclude(group: 'org.mortbay.jetty', module: 'jetty')
+        exclude(group: 'org.mortbay.jetty', module: 'jetty-util')
+        exclude(group: 'org.mortbay.jetty', module: 'servlet-api')
+        exclude(group: 'org.apache.velocity', module: 'velocity')
+        exclude(group: 'log4j', module: 'log4j')
+        exclude(group: 'org.slf4j', module: 'slf4j-log4j12')
+      },
+      avro_mapred: dependencies.create("org.apache.avro:avro-mapred:${avroVersion}:hadoop2") {
+        exclude(group: 'io.netty', module: 'netty')
+        exclude(group: 'org.mortbay.jetty', module: 'jetty')
+        exclude(group: 'org.mortbay.jetty', module: 'jetty-util')
+        exclude(group: 'org.mortbay.jetty', module: 'servlet-api')
+        exclude(group: 'org.apache.velocity', module: 'velocity')
+        exclude(group: 'org.apache.avro', module: 'avro-ipc')
+      },
+      netty_all: dependencies.create("io.netty:netty-all:${nettyAllVersion}") {
+        exclude(group: 'io.netty', module: 'netty-codec-dns')
+        exclude(group: 'io.netty', module: 'netty-codec-haproxy')
+        exclude(group: 'io.netty', module: 'netty-codec-http')
+        exclude(group: 'io.netty', module: 'netty-codec-http2')
+        exclude(group: 'io.netty', module: 'netty-codec-memcache')
+        exclude(group: 'io.netty', module: 'netty-codec-mqtt')
+        exclude(group: 'io.netty', module: 'netty-codec-redis')
+        exclude(group: 'io.netty', module: 'netty-codec-smtp')
+        exclude(group: 'io.netty', module: 'netty-codec-socks')
+        exclude(group: 'io.netty', module: 'netty-codec-stomp')
+        exclude(group: 'io.netty', module: 'netty-codec-xml')
+        exclude(group: 'io.netty', module: 'netty-handler-proxy')
+        exclude(group: 'io.netty', module: 'netty-resolver-dns')
+        exclude(group: 'io.netty', module: 'netty-resolver-dns-classes-macos')
+        exclude(group: 'io.netty', module: 'netty-resolver-dns-native-macos')
+        exclude(group: 'io.netty', module: 'netty-transport-rxtx')
+        exclude(group: 'io.netty', module: 'netty-transport-sctp')
+        exclude(group: 'io.netty', module: 'netty-transport-udt')
+      }
+    ]
   }
 
   // default output directory like in sbt/maven
@@ -285,9 +390,11 @@ subprojects {
     compile 'org.scala-lang:scala-library:' + scalaVersion
     compile 'org.scala-lang:scala-reflect:' + scalaVersion
 
-    compile group: 'log4j', name:'log4j', version: log4jVersion
+    compile 'org.apache.logging.log4j:log4j-api:' + log4j2Version
+    compile 'org.apache.logging.log4j:log4j-core:' + log4j2Version
+    compile 'org.apache.logging.log4j:log4j-1.2-api:' + log4j2Version
     compile 'org.slf4j:slf4j-api:' + slf4jVersion
-    compile 'org.slf4j:slf4j-log4j12:' + slf4jVersion
+    compile 'org.apache.logging.log4j:log4j-slf4j-impl:' + log4j2Version
 
     testCompile "junit:junit:${junitVersion}"
     testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}"
@@ -389,10 +496,11 @@ gradle.taskGraph.whenReady { graph ->
           'SPARK_PREPEND_CLASSES': '1',
           'SPARK_SCALA_VERSION': scalaBinaryVersion,
           'SPARK_TESTING': '1',
+          'SPARK_LOCAL_IP': 'localhost',
           'PYSPARK_PYTHON': sparkPython,
           'PYSPARK_DRIVER_PYTHON': sparkPython,
           'JAVA_HOME': System.getProperty('java.home')
-        systemProperties 'log4j.configuration': "file:${test.project.projectDir}/src/test/resources/log4j.properties",
+        systemProperties 'log4j.configurationFile': "file:${test.project.projectDir}/src/test/resources/log4j2.properties",
           'derby.system.durability': 'test',
           'java.awt.headless': 'true',
           'java.io.tmpdir': "${rootProject.buildDir}/tmp",

common/network-common/build.gradle

@@ -20,7 +20,7 @@ description = 'Spark Project Networking'
 dependencies {
   compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion)
 
-  compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion
+  compile libraries.netty_all
   compile group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version
   compile group: 'com.google.guava', name: 'guava', version: guavaVersion
   compile group: 'org.fusesource.leveldbjni', name: 'leveldbjni-all', version: levelDbJniVersion

common/network-yarn/build.gradle

@@ -21,35 +21,10 @@ dependencies {
   compile project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion)
   compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion)
 
-  compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion
-  compileOnly (group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) {
-    exclude(group: 'asm', module: 'asm')
-    exclude(group: 'org.codehaus.jackson', module: 'jackson-core-asl')
-    exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl')
-    exclude(group: 'org.ow2.asm', module: 'asm')
-    exclude(group: 'org.jboss.netty', module: 'netty')
-    exclude(group: 'commons-logging', module: 'commons-logging')
-    exclude(group: 'org.mockito', module: 'mockito-all')
-    exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5')
-    exclude(group: 'javax.servlet', module: 'servlet-api')
-    exclude(group: 'junit', module: 'junit')
-    exclude(group: 'com.google.guava', module: 'guava')
-    exclude(group: 'com.sun.jersey')
-    exclude(group: 'com.sun.jersey.jersey-test-framework')
-    exclude(group: 'com.sun.jersey.contribs')
-    exclude(group: 'io.netty', module: 'netty')
-    exclude(group: 'io.netty', module: 'netty-all')
-    exclude(group: 'org.apache.directory.server', module: 'apacheds-kerberos-codec')
-  }
+  compile libraries.netty_all
+  compileOnly libraries.hadoop_client
 
   testCompile project(path: subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion, configuration: 'testOutput')
-  /*
-  runtimeJar project(subprojectBase + 'snappy-spark-network-common_' + scalaBinaryVersion)
-  runtimeJar project(subprojectBase + 'snappy-spark-network-shuffle_' + scalaBinaryVersion)
-  runtimeJar group: 'io.netty', name: 'netty-all', version: nettyAllVersion
-  runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion
-  runtimeJar group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion
-  */
 }
 
 shadowJar {

common/unsafe/pom.xml

@@ -65,10 +65,6 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
-    <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-    </dependency>
 
     <!-- Provided dependencies -->
     <dependency>

common/unsafe/src/main/java/org/apache/spark/unsafe/Native.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved.
+ * Copyright (c) 2017-2022 TIBCO Software Inc. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License. You
@@ -23,7 +23,8 @@
 import java.security.CodeSource;
 import java.util.Locale;
 
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Optimized JNI calls.
@@ -64,7 +65,7 @@ private Native() {
     String arch = System.getProperty("os.arch");
     is64Bit = arch.contains("64") || arch.contains("s390x");
 
-    logger = Logger.getLogger(Native.class);
+    logger = LoggerFactory.getLogger(Native.class);
 
     String library = "native" + suffix;
     if (is64Bit()) {
@@ -104,13 +105,9 @@ private Native() {
 
       loaded = true;
     } catch (IOException ioe) {
-      if (logger.isInfoEnabled()) {
-        logger.info("library " + library + " could not be loaded due to " + ioe);
-      }
+      logger.info("library " + library + " could not be loaded due to " + ioe);
     } catch (UnsatisfiedLinkError ule) {
-      if (logger.isInfoEnabled()) {
-        logger.info("library " + library + " could not be loaded");
-      }
+      logger.info("library " + library + " could not be loaded");
     }
     nativeLoaded = loaded;
   }

core/build.gradle

@@ -24,21 +24,8 @@ dependencies {
   compile project(subprojectBase + 'snappy-spark-unsafe_' + scalaBinaryVersion)
   compile project(subprojectBase + 'snappy-spark-tags_' + scalaBinaryVersion)
 
-  compile(group: 'org.apache.avro', name: 'avro-ipc', version: avroVersion) {
-    exclude(group: 'io.netty', module: 'netty')
-    exclude(group: 'org.mortbay.jetty', module: 'jetty')
-    exclude(group: 'org.mortbay.jetty', module: 'jetty-util')
-    exclude(group: 'org.mortbay.jetty', module: 'servlet-api')
-    exclude(group: 'org.apache.velocity', module: 'velocity')
-  }
-  compile(group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion, classifier: 'hadoop2') {
-    exclude(group: 'io.netty', module: 'netty')
-    exclude(group: 'org.mortbay.jetty', module: 'jetty')
-    exclude(group: 'org.mortbay.jetty', module: 'jetty-util')
-    exclude(group: 'org.mortbay.jetty', module: 'servlet-api')
-    exclude(group: 'org.apache.velocity', module: 'velocity')
-    exclude(group: 'org.apache.avro', module: 'avro-ipc')
-  }
+  compile libraries.avro_ipc
+  compile libraries.avro_mapred
   compile group: 'com.google.guava', name: 'guava', version: guavaVersion
   compile group: 'com.esotericsoftware', name: 'kryo-shaded', version: kryoVersion
   compile(group: 'com.twitter', name: 'chill_' + scalaBinaryVersion, version: chillVersion) {
@@ -49,7 +36,9 @@ dependencies {
   }
   compile group: 'org.apache.xbean', name: 'xbean-asm5-shaded', version: xbeanAsm5Version
   // explicitly include netty from akka-remote to not let zookeeper override it
-  compile group: 'io.netty', name: 'netty', version: nettyVersion
+  compile(group: 'io.netty', name: 'netty', version: nettyVersion) {
+    exclude(group: 'log4j', module: 'log4j')
+  }
   // explicitly exclude old netty from zookeeper
   compile(group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion) {
     exclude(group: 'org.jboss.netty', module: 'netty')
@@ -62,25 +51,7 @@ dependencies {
     exclude(group: 'log4j', module: 'log4j')
   }
   compile group: 'com.google.protobuf', name: 'protobuf-java', version: protobufVersion
-  compile(group: 'org.apache.hadoop', name: 'hadoop-client', version: hadoopVersion) {
-    exclude(group: 'asm', module: 'asm')
-    exclude(group: 'org.codehaus.jackson', module: 'jackson-mapper-asl')
-    exclude(group: 'org.ow2.asm', module: 'asm')
-    exclude(group: 'org.apache.zookeeper', module: 'zookeeper')
-    exclude(group: 'org.jboss.netty', module: 'netty')
-    exclude(group: 'jline', module: 'jline')
-    exclude(group: 'commons-logging', module: 'commons-logging')
-    exclude(group: 'org.mockito', module: 'mockito-all')
-    exclude(group: 'org.mortbay.jetty', module: 'servlet-api-2.5')
-    exclude(group: 'javax.servlet', module: 'servlet-api')
-    exclude(group: 'junit', module: 'junit')
-    exclude(group: 'com.google.guava', module: 'guava')
-    exclude(group: 'com.sun.jersey')
-    exclude(group: 'com.sun.jersey.jersey-test-framework')
-    exclude(group: 'com.sun.jersey.contribs')
-    exclude(group: 'com.google.protobuf', module: 'protobuf-java')
-    exclude(group: 'org.apache.directory.server', module: 'apacheds-kerberos-codec')
-  }
+  compile libraries.hadoop_client
   compile group: 'org.codehaus.jackson', name: 'jackson-mapper-asl', version: jackson1Version
   compile(group: 'net.java.dev.jets3t', name: 'jets3t', version: jets3tVersion) {
     exclude(group: 'commons-codec', module: 'commons-codec')
@@ -91,6 +62,9 @@ dependencies {
     exclude(group: 'org.jboss.netty', module: 'netty')
     exclude(group: 'jline', module: 'jline')
     exclude(group: 'com.google.guava', module: 'guava')
+    exclude(group: 'org.slf4j', module: 'slf4j-api')
+    exclude(group: 'org.slf4j', module: 'slf4j-log4j12')
+    exclude(group: 'log4j', module: 'log4j')
   }
 
   compile 'org.scala-lang:scalap:' + scalaVersion
@@ -113,8 +87,7 @@ dependencies {
   compile(group: 'org.apache.commons', name: 'commons-crypto', version: commonsCryptoVersion) {
     exclude(group: 'net.java.dev.jna', module: 'jna')
   }
-  compile group: 'io.netty', name: 'netty', version: nettyVersion
-  compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion
+  compile libraries.netty_all
   compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion
   compile group: 'org.slf4j', name: 'jcl-over-slf4j', version: slf4jVersion
   compile group: 'org.xerial.snappy', name: 'snappy-java', version: snappyJavaVersion
@@ -127,7 +100,6 @@ dependencies {
   compile group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion
   compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet', version: jerseyVersion
   compile group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion
-  compile group: 'io.netty', name: 'netty-all', version: nettyAllVersion
   compile(group: 'com.clearspring.analytics', name: 'stream', version: streamVersion) {
     exclude(group: 'it.unimi.dsi', module: 'fastutil')
   }

core/src/main/scala/org/apache/spark/internal/Logging.scala

@@ -103,13 +103,21 @@ private[spark] trait Logging {
   }
 
   protected def initializeLogIfNecessary(isInterpreter: Boolean): Unit = {
+    initializeLogIfNecessary(isInterpreter, silent = false)
+  }
+
+  protected def initializeLogIfNecessary(
+      isInterpreter: Boolean,
+      silent: Boolean = false): Boolean = {
     if (!Logging.initialized) {
       Logging.initLock.synchronized {
         if (!Logging.initialized) {
-          initializeLogging(isInterpreter)
+          initializeLogging(isInterpreter, silent)
+          return true
         }
       }
     }
+    false
   }
 
   private def initializeLogging(isInterpreter: Boolean, silent: Boolean): Unit = {
@@ -172,8 +180,12 @@ private[spark] trait Logging {
   }
 }
 
-private object Logging {
+private[spark] object Logging {
   @volatile private var initialized = false
+  @volatile private var defaultRootLevel: Level = null
+  @volatile private var defaultSparkLog4jConfig = false
+  @volatile private[spark] var sparkShellThresholdLevel: Level = null
+
   val initLock = new Object()
   try {
     // We use reflection here to handle the case where users remove the

core/src/test/scala/org/apache/spark/internal/LoggingSuite.scala

@@ -52,7 +52,7 @@ class LoggingSuite extends SparkFunSuite {
       assert(ssf.filter(logEvent2) == Filter.Result.NEUTRAL)
 
       // custom log level configured (by log4j2.properties)
-      val jettyLogger = LogManager.getLogger("org.sparkproject.jetty")
+      val jettyLogger = LogManager.getLogger("org.spark_project.jetty")
         .asInstanceOf[Logger]
       val logEvent3 = new Builder().setLevel(Level.INFO)
         .setLoggerName(jettyLogger.getName()).setMessage(new SimpleMessage("Test")).build()