[SPARK-24767] Propagate MDC to spark-submit thread in InProcessAppHandle (apache-spark-on-k8s#389)

[SPARK-24767] Propagate MDC to spark-submit thread in InProcessAppHandle (apache-spark-on-k8s#389)
[SPARK-24813][BUILD][FOLLOW-UP][HOTFIX] HiveExternalCatalogVersionsSuite still flaky; fall back to Apache archive

Author: robert3005

launcher/pom.xml

@@ -54,7 +54,6 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>

launcher/src/main/java/org/apache/spark/launcher/InProcessAppHandle.java

@@ -17,7 +17,10 @@
 
 package org.apache.spark.launcher;
 
+import org.slf4j.MDC;
+
 import java.lang.reflect.Method;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -58,7 +61,11 @@ synchronized void start(String appName, Method main, String[] args) {
       appName = "..." + appName.substring(appName.length() - MAX_APP_NAME_LEN);
     }
 
+    Map<String, String> mdcContextMap = MDC.getCopyOfContextMap();
     app = new Thread(() -> {
+      if (mdcContextMap != null) {
+        MDC.setContextMap(mdcContextMap);
+      }
       try {
         main.invoke(null, (Object) args);
       } catch (Throwable t) {

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala

@@ -195,7 +195,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
 
 object PROCESS_TABLES extends QueryTest with SQLTestUtils {
   // Tests the latest version of every release line.
-  val testingVersions = Seq("2.0.2", "2.1.2", "2.2.1", "2.3.1")
+  val testingVersions = Seq("2.1.3", "2.2.2", "2.3.1")
 
   protected var spark: SparkSession = _