Migrate Hadoop 3.3.4 to patchable

Author: nightkr

hadoop/stackable/patches/3.3.4/001-YARN-11527-3.3.4.patch renamed to hadoop/stackable/patches/3.3.4/0001-Backport-YARN-11527.patch

@@ -1,5 +1,14 @@
+From e48528d39bc37aa72c19301e9a198f2fb33a0b21 Mon Sep 17 00:00:00 2001
+From: Siegfried Weber <[email protected]>
+Date: Thu, 21 Dec 2023 13:51:13 +0100
+Subject: Backport YARN-11527
+
+---
+ hadoop-project/pom.xml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
-index 0b2f6f17157d..9dc8b653eb93 100644
+index 0b2f6f1715..082834e261 100644
 --- a/hadoop-project/pom.xml
 +++ b/hadoop-project/pom.xml
 @@ -212,7 +212,7 @@

hadoop/stackable/patches/3.3.4/002-datanode-registration-override-3.3.4.patch renamed to hadoop/stackable/patches/3.3.4/0002-Allow-overriding-datanode-registration-addresses.patch

@@ -1,5 +1,17 @@
+From a9f87e2811f973e8f96a7e835bd6f6e5115e2e25 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= <[email protected]>
+Date: Thu, 11 Jan 2024 14:01:02 +0100
+Subject: Allow overriding datanode registration addresses
+
+---
+ .../org/apache/hadoop/hdfs/DFSConfigKeys.java |  9 +++
+ .../blockmanagement/DatanodeManager.java      | 43 +++++++-----
+ .../hadoop/hdfs/server/datanode/DNConf.java   | 70 +++++++++++++++++++
+ .../hadoop/hdfs/server/datanode/DataNode.java | 35 ++++++++--
+ 4 files changed, 135 insertions(+), 22 deletions(-)
+
 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
-index 7196def4221..2c00fb4fb1a 100755
+index 7196def422..2c00fb4fb1 100755
 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 @@ -139,6 +139,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
@@ -26,7 +38,7 @@ index 7196def4221..2c00fb4fb1a 100755
    public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true;
 
 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
-index 44dffcbed11..54f6d63fa78 100644
+index 44dffcbed1..54f6d63fa7 100644
 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
 +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
 @@ -179,6 +179,8 @@ public class DatanodeManager {
@@ -50,7 +62,7 @@ index 44dffcbed11..54f6d63fa78 100644
      this.checkIpHostnameInRegistration = conf.getBoolean(
          DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY,
          DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT);
-@@ -1133,27 +1140,29 @@ void startAdminOperationIfNecessary(DatanodeDescriptor nodeReg) {
+@@ -1133,27 +1140,29 @@ public class DatanodeManager {
     */
    public void registerDatanode(DatanodeRegistration nodeReg)
        throws DisallowedDatanodeException, UnresolvedTopologyException {
@@ -98,7 +110,7 @@ index 44dffcbed11..54f6d63fa78 100644
        // it will be disallowed from registering. 
        if (!hostConfigManager.isIncluded(nodeReg)) {
 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
-index d61a17e83fe..eaf4a6d7c1d 100644
+index d61a17e83f..eaf4a6d7c1 100644
 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
 +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
 @@ -99,6 +99,11 @@ public class DNConf {
@@ -113,7 +125,7 @@ index d61a17e83fe..eaf4a6d7c1d 100644
    final boolean overwriteDownstreamDerivedQOP;
    private final boolean pmemCacheRecoveryEnabled;
 
-@@ -187,6 +192,11 @@ public DNConf(final Configurable dn) {
+@@ -187,6 +192,11 @@ public class DNConf {
      connectToDnViaHostname = getConf().getBoolean(
          DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
          DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
@@ -125,7 +137,7 @@ index d61a17e83fe..eaf4a6d7c1d 100644
      this.blockReportInterval = getConf().getLong(
          DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
          DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT);
-@@ -360,6 +370,66 @@ public boolean getConnectToDnViaHostname() {
+@@ -360,6 +370,66 @@ public class DNConf {
      return connectToDnViaHostname;
    }
 
@@ -193,18 +205,18 @@ index d61a17e83fe..eaf4a6d7c1d 100644
     * Returns socket timeout
     * 
 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
-index c1507a45120..d253779e70d 100644
+index c1507a4512..2ff7c272cf 100644
 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
-@@ -82,6 +82,7 @@
+@@ -82,6 +82,7 @@ import java.util.HashSet;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
 +import java.util.Optional;
  import java.util.Map.Entry;
  import java.util.Set;
  import java.util.UUID;
-@@ -1556,11 +1557,35 @@ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) {
+@@ -1556,11 +1557,35 @@ public class DataNode extends ReconfigurableBase
            NodeType.DATA_NODE);
      }
 

hadoop/stackable/patches/3.3.4/003-HADOOP-18055-3.3.4.patch renamed to hadoop/stackable/patches/3.3.4/0003-Add-async-profiler.patch

@@ -1,16 +1,41 @@
+From 19020c39bf1393d9a1ba590d1f3d4a79dc20806d Mon Sep 17 00:00:00 2001
+From: Siegfried Weber <[email protected]>
+Date: Tue, 6 Feb 2024 16:10:54 +0100
+Subject: Add async-profiler
+
+---
+ .../org/apache/hadoop/http/HttpServer2.java   |  21 +
+ .../hadoop/http/ProfileOutputServlet.java     |  87 ++++
+ .../apache/hadoop/http/ProfileServlet.java    | 394 ++++++++++++++++++
+ .../hadoop/http/ProfilerDisabledServlet.java  |  44 ++
+ .../org/apache/hadoop/util/ProcessUtils.java  |  74 ++++
+ .../src/main/resources/core-default.xml       |   2 +-
+ .../src/site/markdown/AsyncProfilerServlet.md | 145 +++++++
+ .../http/TestDisabledProfileServlet.java      |  95 +++++
+ .../hadoop-kms/src/site/markdown/index.md.vm  |   5 +-
+ .../src/site/markdown/ServerSetup.md.vm       |   5 +-
+ hadoop-project/src/site/site.xml              |   1 +
+ 11 files changed, 868 insertions(+), 5 deletions(-)
+ create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java
+ create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
+ create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
+ create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java
+ create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md
+ create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java
+
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
-index 76e77560a58bf..fb090fe4385a1 100644
+index b40d60cf50..a7777f85cc 100644
 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
-@@ -27,6 +27,7 @@
+@@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
  import java.net.MalformedURLException;
  import java.net.URI;
  import java.net.URL;
 +import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.Paths;
  import java.util.List;
-@@ -771,6 +772,26 @@ private void initializeWebServer(String name, String hostName,
+@@ -734,6 +735,26 @@ public final class HttpServer2 implements FilterContainer {
 
      addDefaultServlets();
      addPrometheusServlet(conf);
@@ -39,7 +64,7 @@ index 76e77560a58bf..fb090fe4385a1 100644
    private void addPrometheusServlet(Configuration conf) {
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java
 new file mode 100644
-index 0000000000000..1ecc21f3753ce
+index 0000000000..1ecc21f375
 --- /dev/null
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java
 @@ -0,0 +1,87 @@
@@ -132,7 +157,7 @@ index 0000000000000..1ecc21f3753ce
 +}
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
 new file mode 100644
-index 0000000000000..fc0ec7736ed8e
+index 0000000000..fc0ec7736e
 --- /dev/null
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
 @@ -0,0 +1,394 @@
@@ -532,7 +557,7 @@ index 0000000000000..fc0ec7736ed8e
 +}
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
 new file mode 100644
-index 0000000000000..459485ffa5b56
+index 0000000000..459485ffa5
 --- /dev/null
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
 @@ -0,0 +1,44 @@
@@ -582,7 +607,7 @@ index 0000000000000..459485ffa5b56
 +}
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java
 new file mode 100644
-index 0000000000000..cf653b9c912c4
+index 0000000000..cf653b9c91
 --- /dev/null
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java
 @@ -0,0 +1,74 @@
@@ -661,10 +686,10 @@ index 0000000000000..cf653b9c912c4
 +  }
 +}
 diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
-index 27c86bbc9ac8b..dc2a6ffd8375a 100644
+index f94fdebd03..56bec769c1 100644
 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
 +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
-@@ -78,7 +78,7 @@
+@@ -69,7 +69,7 @@
    <value>false</value>
    <description>
      Indicates if administrator ACLs are required to access
@@ -675,7 +700,7 @@ index 27c86bbc9ac8b..dc2a6ffd8375a 100644
 
 diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md b/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md
 new file mode 100644
-index 0000000000000..4b93cc219a5ee
+index 0000000000..4b93cc219a
 --- /dev/null
 +++ b/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md
 @@ -0,0 +1,145 @@
@@ -826,7 +851,7 @@ index 0000000000000..4b93cc219a5ee
 +
 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java
 new file mode 100644
-index 0000000000000..ce068bb6f1cf6
+index 0000000000..ce068bb6f1
 --- /dev/null
 +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java
 @@ -0,0 +1,95 @@
@@ -926,7 +951,7 @@ index 0000000000000..ce068bb6f1cf6
 +
 +}
 diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
-index 6ea21d5cf407d..09375d5aab528 100644
+index 6ea21d5cf4..09375d5aab 100644
 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
 +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
 @@ -1208,9 +1208,10 @@ Name               | Description
@@ -951,7 +976,7 @@ index 6ea21d5cf407d..09375d5aab528 100644
    </property>
 
 diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
-index 2d0a5b8cd2e7d..e97de0275ca22 100644
+index 2d0a5b8cd2..e97de0275c 100644
 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
 +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
 @@ -162,9 +162,10 @@ Name               | Description
@@ -976,10 +1001,10 @@ index 2d0a5b8cd2e7d..e97de0275ca22 100644
    </property>
 
 diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
-index a150385048f2f..6c0233877b053 100644
+index e2d149da2e..c5413d9089 100644
 --- a/hadoop-project/src/site/site.xml
 +++ b/hadoop-project/src/site/site.xml
-@@ -69,6 +69,7 @@
+@@ -74,6 +74,7 @@
        <item name="Tracing" href="hadoop-project-dist/hadoop-common/Tracing.html"/>
        <item name="Unix Shell Guide" href="hadoop-project-dist/hadoop-common/UnixShellGuide.html"/>
        <item name="Registry" href="hadoop-project-dist/hadoop-common/registry/index.html"/>

hadoop/stackable/patches/3.3.4/004-HADOOP-18077-3.3.4.patch renamed to hadoop/stackable/patches/3.3.4/0004-Backport-HADOOP-18077.patch

@@ -1,8 +1,18 @@
+From 892037bfc230b7d4401e234002f8388859685326 Mon Sep 17 00:00:00 2001
+From: Siegfried Weber <[email protected]>
+Date: Tue, 6 Feb 2024 16:10:54 +0100
+Subject: Backport HADOOP-18077
+
+---
+ .../src/main/java/org/apache/hadoop/http/HttpServer2.java | 6 ++++--
+ .../org/apache/hadoop/http/ProfilerDisabledServlet.java   | 8 +++++++-
+ 2 files changed, 11 insertions(+), 3 deletions(-)
+
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
-index fb090fe4385a1..49807ac4b4597 100644
+index a7777f85cc..bbe0d9993d 100644
 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
-@@ -772,10 +772,11 @@ private void initializeWebServer(String name, String hostName,
+@@ -735,10 +735,11 @@ public final class HttpServer2 implements FilterContainer {
 
      addDefaultServlets();
      addPrometheusServlet(conf);
@@ -16,7 +26,7 @@ index fb090fe4385a1..49807ac4b4597 100644
      final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome();
      if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) {
        addServlet("prof", "/prof", ProfileServlet.class);
-@@ -787,6 +788,7 @@ private void addAsyncProfilerServlet(ContextHandlerCollection contexts) throws I
+@@ -750,6 +751,7 @@ public final class HttpServer2 implements FilterContainer {
        genCtx.addServlet(ProfileOutputServlet.class, "/*");
        genCtx.setResourceBase(tmpDir.toAbsolutePath().toString());
        genCtx.setDisplayName("prof-output-hadoop");
@@ -25,10 +35,10 @@ index fb090fe4385a1..49807ac4b4597 100644
        addServlet("prof", "/prof", ProfilerDisabledServlet.class);
        LOG.info("ASYNC_PROFILER_HOME environment variable and async.profiler.home system property "
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
-index 459485ffa5b56..c488b574990cc 100644
+index 459485ffa5..c488b57499 100644
 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
-@@ -36,9 +36,15 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res
+@@ -36,9 +36,15 @@ public class ProfilerDisabledServlet extends HttpServlet {
        throws IOException {
      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
      ProfileServlet.setResponseHeader(resp);

hadoop/stackable/patches/3.3.4/005-add-perf-event-itimer-3.3.4.patch renamed to hadoop/stackable/patches/3.3.4/0005-Async-profiler-also-grab-itimer-events.patch

@@ -1,8 +1,17 @@
+From 522e3e03546bb977dca8cc3706f9d1fa6cb82acf Mon Sep 17 00:00:00 2001
+From: Siegfried Weber <[email protected]>
+Date: Tue, 6 Feb 2024 16:10:54 +0100
+Subject: Async-profiler: also grab itimer events
+
+---
+ .../src/main/java/org/apache/hadoop/http/ProfileServlet.java    | 2 ++
+ 1 file changed, 2 insertions(+)
+
 diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
-index fc0ec7736ed8..e324ad6d49fd 100644
+index fc0ec7736e..e324ad6d49 100644
 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
 +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
-@@ -76,6 +76,7 @@
+@@ -76,6 +76,7 @@ import org.apache.hadoop.util.ProcessUtils;
   * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events)
   * // Perf events:
   * //    cpu