feat(zookeeper): backport ZOOKEEPER-4846, ZOOKEEPER-4921, ZOOKEEPER-4925 into Zookeeper 3.9.3

CHANGELOG.md

@@ -43,14 +43,14 @@ All notable changes to this project will be documented in this file.
 - nifi: Add [nifi-iceberg-bundle](https://github.com/stackabletech/nifi-iceberg-bundle) for NiFi `2.2.0` ([#1060], [#1106]).
 - java: Add JDK 24 ([#1097]).
 - ci: Add golang image to mirror workflow ([#1103]).
-- omid: bump version to 1.1.3 ([#1105])
-- hbase: add 2.6.2 and upgrade dependencies ([#1101])
-- kafka: Add `4.0.0` ([#1117])
-- Include `.tar.gz` snapshots of the product source code in container images ([#1126])
-- airflow: OPA authorizer for Airflow 3.x ([#1127])
-- kafka: Add `3.9.1` ([#1149])
-- spark-k8s: Add `3.5.6` ([#1142])
-- spark-connect-client: Add `3.5.6` ([#1142])
+- omid: bump version to 1.1.3 ([#1105]).
+- hbase: add 2.6.2 and upgrade dependencies ([#1101]).
+- kafka: Add `4.0.0` ([#1117]).
+- Include `.tar.gz` snapshots of the product source code in container images ([#1126]).
+- airflow: OPA authorizer for Airflow 3.x ([#1127]).
+- kafka: Add `3.9.1` ([#1149]).
+- spark-k8s: Add `3.5.6` ([#1142]).
+- spark-connect-client: Add `3.5.6` ([#1142]).
 - git-sync: Bump version to 4.4.1 ([#1151]).
 
 ### Changed
@@ -66,16 +66,17 @@ All notable changes to this project will be documented in this file.
 - yq: Bump products to use `4.45.2` ([#1090]).
 - cyclonedx-bom: Bump airflow and superset to use `6.0.0` ([#1090]).
 - vector: Bump to `0.46.1` ([#1098]).
-- spark: update dependencies for 3.5.5 ([#1094])
-- nifi: include NAR SBOMs ([#1119])
+- spark: update dependencies for 3.5.5 ([#1094]).
+- nifi: include NAR SBOMs ([#1119]).
 - nifi: update patch allowing to bypass host header validation starting with NiFi 2.4.0 ([#1125]).
 - BREAKING: kcat: Stop building kcat image ([#1124]).
-- containerdebug updated to 0.2.0 ([#1128])
-- Build Hadoop as `stackable` and configure the Stackable Nexus build-repo for the `root` user ([#1133])
+- containerdebug updated to 0.2.0 ([#1128]).
+- Build Hadoop as `stackable` and configure the Stackable Nexus build-repo for the `root` user ([#1133]).
 - patchable: The base branch is now configured as the git upstream branch ([#1131]).
-- airflow: Updates the entrypoint script and removes the check for GID == 0 ([#1138])
+- airflow: Updates the entrypoint script and removes the check for GID == 0 ([#1138]).
 - druid: Bump druiod-opa-authorizer to `0.7.0` ([#1139]).
 - vector: Bump to `0.47.0` ([#1152]).
+- zookeeper: backport ZOOKEEPER-4846, ZOOKEEPER-4921, ZOOKEEPER-4925 into Zookeeper 3.9.3 ([#1150]).
 
 ### Fixed
 
@@ -105,16 +106,16 @@ All notable changes to this project will be documented in this file.
   Also remove the old release workflow.
 - zookeeper: Remove 3.9.2 ([#1093]).
 - Remove ubi8-rust-builder image ([#1091]).
-- spark: remove 3.5.2 ([#1094])
+- spark: remove 3.5.2 ([#1094]).
 - hadoop: Remove `3.3.4` and `3.4.0` ([#1099]).
 - opa: Remove `0.67.1` ([#1103]).
 - opa: Remove legacy bundle-builder from container build ([#1103]).
 - omid: Remove 1.1.3-SNAPSHOT ([#1105]).
 - hbase: Remove 2.4.18 ([#1101])
 - druid: Remove `30.0.0` ([#1110]).
 - nifi: Remove `2.2.0` ([#1114]).
-- kafka: Remove `3.7.1` and `3.8.0` ([#1117])
-- spark-connect-client: Remove `3.5.5` ([#1142])
+- kafka: Remove `3.7.1` and `3.8.0` ([#1117]).
+- spark-connect-client: Remove `3.5.5` ([#1142]).
 
 [nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle
 [#1025]: https://github.com/stackabletech/docker-images/pull/1025
@@ -169,8 +170,9 @@ All notable changes to this project will be documented in this file.
 [#1137]: https://github.com/stackabletech/docker-images/pull/1137
 [#1138]: https://github.com/stackabletech/docker-images/pull/1138
 [#1139]: https://github.com/stackabletech/docker-images/pull/1139
-[#1149]: https://github.com/stackabletech/docker-images/pull/1149
 [#1142]: https://github.com/stackabletech/docker-images/pull/1142
+[#1149]: https://github.com/stackabletech/docker-images/pull/1149
+[#1150]: https://github.com/stackabletech/docker-images/pull/1150
 [#1151]: https://github.com/stackabletech/docker-images/pull/1151
 [#1152]: https://github.com/stackabletech/docker-images/pull/1152
 

zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch

@@ -8,7 +8,7 @@ Subject: Add CycloneDX plugin
  1 file changed, 6 insertions(+), 1 deletion(-)
 
 diff --git a/pom.xml b/pom.xml
-index 6ef4011f..07ae7538 100644
+index 6ef4011fe..07ae75387 100644
 --- a/pom.xml
 +++ b/pom.xml
 @@ -925,7 +925,7 @@

zookeeper/stackable/patches/3.9.3/0002-ZOOKEEPER-4846-Failure-to-reload-database-due-to-mis.patch

@@ -0,0 +1,64 @@
+From 4004002a9ff08a539a94842ea12a2a449274e968 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Andor=20Moln=C3=A1r?= <[email protected]>
+Date: Tue, 11 Feb 2025 10:43:20 -0600
+Subject: ZOOKEEPER-4846: Failure to reload database due to missing ACL
+
+ZOOKEEPER-4846. Fix ACL reference on existing znode when trying to create
+Reviewers: cnauroth, eolivelli, ztzg
+Author: anmolnar
+Closes #2222 from anmolnar/ZOOKEEPER-4846
+---
+ .../org/apache/zookeeper/server/DataTree.java    |  5 +++--
+ .../apache/zookeeper/server/DataTreeTest.java    | 16 ++++++++++++++++
+ 2 files changed, 19 insertions(+), 2 deletions(-)
+
+diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java
+index 3b61c80d8..af937f834 100644
+--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java
++++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java
+@@ -462,8 +462,9 @@ public class DataTree {
+             // we did for the global sessions.
+             Long acls = aclCache.convertAcls(acl);
+
+-            Set<String> children = parent.getChildren();
+-            if (children.contains(childName)) {
++            DataNode existingChild = nodes.get(path);
++            if (existingChild != null) {
++                existingChild.acl = acls;
+                 throw new NodeExistsException();
+             }
+
+diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java
+index 07a69f14f..fc20ed320 100644
+--- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java
++++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/DataTreeTest.java
+@@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
+ import static org.junit.jupiter.api.Assertions.assertNotEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ import static org.junit.jupiter.api.Assertions.assertNull;
++import static org.junit.jupiter.api.Assertions.assertThrows;
+ import static org.junit.jupiter.api.Assertions.assertTrue;
+ import java.io.ByteArrayInputStream;
+ import java.io.ByteArrayOutputStream;
+@@ -631,6 +632,21 @@ public class DataTreeTest extends ZKTestCase {
+         }
+     }
+
++    @Test
++    public void testCreateNodeFixMissingACL() throws Exception {
++        DataTree dt = new DataTree();
++        ReferenceCountedACLCache aclCache = dt.getReferenceCountedAclCache();
++
++        dt.createNode("/the_parent", new byte[0], ZooDefs.Ids.CREATOR_ALL_ACL, -1, 1, 1, 0);
++        Long aclId = dt.getNode("/the_parent").acl;
++        aclCache.removeUsage(aclId);
++        aclCache.purgeUnused();
++        // try to re-create the parent -> throws NodeExistsException, but fixes the deleted ACL
++        assertThrows(NodeExistsException.class, () ->
++            dt.createNode("/the_parent", new byte[0], ZooDefs.Ids.CREATOR_ALL_ACL, -1, 1, 1, 0));
++        dt.createNode("/the_parent/the_child", new byte[0], ZooDefs.Ids.CREATOR_ALL_ACL, -1, 2, 2, 2);
++    }
++
+     private DataTree buildDataTreeForTest() {
+         final DataTree dt = new DataTree();
+         assertEquals(dt.lastProcessedZxid, 0);

zookeeper/stackable/patches/3.9.3/0003-ZOOKEEPER-4921-Retry-endlessly-to-establish-a-brand-.patch

@@ -0,0 +1,143 @@
+From 90e8e0f44e8a884765b6e7afe8bd779d59136fad Mon Sep 17 00:00:00 2001
+From: Kezhu Wang <[email protected]>
+Date: Sat, 26 Apr 2025 12:04:01 +0800
+Subject: ZOOKEEPER-4921: Retry endlessly to establish a brand-new session
+
+This partially rollback ZOOKEEPER-4508 to keep consistent with versions
+prior to 3.9.3 (excluded), so to maintain compatibility with third party
+libraries.
+
+Refs: ZOOKEEPER-4508, ZOOKEEPER-4921, ZOOKEEPER-4923 and
+https://lists.apache.org/thread/nfb9z7rhgglbjzfxvg4z2m3pks53b3c1
+---
+ .../java/org/apache/zookeeper/ClientCnxn.java |  2 +-
+ .../zookeeper/test/SessionTimeoutTest.java    | 65 +++++++++++++------
+ 2 files changed, 47 insertions(+), 20 deletions(-)
+
+diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java
+index 0bf616c61..207bb8c49 100644
+--- a/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java
++++ b/zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java
+@@ -1242,7 +1242,7 @@ public class ClientCnxn {
+                         to = connectTimeout - clientCnxnSocket.getIdleSend();
+                     }
+
+-                    int expiration = expirationTimeout - clientCnxnSocket.getIdleRecv();
++                    int expiration = sessionId == 0 ? Integer.MAX_VALUE : expirationTimeout - clientCnxnSocket.getIdleRecv();
+                     if (expiration <= 0) {
+                         String warnInfo = String.format(
+                             "Client session timed out, have not heard from server in %dms for session id 0x%s",
+diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java
+index 7a59f5eb9..9f5943f68 100644
+--- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java
++++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/SessionTimeoutTest.java
+@@ -18,6 +18,9 @@
+
+ package org.apache.zookeeper.test;
+
++import static org.hamcrest.MatcherAssert.assertThat;
++import static org.hamcrest.Matchers.greaterThanOrEqualTo;
++import static org.hamcrest.Matchers.lessThan;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ import static org.junit.jupiter.api.Assertions.assertNull;
+ import static org.junit.jupiter.api.Assertions.assertThrows;
+@@ -31,12 +34,15 @@ import java.util.List;
+ import java.util.concurrent.CompletableFuture;
+ import java.util.concurrent.CountDownLatch;
+ import java.util.concurrent.TimeUnit;
++import java.util.concurrent.TimeoutException;
+ import org.apache.zookeeper.CreateMode;
+ import org.apache.zookeeper.KeeperException;
+ import org.apache.zookeeper.TestableZooKeeper;
+ import org.apache.zookeeper.WatchedEvent;
+ import org.apache.zookeeper.Watcher;
+ import org.apache.zookeeper.ZooDefs;
++import org.apache.zookeeper.ZooKeeper;
++import org.apache.zookeeper.common.Time;
+ import org.junit.jupiter.api.BeforeEach;
+ import org.junit.jupiter.api.Test;
+ import org.slf4j.Logger;
+@@ -54,6 +60,21 @@ public class SessionTimeoutTest extends ClientBase {
+         zk = createClient();
+     }
+
++    private static class ExpiredWatcher implements Watcher {
++        public volatile CompletableFuture<Void> expired = new CompletableFuture<>();
++
++        synchronized void reset() {
++            expired = new CompletableFuture<>();
++        }
++
++        @Override
++        public synchronized void process(WatchedEvent event) {
++            if (event.getState() == Event.KeeperState.Expired) {
++                expired.complete(null);
++            }
++        }
++    }
++
+     private static class BusyServer implements AutoCloseable {
+         private final ServerSocket server;
+         private final Socket client;
+@@ -143,17 +164,24 @@ public class SessionTimeoutTest extends ClientBase {
+         // stop client also to gain less distraction
+         zk.close();
+
+-        // small connection timeout to gain quick ci feedback
+-        int sessionTimeout = 3000;
+-        CompletableFuture<Void> expired = new CompletableFuture<>();
++        // given: established session
++        int sessionTimeout = 3000; // small connection timeout to gain quick ci feedback
++        ExpiredWatcher watcher = new ExpiredWatcher();
+         zk = createClient(new CountdownWatcher(), hostPort, sessionTimeout);
+-        zk.register(event -> {
+-            if (event.getState() == Watcher.Event.KeeperState.Expired) {
+-                expired.complete(null);
+-            }
+-        });
++        zk.register(watcher);
++
++        // when: all server down
++        long start = Time.currentElapsedTime();
++        zk.sync("/"); // touch timeout counts
+         stopServer();
+-        expired.join();
++
++        // then: get Expired after session timeout
++        watcher.expired.join();
++        long elapsed = Time.currentElapsedTime() - start;
++        assertThat(elapsed, greaterThanOrEqualTo((long) zk.getSessionTimeout()));
++        assertThat(elapsed, lessThan(zk.getSessionTimeout() * 10L));
++
++        // then: future request will get SessionExpiredException
+         assertThrows(KeeperException.SessionExpiredException.class, () -> zk.exists("/", null));
+     }
+
+@@ -162,18 +190,17 @@ public class SessionTimeoutTest extends ClientBase {
+         // stop client also to gain less distraction
+         zk.close();
+
++        // given: unavailable cluster
+         stopServer();
+
+-        // small connection timeout to gain quick ci feedback
+-        int sessionTimeout = 3000;
+-        CompletableFuture<Void> expired = new CompletableFuture<>();
+-        new TestableZooKeeper(hostPort, sessionTimeout, event -> {
+-            if (event.getState() == Watcher.Event.KeeperState.Expired) {
+-                expired.complete(null);
+-            }
+-        });
+-        expired.join();
+-        assertThrows(KeeperException.SessionExpiredException.class, () -> zk.exists("/", null));
++        // when: try to establish a brand-new session
++        int sessionTimeout = 300; // small connection timeout to gain quick ci feedback
++        ExpiredWatcher watcher = new ExpiredWatcher();
++        try (ZooKeeper zk = new ZooKeeper(hostPort, sessionTimeout, watcher)) {
++            // then: never Expired
++            assertThrows(TimeoutException.class, () -> watcher.expired.get(3 * sessionTimeout, TimeUnit.MILLISECONDS));
++            assertThrows(KeeperException.ConnectionLossException.class, () -> zk.exists("/", null));
++        }
+     }
+
+     @Test