remove client ports, adapt readiness probe & tests

Author: maltesander

rust/operator-binary/src/crd/security.rs

@@ -65,12 +65,11 @@ pub struct ZookeeperSecurity {
 impl ZookeeperSecurity {
     // ports
     pub const CLIENT_PORT: u16 = 2181;
-    pub const CLIENT_PORT_NAME: &'static str = "clientPort";
+    pub const SECURE_CLIENT_PORT: u16 = 2282;
+    pub const ADMIN_PORT: u16 = 8080;
     // directories
     pub const QUORUM_TLS_DIR: &'static str = "/stackable/quorum_tls";
     pub const QUORUM_TLS_MOUNT_DIR: &'static str = "/stackable/quorum_tls_mount";
-    pub const SECURE_CLIENT_PORT: u16 = 2282;
-    pub const SECURE_CLIENT_PORT_NAME: &'static str = "secureClientPort";
     pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory";
     pub const SERVER_TLS_DIR: &'static str = "/stackable/server_tls";
     pub const SERVER_TLS_MOUNT_DIR: &'static str = "/stackable/server_tls_mount";
@@ -220,42 +219,6 @@ impl ZookeeperSecurity {
 
         // Server TLS
         if self.tls_enabled() {
-            // We set only the clientPort and portUnification here because otherwise there is a port bind exception
-            // See: https://issues.apache.org/jira/browse/ZOOKEEPER-4276
-            // --> Normally we would like to only set the secureClientPort (check out commented code below)
-            // What we tried:
-            // 1) Set clientPort and secureClientPort will fail with
-            // "static.config different from dynamic config .. "
-            // config.insert(
-            //     Self::CLIENT_PORT_NAME.to_string(),
-            //     CLIENT_PORT.to_string(),
-            // );
-            // config.insert(
-            //     Self::SECURE_CLIENT_PORT_NAME.to_string(),
-            //     SECURE_CLIENT_PORT.to_string(),
-            // );
-
-            // 2) Setting only secureClientPort will config in the above mentioned bind exception.
-            // The NettyFactory tries to bind multiple times on the secureClientPort.
-            // config.insert(
-            //     Self::SECURE_CLIENT_PORT_NAME.to_string(),
-            //     self.client_port(.to_string()),
-            // );
-
-            // 3) Using the clientPort and portUnification still allows plaintext connection without
-            // authentication, but at least TLS and authentication works when connecting securely.
-            config.insert(
-                Self::CLIENT_PORT_NAME.to_string(),
-                self.client_port().to_string(),
-            );
-            config.insert("client.portUnification".to_string(), "true".to_string());
-            // TODO: Remove clientPort and portUnification (above) in favor of secureClientPort once the bug is fixed
-            // config.insert(
-            //     Self::SECURE_CLIENT_PORT_NAME.to_string(),
-            //     self.client_port(.to_string()),
-            // );
-            // END TICKET
-
             config.insert(
                 Self::SSL_HOST_NAME_VERIFICATION.to_string(),
                 "true".to_string(),
@@ -278,11 +241,6 @@ impl ZookeeperSecurity {
             {
                 config.insert(Self::SSL_CLIENT_AUTH.to_string(), "need".to_string());
             }
-        } else {
-            config.insert(
-                Self::CLIENT_PORT_NAME.to_string(),
-                self.client_port().to_string(),
-            );
         }
 
         config

rust/operator-binary/src/zk_controller.rs

@@ -573,12 +573,20 @@ fn build_server_rolegroup_config_map(
         .into_iter()
         .flatten()
         .map(|pod| {
+            let port = if zookeeper_security.tls_enabled() {
+                // The docs state for TLS-only server
+                // server.5 = <host>:<leader_port>:<election_port>;;<secureClientPort> (TLS-only server)
+                format!(";{port}", port = zookeeper_security.client_port())
+            } else {
+                // The docs state for non-TLS server
+                // server.5 = <host>:<leader_port>:<election_port>;<clientPort> (TLS-only server)
+                format!("{port}", port = zookeeper_security.client_port())
+            };
             (
                 format!("server.{id}", id = pod.zookeeper_myid),
                 format!(
-                    "{internal_fqdn}:{ZOOKEEPER_LEADER_PORT}:{ZOOKEEPER_ELECTION_PORT};{client_port}",
+                    "{internal_fqdn}:{ZOOKEEPER_LEADER_PORT}:{ZOOKEEPER_ELECTION_PORT};{port}",
                     internal_fqdn = pod.internal_fqdn(cluster_info),
-                    client_port = zookeeper_security.client_port()
                 ),
             )
         })
@@ -859,11 +867,10 @@ fn build_server_rolegroup_statefulset(
                 command: Some(vec![
                     "bash".to_string(),
                     "-c".to_string(),
-                    // We don't have telnet or netcat in the container images, but
-                    // we can use Bash's virtual /dev/tcp filesystem to accomplish the same thing
+                    // NOTE: the adminPort property is currently generated in the product config machinery properties.yaml.
                     format!(
-                        "exec 3<>/dev/tcp/127.0.0.1/{} && echo srvr >&3 && grep '^Mode: ' <&3",
-                        zookeeper_security.client_port()
+                        r#"curl -s http://127.0.0.1:{admin_port}/commands/mntr | grep -q '"server_state"[ ]*:[ ]*"\(leader\|follower\)"'"#,
+                        admin_port = ZookeeperSecurity::ADMIN_PORT
                     ),
                 ]),
             }),

tests/templates/kuttl/smoke/test_tls.sh.j2

@@ -18,12 +18,12 @@ echo "Start TLS testing..."
 ############################################################################
 # Test the plaintext unsecured connection
 ############################################################################
-if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null;
+if /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null;
 then
-  echo "[ERROR] Could not establish unsecure connection!"
+  echo "[ERROR] Could establish unsecure connection!"
   exit 1
 fi
-echo "[SUCCESS] Unsecure client connection established!"
+echo "[SUCCESS] Could not establish unsecure connection!"
 
 ############################################################################
 # We set the correct client tls credentials and expect to be able to connect