adapt to op-rs 0.79.0

Author: maltesander

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- The operator can now run on Kubernetes clusters using a non-default cluster domain. It should automatically detect the
+  correct domain to use, but you can also use the env var `KUBERNETES_CLUSTER_DOMAIN` to set the domain explicitly
+  or use the helm-chart property `kubernetesClusterDomain` ([#xxx]).
+
 ### Changed
 
 - Reduce CRD size from `483KB` to `57KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#853]):
@@ -16,6 +22,7 @@ All notable changes to this project will be documented in this file.
 
 [#853]: https://github.com/stackabletech/zookeeper-operator/pull/853
 [#857]: https://github.com/stackabletech/zookeeper-operator/pull/857
+[#xxx]: https://github.com/stackabletech/zookeeper-operator/pull/xxx
 
 ## [24.7.0] - 2024-07-24
 

Cargo.lock

@@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.76.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.79.0" }
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.40", features = ["full"] }

Cargo.toml

@@ -47,3 +47,8 @@ nodeSelector: {}
 tolerations: []
 
 affinity: {}
+
+# When running on a non-default Kubernetes cluster domain and the auto detection is not working correctly,
+# you can set your custom cluster domain here.
+# See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details
+# kubernetesClusterDomain: my-cluster.local

deploy/helm/zookeeper-operator/values.yaml

@@ -73,7 +73,8 @@ impl ResolvedAuthenticationClasses {
                 AuthenticationClassProvider::Tls(_) => {}
                 AuthenticationClassProvider::Ldap(_)
                 | AuthenticationClassProvider::Oidc(_)
-                | AuthenticationClassProvider::Static(_) => {
+                | AuthenticationClassProvider::Static(_)
+                | AuthenticationClassProvider::Kerberos(_) => {
                     return Err(Error::AuthenticationMethodNotSupported {
                         authentication_class: ObjectRef::from_obj(auth_class),
                         method: auth_class.spec.provider.to_string(),

rust/crd/src/authentication.rs

@@ -29,6 +29,7 @@ use stackable_operator::{
     schemars::{self, JsonSchema},
     status::condition::{ClusterCondition, HasStatusCondition},
     time::Duration,
+    utils::cluster_domain::KUBERNETES_CLUSTER_DOMAIN,
 };
 use strum::{Display, EnumIter, EnumString, IntoEnumIterator};
 
@@ -483,10 +484,14 @@ impl ZookeeperCluster {
 
     /// The fully-qualified domain name of the role-level load-balanced Kubernetes `Service`
     pub fn server_role_service_fqdn(&self) -> Option<String> {
+        let cluster_domain = KUBERNETES_CLUSTER_DOMAIN
+            .get()
+            .expect("Could not resolve the Kubernetes cluster domain!");
         Some(format!(
-            "{}.{}.svc.cluster.local",
+            "{}.{}.svc.{}",
             self.server_role_service_name()?,
-            self.metadata.namespace.as_ref()?
+            self.metadata.namespace.as_ref()?,
+            cluster_domain
         ))
     }
 
@@ -667,9 +672,12 @@ pub struct ZookeeperPodRef {
 
 impl ZookeeperPodRef {
     pub fn fqdn(&self) -> String {
+        let cluster_domain = KUBERNETES_CLUSTER_DOMAIN
+            .get()
+            .expect("Could not resolve the Kubernetes cluster domain!");
         format!(
-            "{}.{}.{}.svc.cluster.local",
-            self.pod_name, self.role_group_service_name, self.namespace
+            "{}.{}.{}.svc.{}",
+            self.pod_name, self.role_group_service_name, self.namespace, cluster_domain
         )
     }
 }

rust/crd/src/lib.rs

@@ -8,13 +8,16 @@ use std::collections::BTreeMap;
 
 use snafu::{ResultExt, Snafu};
 use stackable_operator::{
-    builder::pod::{
-        container::ContainerBuilder,
-        volume::{
-            SecretFormat, SecretOperatorVolumeSourceBuilder,
-            SecretOperatorVolumeSourceBuilderError, VolumeBuilder,
+    builder::{
+        self,
+        pod::{
+            container::ContainerBuilder,
+            volume::{
+                SecretFormat, SecretOperatorVolumeSourceBuilder,
+                SecretOperatorVolumeSourceBuilderError, VolumeBuilder,
+            },
+            PodBuilder,
         },
-        PodBuilder,
     },
     client::Client,
     commons::authentication::AuthenticationClassProvider,
@@ -35,6 +38,14 @@ pub enum Error {
         source: SecretOperatorVolumeSourceBuilderError,
         volume_name: String,
     },
+
+    #[snafu(display("failed to add needed volume"))]
+    AddVolume { source: builder::pod::Error },
+
+    #[snafu(display("failed to add needed volumeMount"))]
+    AddVolumeMount {
+        source: builder::pod::container::Error,
+    },
 }
 
 /// Helper struct combining TLS settings for server and quorum with the resolved AuthenticationClasses
@@ -141,17 +152,25 @@ impl ZookeeperSecurity {
 
         if let Some(secret_class) = tls_secret_class {
             let tls_volume_name = "server-tls";
-            cb_zookeeper.add_volume_mount(tls_volume_name, Self::SERVER_TLS_DIR);
-            pod_builder.add_volume(Self::create_tls_volume(tls_volume_name, secret_class)?);
+            cb_zookeeper
+                .add_volume_mount(tls_volume_name, Self::SERVER_TLS_DIR)
+                .context(AddVolumeMountSnafu)?;
+            pod_builder
+                .add_volume(Self::create_tls_volume(tls_volume_name, secret_class)?)
+                .context(AddVolumeSnafu)?;
         }
 
         // quorum
         let tls_volume_name = "quorum-tls";
-        cb_zookeeper.add_volume_mount(tls_volume_name, Self::QUORUM_TLS_DIR);
-        pod_builder.add_volume(Self::create_tls_volume(
-            tls_volume_name,
-            &self.quorum_secret_class,
-        )?);
+        cb_zookeeper
+            .add_volume_mount(tls_volume_name, Self::QUORUM_TLS_DIR)
+            .context(AddVolumeMountSnafu)?;
+        pod_builder
+            .add_volume(Self::create_tls_volume(
+                tls_volume_name,
+                &self.quorum_secret_class,
+            )?)
+            .context(AddVolumeSnafu)?;
 
         Ok(())
     }
@@ -264,7 +283,8 @@ impl ZookeeperSecurity {
                 AuthenticationClassProvider::Tls(tls) => tls.client_cert_secret_class.as_ref(),
                 AuthenticationClassProvider::Ldap(_)
                 | AuthenticationClassProvider::Oidc(_)
-                | AuthenticationClassProvider::Static(_) => None,
+                | AuthenticationClassProvider::Static(_)
+                | AuthenticationClassProvider::Kerberos(_) => None,
             })
             .or(self.server_secret_class.as_ref())
     }

rust/crd/src/security.rs

@@ -69,7 +69,9 @@ async fn main() -> anyhow::Result<()> {
                 "/etc/stackable/zookeeper-operator/config-spec/properties.yaml",
             ])?;
             let client =
-                stackable_operator::client::create_client(Some(OPERATOR_NAME.to_string())).await?;
+                stackable_operator::client::initialize_operator(Some(OPERATOR_NAME.to_string()))
+                    .await?;
+
             let zk_controller_builder = Controller::new(
                 watch_namespace.get_api::<ZookeeperCluster>(&client),
                 watcher::Config::default(),

rust/operator-binary/src/main.rs

@@ -17,6 +17,7 @@ use product_config::{
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::{
     builder::{
+        self,
         configmap::ConfigMapBuilder,
         meta::ObjectMetaBuilder,
         pod::{container::ContainerBuilder, resources::ResourceRequirementsBuilder, PodBuilder},
@@ -245,6 +246,14 @@ pub enum Error {
 
     #[snafu(display("failed to add TLS volume mounts"))]
     AddTlsVolumeMounts { source: security::Error },
+
+    #[snafu(display("failed to add needed volume"))]
+    AddVolume { source: builder::pod::Error },
+
+    #[snafu(display("failed to add needed volumeMount"))]
+    AddVolumeMount {
+        source: builder::pod::container::Error,
+    },
 }
 
 impl ReconcilerError for Error {
@@ -285,6 +294,8 @@ impl ReconcilerError for Error {
             Error::BuildLabel { .. } => None,
             Error::ObjectMeta { .. } => None,
             Error::AddTlsVolumeMounts { .. } => None,
+            Error::AddVolume { .. } => None,
+            Error::AddVolumeMount { .. } => None,
         }
     }
 }
@@ -801,9 +812,13 @@ fn build_server_rolegroup_statefulset(
             ..EnvVar::default()
         }])
         .add_volume_mount("data", STACKABLE_DATA_DIR)
+        .unwrap()
         .add_volume_mount("config", STACKABLE_CONFIG_DIR)
+        .unwrap()
         .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR)
+        .unwrap()
         .add_volume_mount("log", STACKABLE_LOG_DIR)
+        .unwrap()
         .resources(
             ResourceRequirementsBuilder::new()
                 .with_cpu_request("200m")
@@ -860,10 +875,15 @@ fn build_server_rolegroup_statefulset(
         .add_container_port("zk-election", 3888)
         .add_container_port("metrics", 9505)
         .add_volume_mount("data", STACKABLE_DATA_DIR)
+        .unwrap()
         .add_volume_mount("config", STACKABLE_CONFIG_DIR)
+        .unwrap()
         .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR)
+        .unwrap()
         .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR)
+        .unwrap()
         .add_volume_mount("log", STACKABLE_LOG_DIR)
+        .unwrap()
         .resources(resources)
         .build();
 
@@ -892,6 +912,7 @@ fn build_server_rolegroup_statefulset(
             }),
             ..Volume::default()
         })
+        .context(AddVolumeSnafu)?
         .add_volume(Volume {
             empty_dir: Some(EmptyDirVolumeSource {
                 medium: None,
@@ -900,12 +921,14 @@ fn build_server_rolegroup_statefulset(
             name: "rwconfig".to_string(),
             ..Volume::default()
         })
+        .context(AddVolumeSnafu)?
         .add_empty_dir_volume(
             "log",
             Some(product_logging::framework::calculate_log_volume_size_limit(
                 &[MAX_ZK_LOG_FILES_SIZE, MAX_PREPARE_LOG_FILE_SIZE],
             )),
         )
+        .context(AddVolumeSnafu)?
         .security_context(PodSecurityContext {
             run_as_user: Some(ZK_UID),
             run_as_group: Some(0),
@@ -921,38 +944,45 @@ fn build_server_rolegroup_statefulset(
             })),
     }) = logging.containers.get(&Container::Zookeeper)
     {
-        pod_builder.add_volume(Volume {
-            name: "log-config".to_string(),
-            config_map: Some(ConfigMapVolumeSource {
-                name: config_map.into(),
-                ..ConfigMapVolumeSource::default()
-            }),
-            ..Volume::default()
-        });
+        pod_builder
+            .add_volume(Volume {
+                name: "log-config".to_string(),
+                config_map: Some(ConfigMapVolumeSource {
+                    name: config_map.into(),
+                    ..ConfigMapVolumeSource::default()
+                }),
+                ..Volume::default()
+            })
+            .context(AddVolumeSnafu)?;
     } else {
-        pod_builder.add_volume(Volume {
-            name: "log-config".to_string(),
-            config_map: Some(ConfigMapVolumeSource {
-                name: rolegroup_ref.object_name(),
-                ..ConfigMapVolumeSource::default()
-            }),
-            ..Volume::default()
-        });
+        pod_builder
+            .add_volume(Volume {
+                name: "log-config".to_string(),
+                config_map: Some(ConfigMapVolumeSource {
+                    name: rolegroup_ref.object_name(),
+                    ..ConfigMapVolumeSource::default()
+                }),
+                ..Volume::default()
+            })
+            .context(AddVolumeSnafu)?;
     }
 
     if logging.enable_vector_agent {
-        pod_builder.add_container(product_logging::framework::vector_container(
-            resolved_product_image,
-            "config",
-            "log",
-            logging.containers.get(&Container::Vector),
-            ResourceRequirementsBuilder::new()
-                .with_cpu_request("250m")
-                .with_cpu_limit("500m")
-                .with_memory_request("128Mi")
-                .with_memory_limit("128Mi")
-                .build(),
-        ));
+        pod_builder.add_container(
+            product_logging::framework::vector_container(
+                resolved_product_image,
+                "config",
+                "log",
+                logging.containers.get(&Container::Vector),
+                ResourceRequirementsBuilder::new()
+                    .with_cpu_request("250m")
+                    .with_cpu_limit("500m")
+                    .with_memory_request("128Mi")
+                    .with_memory_limit("128Mi")
+                    .build(),
+            )
+            .unwrap(),
+        );
     }
 
     add_graceful_shutdown_config(config, &mut pod_builder).context(GracefulShutdownSnafu)?;