From 47ac444f42a51cb98e56b3b5b30d2bc421157246 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 2 Jul 2025 14:19:18 +0200 Subject: [PATCH 01/10] updating according to decision --- rust/operator-binary/src/controller.rs | 140 ++++++++++++++++--------- rust/operator-binary/src/crd/mod.rs | 39 ++++++- rust/operator-binary/src/discovery.rs | 4 - rust/operator-binary/src/listener.rs | 2 +- 4 files changed, 127 insertions(+), 58 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4c41390e..b3be9146 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -94,12 +94,12 @@ use crate::{ STACKABLE_LOG_DIR_NAME, v1alpha1::{self, HiveMetastoreRoleConfig}, }, - discovery::{self, build_headless_role_group_metrics_service_name}, + discovery::{self}, kerberos::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, }, - listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_role_listener}, + listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_role_listener, listener_ports}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::extend_role_group_config_map, }; @@ -456,7 +456,7 @@ pub async fn reconcile_hive( .merged_config(&HiveRole::MetaStore, &rolegroup) .context(FailedToResolveResourceConfigSnafu)?; - let rg_service = build_rolegroup_service(hive, &resolved_product_image, &rolegroup)?; + let rg_services = build_rolegroup_service(hive, &resolved_product_image, &rolegroup)?; let rg_configmap = build_metastore_rolegroup_config_map( hive, &hive_namespace, @@ -478,12 +478,13 @@ pub async fn reconcile_hive( &rbac_sa.name_any(), )?; - cluster_resources - .add(client, rg_service) - .await - .context(ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - })?; + for rg_service in rg_services { + cluster_resources.add(client, rg_service).await.context( + ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + }, + )?; + } cluster_resources .add(client, rg_configmap) @@ -720,39 +721,84 @@ fn build_rolegroup_service( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, -) -> Result { - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(hive) - .name(build_headless_role_group_metrics_service_name( - rolegroup.object_name(), - )) - .ownerreference_from_resource(hive, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(service_ports()), - selector: Some( - Labels::role_group_selector(hive, APP_NAME, &rolegroup.role, &rolegroup.role_group) +) -> Result> { + let services = vec![ + Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(hive) + // TODO: Use method on RoleGroupRef once op-rs is released + .name(hive.rolegroup_headless_metrics_service_name(rolegroup)) + .ownerreference_from_resource(hive, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .with_label( + Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?, + ) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(hive.metrics_ports()), + selector: Some( + Labels::role_group_selector( + hive, + APP_NAME, + &rolegroup.role, + &rolegroup.role_group, + ) .context(LabelBuildSnafu)? .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }) + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }, + Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(hive) + // TODO: Use method on RoleGroupRef once op-rs is released + .name(hive.rolegroup_headless_service_name(rolegroup)) + .ownerreference_from_resource(hive, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + // Expecting same ports as on listener service, just as a headless, internal service + ports: Some(hive.service_ports()), + selector: Some( + Labels::role_group_selector( + hive, + APP_NAME, + &rolegroup.role, + &rolegroup.role_group, + ) + .context(LabelBuildSnafu)? + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }, + ]; + Ok(services) } /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. @@ -1100,9 +1146,8 @@ fn build_metastore_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(build_headless_role_group_metrics_service_name( - rolegroup_ref.object_name(), - )), + // TODO: Use method on RoleGroupRef once op-rs is released + service_name: Some(hive.rolegroup_headless_metrics_service_name(rolegroup_ref)), template: pod_template, volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() @@ -1123,15 +1168,6 @@ pub fn error_policy( } } -pub fn service_ports() -> Vec { - vec![ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] -} - /// Creates recommended `ObjectLabels` to be used in deployed resources pub fn build_recommended_labels<'a, T>( owner: &'a T, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index de82443f..2badef4c 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -18,7 +18,7 @@ use stackable_operator::{ merge::Merge, }, crd::s3, - k8s_openapi::apimachinery::pkg::api::resource::Quantity, + k8s_openapi::{api::core::v1::ServicePort, apimachinery::pkg::api::resource::Quantity}, kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, product_config_utils::{self, Configuration}, product_logging::{self, spec::Logging}, @@ -246,6 +246,43 @@ impl v1alpha1::HiveCluster { format!("{name}-{role}", name = self.name_any(), role = hive_role) } + /// Set of functions to define service names on rolegroup level. + /// Headless service for cluster internal purposes only. + // TODO: Move to operator-rs + pub fn rolegroup_headless_service_name( + &self, + rolegroup: &RoleGroupRef, + ) -> String { + format!("{name}-headless", name = rolegroup.object_name()) + } + + /// Headless metrics service exposes Prometheus endpoint only + // TODO: Move to operator-rs + pub fn rolegroup_headless_metrics_service_name( + &self, + rolegroup: &RoleGroupRef, + ) -> String { + format!("{name}-metrics", name = rolegroup.object_name()) + } + + pub fn metrics_ports(&self) -> Vec { + vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] + } + + pub fn service_ports(&self) -> Vec { + vec![ServicePort { + name: Some(HIVE_PORT_NAME.to_string()), + port: HIVE_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] + } + pub fn rolegroup( &self, rolegroup_ref: &RoleGroupRef, diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 2e9e16fb..12b9ec45 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -111,7 +111,3 @@ fn build_discovery_configmap( obj_ref: ObjectRef::from_obj(hive), }) } - -pub fn build_headless_role_group_metrics_service_name(name: String) -> String { - format!("{name}-metrics") -} diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 0b24920e..48d7404f 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -104,7 +104,7 @@ pub fn build_role_listener( Ok(listener) } -fn listener_ports() -> Vec { +pub fn listener_ports() -> Vec { vec![ListenerPort { name: HIVE_PORT_NAME.to_owned(), port: HIVE_PORT.into(), From 2add5eeeec2bde6912ccb4d2dbfda51634c0b6de Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 2 Jul 2025 14:44:18 +0200 Subject: [PATCH 02/10] remove unused input --- rust/operator-binary/src/controller.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b3be9146..f4587153 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -44,7 +44,7 @@ use stackable_operator::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ ConfigMap, ConfigMapVolumeSource, EmptyDirVolumeSource, Probe, Service, - ServicePort, ServiceSpec, TCPSocketAction, Volume, + ServiceSpec, TCPSocketAction, Volume, }, }, apimachinery::pkg::{ @@ -99,7 +99,7 @@ use crate::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, }, - listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_role_listener, listener_ports}, + listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_role_listener}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::extend_role_group_config_map, }; From 439e178f3d48fb0c57b15d9025087b9cb5e33c6c Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 2 Jul 2025 15:56:34 +0200 Subject: [PATCH 03/10] update test to also test for headless service --- tests/templates/kuttl/external-access/20-assert.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index 58842f02..7b43ef4e 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -37,3 +37,10 @@ metadata: name: test-hive-metastore-default-metrics spec: type: ClusterIP # exposed metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hive-metastore-default-headless +spec: + type: ClusterIP # exposed metrics From 6f1ba14d7035e5a2482b90268f5911552a99c53b Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 2 Jul 2025 17:03:42 +0200 Subject: [PATCH 04/10] update function name and smoke test --- rust/operator-binary/src/controller.rs | 4 ++-- tests/templates/kuttl/smoke/80-assert.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index f4587153..bfc55010 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -456,7 +456,7 @@ pub async fn reconcile_hive( .merged_config(&HiveRole::MetaStore, &rolegroup) .context(FailedToResolveResourceConfigSnafu)?; - let rg_services = build_rolegroup_service(hive, &resolved_product_image, &rolegroup)?; + let rg_services = build_rolegroup_services(hive, &resolved_product_image, &rolegroup)?; let rg_configmap = build_metastore_rolegroup_config_map( hive, &hive_namespace, @@ -717,7 +717,7 @@ fn build_metastore_rolegroup_config_map( /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -fn build_rolegroup_service( +fn build_rolegroup_services( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, diff --git a/tests/templates/kuttl/smoke/80-assert.yaml b/tests/templates/kuttl/smoke/80-assert.yaml index a6494eb5..e1d8c389 100644 --- a/tests/templates/kuttl/smoke/80-assert.yaml +++ b/tests/templates/kuttl/smoke/80-assert.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore-default-headless.$NAMESPACE.svc.cluster.local From c0f4b580883eb6d3cb6d9cd82f114637cc29b2ee Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 2 Jul 2025 18:59:38 +0200 Subject: [PATCH 05/10] Updating function name everywhere --- rust/operator-binary/src/controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index bfc55010..9b06ea7c 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -804,7 +804,7 @@ fn build_rolegroup_services( /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`Service`] (from [`build_rolegroup_service`]). +/// corresponding [`Service`] (from [`build_rolegroup_services`]). #[allow(clippy::too_many_arguments)] fn build_metastore_rolegroup_statefulset( hive: &v1alpha1::HiveCluster, From 4298a8f4007f4643d5a29a6f3364e6f43301dd34 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 3 Jul 2025 15:55:11 +0200 Subject: [PATCH 06/10] Updating changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7866a8d..a1c89163 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. - BREAKING: Add Listener support for Hive ([#605]). +- Add internal headless service in addition to the metrics service ([#613]). ### Changed @@ -43,6 +44,7 @@ All notable changes to this project will be documented in this file. [#603]: https://github.com/stackabletech/hive-operator/pull/603 [#604]: https://github.com/stackabletech/hive-operator/pull/604 [#605]: https://github.com/stackabletech/hive-operator/pull/605 +[#613]: https://github.com/stackabletech/hive-operator/pull/613 ## [25.3.0] - 2025-03-21 From 9f2a57ffdaf8e312f992ac0534200af58e55296d Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 3 Jul 2025 18:48:50 +0200 Subject: [PATCH 07/10] Moving services to own module, splitting up functions --- rust/operator-binary/src/controller.rs | 133 ++++++----------------- rust/operator-binary/src/crd/mod.rs | 39 +------ rust/operator-binary/src/main.rs | 1 + rust/operator-binary/src/service.rs | 141 +++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 138 deletions(-) create mode 100644 rust/operator-binary/src/service.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 9b06ea7c..6d7b7562 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -43,8 +43,8 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, EmptyDirVolumeSource, Probe, Service, - ServiceSpec, TCPSocketAction, Volume, + ConfigMap, ConfigMapVolumeSource, EmptyDirVolumeSource, Probe, TCPSocketAction, + Volume, }, }, apimachinery::pkg::{ @@ -56,7 +56,7 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Label, Labels, ObjectLabels}, + kvp::{Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, @@ -102,6 +102,10 @@ use crate::{ listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_role_listener}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::extend_role_group_config_map, + service::{ + build_rolegroup_headless_service, build_rolegroup_metrics_service, + rolegroup_metrics_service_name, + }, }; pub const HIVE_CONTROLLER_NAME: &str = "hivecluster"; @@ -345,6 +349,9 @@ pub enum Error { BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, }, + + #[snafu(display("faild to configure service"))] + ServiceConfiguration { source: crate::service::Error }, } type Result = std::result::Result; @@ -456,7 +463,14 @@ pub async fn reconcile_hive( .merged_config(&HiveRole::MetaStore, &rolegroup) .context(FailedToResolveResourceConfigSnafu)?; - let rg_services = build_rolegroup_services(hive, &resolved_product_image, &rolegroup)?; + let rg_metrics_service = + build_rolegroup_metrics_service(hive, &resolved_product_image, &rolegroup) + .context(ServiceConfigurationSnafu)?; + + let rg_headless_service = + build_rolegroup_headless_service(hive, &resolved_product_image, &rolegroup) + .context(ServiceConfigurationSnafu)?; + let rg_configmap = build_metastore_rolegroup_config_map( hive, &hive_namespace, @@ -478,13 +492,19 @@ pub async fn reconcile_hive( &rbac_sa.name_any(), )?; - for rg_service in rg_services { - cluster_resources.add(client, rg_service).await.context( - ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), - }, - )?; - } + cluster_resources + .add(client, rg_metrics_service) + .await + .context(ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + })?; + + cluster_resources + .add(client, rg_headless_service) + .await + .context(ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + })?; cluster_resources .add(client, rg_configmap) @@ -714,97 +734,10 @@ fn build_metastore_rolegroup_config_map( }) } -/// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup -/// -/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -fn build_rolegroup_services( - hive: &v1alpha1::HiveCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, -) -> Result> { - let services = vec![ - Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(hive) - // TODO: Use method on RoleGroupRef once op-rs is released - .name(hive.rolegroup_headless_metrics_service_name(rolegroup)) - .ownerreference_from_resource(hive, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label( - Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?, - ) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(hive.metrics_ports()), - selector: Some( - Labels::role_group_selector( - hive, - APP_NAME, - &rolegroup.role, - &rolegroup.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }, - Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(hive) - // TODO: Use method on RoleGroupRef once op-rs is released - .name(hive.rolegroup_headless_service_name(rolegroup)) - .ownerreference_from_resource(hive, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - // Expecting same ports as on listener service, just as a headless, internal service - ports: Some(hive.service_ports()), - selector: Some( - Labels::role_group_selector( - hive, - APP_NAME, - &rolegroup.role, - &rolegroup.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }, - ]; - Ok(services) -} - /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`Service`] (from [`build_rolegroup_services`]). +/// corresponding [`Service`] (from [`build_rolegroup_headless_service`] and [`build_rolegroup_metrics_service`]). #[allow(clippy::too_many_arguments)] fn build_metastore_rolegroup_statefulset( hive: &v1alpha1::HiveCluster, @@ -1147,7 +1080,7 @@ fn build_metastore_rolegroup_statefulset( ..LabelSelector::default() }, // TODO: Use method on RoleGroupRef once op-rs is released - service_name: Some(hive.rolegroup_headless_metrics_service_name(rolegroup_ref)), + service_name: Some(rolegroup_metrics_service_name(rolegroup_ref)), template: pod_template, volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 2badef4c..de82443f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -18,7 +18,7 @@ use stackable_operator::{ merge::Merge, }, crd::s3, - k8s_openapi::{api::core::v1::ServicePort, apimachinery::pkg::api::resource::Quantity}, + k8s_openapi::apimachinery::pkg::api::resource::Quantity, kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, product_config_utils::{self, Configuration}, product_logging::{self, spec::Logging}, @@ -246,43 +246,6 @@ impl v1alpha1::HiveCluster { format!("{name}-{role}", name = self.name_any(), role = hive_role) } - /// Set of functions to define service names on rolegroup level. - /// Headless service for cluster internal purposes only. - // TODO: Move to operator-rs - pub fn rolegroup_headless_service_name( - &self, - rolegroup: &RoleGroupRef, - ) -> String { - format!("{name}-headless", name = rolegroup.object_name()) - } - - /// Headless metrics service exposes Prometheus endpoint only - // TODO: Move to operator-rs - pub fn rolegroup_headless_metrics_service_name( - &self, - rolegroup: &RoleGroupRef, - ) -> String { - format!("{name}-metrics", name = rolegroup.object_name()) - } - - pub fn metrics_ports(&self) -> Vec { - vec![ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] - } - - pub fn service_ports(&self) -> Vec { - vec![ServicePort { - name: Some(HIVE_PORT_NAME.to_string()), - port: HIVE_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] - } - pub fn rolegroup( &self, rolegroup_ref: &RoleGroupRef, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 22d6376a..cc96642f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -7,6 +7,7 @@ mod kerberos; mod listener; mod operations; mod product_logging; +mod service; use std::sync::Arc; diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs new file mode 100644 index 00000000..03283460 --- /dev/null +++ b/rust/operator-binary/src/service.rs @@ -0,0 +1,141 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, + kvp::{Label, Labels}, + role_utils::RoleGroupRef, +}; + +use crate::{ + controller::build_recommended_labels, + crd::{APP_NAME, HIVE_PORT, HIVE_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}, +}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, +} + +/// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup +/// +/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. +pub fn build_rolegroup_headless_service( + hive: &v1alpha1::HiveCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, +) -> Result { + let headless_service = Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(hive) + // TODO: Use method on RoleGroupRef once op-rs is released + .name(rolegroup_headless_service_name(rolegroup)) + .ownerreference_from_resource(hive, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + // Expecting same ports as on listener service, just as a headless, internal service + ports: Some(service_ports()), + selector: Some( + Labels::role_group_selector(hive, APP_NAME, &rolegroup.role, &rolegroup.role_group) + .context(LabelBuildSnafu)? + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }; + Ok(headless_service) +} + +/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label +pub fn build_rolegroup_metrics_service( + hive: &v1alpha1::HiveCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, +) -> Result { + let metrics_service = Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(hive) + // TODO: Use method on RoleGroupRef once op-rs is released + .name(rolegroup_metrics_service_name(rolegroup)) + .ownerreference_from_resource(hive, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(metrics_ports()), + selector: Some( + Labels::role_group_selector(hive, APP_NAME, &rolegroup.role, &rolegroup.role_group) + .context(LabelBuildSnafu)? + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }; + Ok(metrics_service) +} + +/// Headless service for cluster internal purposes only. +// TODO: Move to operator-rs +pub fn rolegroup_headless_service_name(rolegroup: &RoleGroupRef) -> String { + format!("{name}-headless", name = rolegroup.object_name()) +} + +/// Headless metrics service exposes Prometheus endpoint only +// TODO: Move to operator-rs +pub fn rolegroup_metrics_service_name(rolegroup: &RoleGroupRef) -> String { + format!("{name}-metrics", name = rolegroup.object_name()) +} + +fn metrics_ports() -> Vec { + vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} + +fn service_ports() -> Vec { + vec![ServicePort { + name: Some(HIVE_PORT_NAME.to_string()), + port: HIVE_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} From 7f34283df7df34c0919bfbadf7501fd7a5efc4dc Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 3 Jul 2025 18:58:33 +0200 Subject: [PATCH 08/10] Fixing rust docs --- rust/operator-binary/src/controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 6d7b7562..458095bb 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -737,7 +737,7 @@ fn build_metastore_rolegroup_config_map( /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`Service`] (from [`build_rolegroup_headless_service`] and [`build_rolegroup_metrics_service`]). +/// corresponding [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (from [`build_rolegroup_headless_service`] and [`build_rolegroup_metrics_service`]). #[allow(clippy::too_many_arguments)] fn build_metastore_rolegroup_statefulset( hive: &v1alpha1::HiveCluster, From fa30fe41ae8da4d5dfbd749ec8aacb3822101696 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Fri, 4 Jul 2025 15:35:46 +0200 Subject: [PATCH 09/10] Update according to review Co-authored-by: Malte Sander --- rust/operator-binary/src/controller.rs | 4 ++-- tests/templates/kuttl/smoke/80-assert.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 458095bb..8bc41014 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -737,7 +737,7 @@ fn build_metastore_rolegroup_config_map( /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (from [`build_rolegroup_headless_service`] and [`build_rolegroup_metrics_service`]). +/// corresponding [`Service`](`stackable_operator::k8s_openapi::api::core::v1::Service`) (via [`build_rolegroup_headless_service`] and metrics from [`build_rolegroup_metrics_service`]). #[allow(clippy::too_many_arguments)] fn build_metastore_rolegroup_statefulset( hive: &v1alpha1::HiveCluster, @@ -1080,7 +1080,7 @@ fn build_metastore_rolegroup_statefulset( ..LabelSelector::default() }, // TODO: Use method on RoleGroupRef once op-rs is released - service_name: Some(rolegroup_metrics_service_name(rolegroup_ref)), + service_name: Some(rolegroup_headless_service_name(rolegroup_ref)), template: pod_template, volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() diff --git a/tests/templates/kuttl/smoke/80-assert.yaml b/tests/templates/kuttl/smoke/80-assert.yaml index e1d8c389..3d4bd846 100644 --- a/tests/templates/kuttl/smoke/80-assert.yaml +++ b/tests/templates/kuttl/smoke/80-assert.yaml @@ -2,4 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: + - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore-default-headless.$NAMESPACE.svc.cluster.local From 840cb82bf1bbfb5ce1fc2eb8cb85d79b8daea3b0 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 4 Jul 2025 15:38:02 +0200 Subject: [PATCH 10/10] Fix function import --- rust/operator-binary/src/controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8bc41014..ba6de6ed 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -104,7 +104,7 @@ use crate::{ product_logging::extend_role_group_config_map, service::{ build_rolegroup_headless_service, build_rolegroup_metrics_service, - rolegroup_metrics_service_name, + rolegroup_headless_service_name, }, };