From 406085becf3a5446e7a1e07727deee58cf1eaabb Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 7 Oct 2025 11:54:51 +0200 Subject: [PATCH 1/5] feat!: Make NodePort stickiness configurable --- Cargo.lock | 14 ++++---- Cargo.toml | 2 +- deploy/helm/listener-operator/crds/crds.yaml | 15 ++++++++ .../src/csi_server/controller.rs | 34 +++++++++++-------- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99b0de36..4d417520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "darling", "regex", @@ -2612,7 +2612,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.99.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "chrono", "clap", @@ -2650,7 +2650,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "darling", "proc-macro2", @@ -2661,7 +2661,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "chrono", "k8s-openapi", @@ -2678,7 +2678,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "axum", "clap", @@ -2702,7 +2702,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "schemars", "serde", @@ -2715,7 +2715,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#be422046081be2fb9f37dfece660e007273f4e32" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#16a07e19197f1ba2bb129dae8de724cc149caae0" dependencies = [ "convert_case", "darling", diff --git a/Cargo.toml b/Cargo.toml index 2126ccdb..453fed99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,4 +39,4 @@ walkdir = "2.5.0" [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/listenerclass-stickiness" } diff --git a/deploy/helm/listener-operator/crds/crds.yaml b/deploy/helm/listener-operator/crds/crds.yaml index 641241c3..15cd581d 100644 --- a/deploy/helm/listener-operator/crds/crds.yaml +++ b/deploy/helm/listener-operator/crds/crds.yaml @@ -82,6 +82,21 @@ spec: - LoadBalancer - ClusterIP type: string + stickyNodePorts: + default: false + description: |- + Wether a Pod exposed using a NodePort should be pinned to a specific Kubernetes node. + + By pinning the Pod to a specific (stable) Kubernetes node, stable addresses can be + provided using NodePorts. The stickiness is achieved by listener-operator by setting the + `volume.kubernetes.io/selected-node` annotation on the Listener PVC. + + However, this only works on setups with long-living nodes. If your nodes are rotated on + a regular basis, the Pods previously running on a removed node will be stuck in Pending + until you delete the PVC with the stickiness. + + Because of this we don't enable stickiness by default to support all environments. + type: boolean required: - serviceType type: object diff --git a/rust/operator-binary/src/csi_server/controller.rs b/rust/operator-binary/src/csi_server/controller.rs index 5d8581b7..a8444378 100644 --- a/rust/operator-binary/src/csi_server/controller.rs +++ b/rust/operator-binary/src/csi_server/controller.rs @@ -127,26 +127,32 @@ impl csi::v1::controller_server::Controller for ListenerOperatorController { .within(&ns) .erase(), })?; + + // We only configure a node stickiness in case it is enabled and the Service is of type + // NodePort. + let accessible_topology = if listener_class.spec.sticky_node_ports + && listener_class.spec.service_type == listener::v1alpha1::ServiceType::NodePort + { + // Pick the top node (as selected by the CSI client) and "stick" to that + // Since we want clients to have a stable address to connect to + request + .accessibility_requirements + .unwrap_or_default() + .preferred + .into_iter() + .take(1) + .collect() + } else { + Vec::new() + }; + Ok(Response::new(csi::v1::CreateVolumeResponse { volume: Some(csi::v1::Volume { capacity_bytes: 0, volume_id: request.name, volume_context: raw_volume_context.into_iter().collect(), content_source: None, - accessible_topology: match listener_class.spec.service_type { - // Pick the top node (as selected by the CSI client) and "stick" to that - // Since we want clients to have a stable address to connect to - listener::v1alpha1::ServiceType::NodePort => request - .accessibility_requirements - .unwrap_or_default() - .preferred - .into_iter() - .take(1) - .collect(), - // Load balancers and services of type ClusterIP have no relationship to any particular node, so don't try to be sticky - listener::v1alpha1::ServiceType::LoadBalancer - | listener::v1alpha1::ServiceType::ClusterIP => Vec::new(), - }, + accessible_topology, }), })) } From 5a2d2f2b0c1c826242383b995f25ed3be142b080 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 7 Oct 2025 11:58:57 +0200 Subject: [PATCH 2/5] change helm default --- deploy/helm/listener-operator/templates/listener-classes.yaml | 3 +++ deploy/helm/listener-operator/values.yaml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/helm/listener-operator/templates/listener-classes.yaml b/deploy/helm/listener-operator/templates/listener-classes.yaml index b57a0fae..9746877b 100644 --- a/deploy/helm/listener-operator/templates/listener-classes.yaml +++ b/deploy/helm/listener-operator/templates/listener-classes.yaml @@ -14,6 +14,7 @@ metadata: name: external-unstable spec: serviceType: NodePort + stickyNodePorts: false --- apiVersion: listeners.stackable.tech/v1alpha1 kind: ListenerClass @@ -36,6 +37,7 @@ metadata: name: external-unstable spec: serviceType: NodePort + stickyNodePorts: false --- apiVersion: listeners.stackable.tech/v1alpha1 kind: ListenerClass @@ -51,6 +53,7 @@ spec: # or on-premise environments that don't support external LoadBalancer peering (such as Calico (https://docs.tigera.io/calico/latest/networking/configuring/advertise-service-ips) # or MetalLB (https://metallb.org/)). serviceType: NodePort + stickyNodePorts: true {{ else }} {{ fail "An invalid preset was configured" }} {{ end }} diff --git a/deploy/helm/listener-operator/values.yaml b/deploy/helm/listener-operator/values.yaml index 940ede0b..be297d88 100644 --- a/deploy/helm/listener-operator/values.yaml +++ b/deploy/helm/listener-operator/values.yaml @@ -134,11 +134,11 @@ labels: # Kubelet dir may vary in environments such as microk8s, see https://github.com/stackabletech/secret-operator/issues/229 kubeletDir: /var/lib/kubelet -# Options: none, stable-nodes, ephemeral-nodes +# Options: none, stable-nodes, ephemeral-nodes (default) # none: No ListenerClasses are preinstalled, administrators must supply them themselves # stable-nodes: ListenerClasses are preinstalled that are suitable for on-prem/"pet" environments, assuming long-running Nodes but not requiring a LoadBalancer controller # ephemeral-nodes: ListenerClasses are preinstalled that are suitable for cloud/"cattle" environments with short-lived nodes, however this requires a LoadBalancer controller to be installed -preset: stable-nodes +preset: ephemeral-nodes # See all available options and detailed explanations about the concept here: # https://docs.stackable.tech/home/stable/concepts/telemetry/ From 425e1026dc74e8eeff700c6511eea4ce1291aee6 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 7 Oct 2025 12:25:27 +0200 Subject: [PATCH 3/5] docs --- .../pages/listenerclass.adoc | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/modules/listener-operator/pages/listenerclass.adoc b/docs/modules/listener-operator/pages/listenerclass.adoc index c76a8742..662f5518 100644 --- a/docs/modules/listener-operator/pages/listenerclass.adoc +++ b/docs/modules/listener-operator/pages/listenerclass.adoc @@ -48,7 +48,7 @@ The Stackable Data Platform expects these three ListenerClasses to exist: == Presets To help users get started, the Stackable Listener Operator ships different ListenerClass _presets_ for different environments. -These are configured using the `preset` Helm value, with `stable-nodes` being the default. +These are configured using the `preset` Helm value, with `ephemeral-nodes` being the default. === Installation Commands @@ -83,21 +83,25 @@ Both `stable-nodes` and `ephemeral-nodes` create the same three ListenerClasses |ClusterIP |`external-unstable` -|NodePort -|NodePort +|NodePort (non-sticky) +|NodePort (non-sticky) |`external-stable` -|NodePort +|NodePort (sticky) |LoadBalancer |=== +A sticky NodePort pins the Pod to a particular Kubernetes node, so that the endpoint is stable across Pod restarts. +This is achieved via the `volume.kubernetes.io/selected-node` annotation on the Listener PVC. + ==== Why the Difference? -* **stable-nodes**: Uses NodePort for external access and pins pods to specific nodes for address stability. +* **stable-nodes**: Uses NodePort for external access and pins external-stable pods to specific nodes for address stability. + [CAUTION] ==== -This creates a dependency on specific nodes. If a pinned node becomes unavailable, the pod cannot start on other nodes until you either restore the node or manually delete the PVC to allow rescheduling. +This creates a dependency on specific nodes when external-stable is used. +If a pinned node becomes unavailable, the pod cannot start on other nodes until you either restore the node or manually delete the PVC to allow rescheduling. ==== + .To recover from node failures: @@ -131,7 +135,7 @@ The key is understanding your environment's requirements. ==== NodePort * **Use for**: External access (from outside the Kubernetes cluster) in environments with stable nodes * **Access**: From outside the cluster via `:` -* **Behavior**: Pins pods to specific nodes for address stability +* **Behavior**: You can configure if Pods should be pinned to specific nodes for address stability [WARNING] ==== @@ -139,15 +143,16 @@ NodePort services may expose your applications to the internet if your Kubernete Ensure you understand your cluster's network topology and have appropriate firewall rules in place. ==== +===== Node stickiness + +Using `.spec.stickyNodePorts` (defaults to `false`) you can enable that Pods are xref:volume.adoc#pinning[pinned] to a specific Kubernetes node. + [CAUTION] ==== -When using NodePort with pinned pods, service addresses depend on specific nodes. If a pinned node becomes unavailable, the service may become unreachable until the pod can be rescheduled to a new node, potentially changing the service address. +When using NodePort with pinned pods, service addresses depend on specific nodes. +If a pinned node becomes unavailable, the service may become unreachable until the pod can be rescheduled to a new node, potentially changing the service address. ==== -Pods bound to `NodePort` listeners will be xref:volume.adoc#pinning[pinned] to a specific Node for address stability. -If this behavior is undesirable, consider using xref:#servicetype-loadbalancer[] instead. - - [#servicetype-loadbalancer] ==== LoadBalancer * **Use for**: External access in environments without stable nodes or other reasons for a LoadBalancer From 079383200a4003fb7c49327726ae845d95368ec2 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 7 Oct 2025 12:31:13 +0200 Subject: [PATCH 4/5] nix --- Cargo.nix | 42 +++++++++++++++++++++--------------------- crate-hashes.json | 14 +++++++------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index d0ef59d3..9c4222a0 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4364,9 +4364,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; libName = "k8s_version"; authors = [ @@ -8637,9 +8637,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; libName = "stackable_operator"; authors = [ @@ -8806,9 +8806,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8841,9 +8841,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; libName = "stackable_shared"; authors = [ @@ -8923,9 +8923,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; libName = "stackable_telemetry"; authors = [ @@ -9033,9 +9033,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; libName = "stackable_versioned"; authors = [ @@ -9077,9 +9077,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "be422046081be2fb9f37dfece660e007273f4e32"; - sha256 = "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "16a07e19197f1ba2bb129dae8de724cc149caae0"; + sha256 = "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/crate-hashes.json b/crate-hashes.json index 676bb1d9..488b806e 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,10 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#k8s-version@0.1.3": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#stackable-operator-derive@0.3.1": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#stackable-operator@0.99.0": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#stackable-shared@0.0.3": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#stackable-telemetry@0.6.1": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#stackable-versioned-macros@0.8.2": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.99.0#stackable-versioned@0.8.2": "14mp7m9x1d6s1xxhdh5rqvkxdksmz96km1hfn0yshdfw6ic5m8cv", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#k8s-version@0.1.3": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#stackable-operator-derive@0.3.1": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#stackable-operator@0.99.0": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#stackable-shared@0.0.3": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#stackable-telemetry@0.6.1": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#stackable-versioned-macros@0.8.2": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Flistenerclass-stickiness#stackable-versioned@0.8.2": "0p04mcfh9gj8cb8gb6gw5b3zz8nwfq89hx74zrn803ai1vmwhc0c", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 74506d39468ad3e1db939ee8e530ea3053aff20d Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 8 Oct 2025 08:39:59 +0200 Subject: [PATCH 5/5] Improve comment --- rust/operator-binary/src/csi_server/controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/csi_server/controller.rs b/rust/operator-binary/src/csi_server/controller.rs index a8444378..3ab2acd5 100644 --- a/rust/operator-binary/src/csi_server/controller.rs +++ b/rust/operator-binary/src/csi_server/controller.rs @@ -129,7 +129,8 @@ impl csi::v1::controller_server::Controller for ListenerOperatorController { })?; // We only configure a node stickiness in case it is enabled and the Service is of type - // NodePort. + // NodePort. Load balancers and services of type ClusterIP have no relationship to any + // particular node, so don't try to be sticky. let accessible_topology = if listener_class.spec.sticky_node_ports && listener_class.spec.service_type == listener::v1alpha1::ServiceType::NodePort {