feat(regionserver): add graceful shutdown configuration

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Reduce CRD size from `1.4MB` to `96KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#548]):
   - `podOverrides`
   - `affinity`
+- Optionally move regions to other pods before shutting down a region server ([#570]).
 
 ### Fixed
 
@@ -21,6 +22,7 @@
 [#550]: https://github.com/stackabletech/hbase-operator/pull/550
 [#556]: https://github.com/stackabletech/hbase-operator/pull/556
 [#558]: https://github.com/stackabletech/hbase-operator/pull/558
+[#570]: https://github.com/stackabletech/hbase-operator/pull/570
 
 ## [24.7.0] - 2024-07-24
 

Cargo.toml

@@ -20,6 +20,7 @@ rstest = "0.22"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
+shell-escape = "0.1"
 snafu = "0.8"
 stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.76.0" }
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }

deploy/helm/hbase-operator/crds/crds.yaml

@@ -724,6 +724,32 @@ spec:
                               nullable: true
                               type: boolean
                           type: object
+                        regionMover:
+                          description: Before terminating a region server pod, the RegionMover tool can be invoked to transfer local regions to other servers. This may cause a lot of network traffic in the Kubernetes cluster if the entire HBase stacklet is being restarted. The operator will compute a timeout period for the region move that will not exceed the graceful shutdown timeout.
+                          nullable: true
+                          properties:
+                            ack:
+                              description: If enabled (default), the region mover will confirm that regions are available on the source as well as the target pods before and after the move.
+                              type: boolean
+                            extraOpts:
+                              default: []
+                              description: Additional options to pass to the region mover.
+                              items:
+                                type: string
+                              type: array
+                            maxThreads:
+                              description: Maximum number of threads to use for moving regions.
+                              format: uint16
+                              minimum: 0.0
+                              type: integer
+                            runBeforeShutdown:
+                              description: Move local regions to other servers before terminating a region server's pod.
+                              type: boolean
+                          required:
+                            - ack
+                            - maxThreads
+                            - runBeforeShutdown
+                          type: object
                         resources:
                           default:
                             cpu:
@@ -947,6 +973,32 @@ spec:
                                     nullable: true
                                     type: boolean
                                 type: object
+                              regionMover:
+                                description: Before terminating a region server pod, the RegionMover tool can be invoked to transfer local regions to other servers. This may cause a lot of network traffic in the Kubernetes cluster if the entire HBase stacklet is being restarted. The operator will compute a timeout period for the region move that will not exceed the graceful shutdown timeout.
+                                nullable: true
+                                properties:
+                                  ack:
+                                    description: If enabled (default), the region mover will confirm that regions are available on the source as well as the target pods before and after the move.
+                                    type: boolean
+                                  extraOpts:
+                                    default: []
+                                    description: Additional options to pass to the region mover.
+                                    items:
+                                      type: string
+                                    type: array
+                                  maxThreads:
+                                    description: Maximum number of threads to use for moving regions.
+                                    format: uint16
+                                    minimum: 0.0
+                                    type: integer
+                                  runBeforeShutdown:
+                                    description: Move local regions to other servers before terminating a region server's pod.
+                                    type: boolean
+                                required:
+                                  - ack
+                                  - maxThreads
+                                  - runBeforeShutdown
+                                type: object
                               resources:
                                 default:
                                   cpu:

docs/modules/hbase/pages/usage-guide/operations/graceful-shutdown.adoc

@@ -1,6 +1,6 @@
 = Graceful shutdown
 
-You can configure the graceful shutdown as described in xref:concepts:operations/graceful_shutdown.adoc[].
+You can configure the graceful shutdown grace period as described in xref:concepts:operations/graceful_shutdown.adoc[].
 
 == Masters
 
@@ -15,7 +15,7 @@
 
 == RegionServers
 
-As a default, RegionServers have `60 minutes` to shut down gracefully.
+By default, RegionServers have `60 minutes` to shut down gracefully.
 
 They use the same mechanism described above.
 In contrast to the Master servers, they will, however, acknowledge the graceful shutdown with a message in the logs:
@@ -26,6 +26,30 @@
 2023-10-11 12:38:05,060 INFO  [shutdown-hook-0] regionserver.HRegionServer: ***** STOPPING region server 'test-hbase-regionserver-default-0.test-hbase-regionserver-default.kuttl-test-topical-parakeet.svc.cluster.local,16020,1697027870348' *****
 ----
 
+The operator allows for finer control over the shutdown process of region servers.
+For each region server pod, the region mover tool may be invoked before terminating the region server's pod.
+The affected regions are transferred to other pods thus ensuring that the data is not lost.
+
+Here is a an example:
+
+[source,yaml]
+----
+spec:
+  regionServers:
+    config:
+      regionMover
+        runBeforeShutdown: true, <1>
+        maxThreads: 5, <2>
+        ack: false, <3>
+        extraOpts: ["--designatedFile", "/path/to/designatedFile"] <4>
+----
+<1>: Run the region mover tool before shutting down the region server. Default is `false`.
+<2>: Maximum number of threads to use for moving regions. Default is 1.
+<3>: Enable or disable region confirmation on the present and target servers. Default is `true`.
+<4>: Extra options to pass to the region mover tool.
+
+NOTE: There is no need to explicitly specify a timeout for the region movement. The operator will compute an appropriate timeout that cannot exceed the `gracefulShutdownTimeout` for region servers.
+
 == RestServers
 
 As a default, RestServers have `5 minutes` to shut down gracefully.

rust/crd/Cargo.toml

@@ -12,6 +12,7 @@ publish = false
 product-config.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+shell-escape.workspace = true
 snafu.workspace = true
 stackable-operator.workspace = true
 strum.workspace = true

rust/crd/src/affinity.rs

@@ -123,13 +123,15 @@ mod tests {
                 replicas: 1
         "#;
         let hbase: HbaseCluster = serde_yaml::from_str(input).expect("illegal test input");
-        let merged_config = hbase
+        let affinity = hbase
             .merged_config(
                 &role,
                 "default",
                 &hbase.spec.cluster_config.hdfs_config_map_name,
             )
-            .unwrap();
+            .unwrap()
+            .affinity()
+            .clone();
 
         let mut expected_affinities = vec![WeightedPodAffinityTerm {
             pod_affinity_term: PodAffinityTerm {
@@ -184,7 +186,7 @@ mod tests {
         };
 
         assert_eq!(
-            merged_config.affinity,
+            affinity,
             StackableAffinity {
                 pod_affinity: Some(PodAffinity {
                     preferred_during_scheduling_ignored_during_execution: Some(expected_affinities),