[wormhole-attester] Add a healthcheck on the metrics port (#429)

* wormhole attester: Add a healthcheck on the metrics port

* pyth2wormhole healthcheck: apply review advice

- Move metrics/healthcheck counter updates to be next to each other
- Change "0-sized window disables healthcheck" into an explicit config value
- move healthcheck updates past the atomic counter updates

Author: Stanisław Drozd

devnet/p2w-attest.yaml

@@ -45,11 +45,35 @@ spec:
             - name: P2W_EXIT_ON_ERROR
               value: "true"
           tty: true
-          readinessProbe:
-            tcpSocket:
-              port: 2000
-            periodSeconds: 1
-            failureThreshold: 300
+          # Probes, in order of appearance https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+          #
+          # Startup probe - delays other probes until it gets its first success
+          startupProbe:
+            httpGet:
+              path: /healthcheck
+              port: 3000
+            failureThreshold: 100 # up to 100 * 10 seconds to report initial healthy status
+            periodSeconds: 10
+          # Readiness probe - Used to tell load balancers to
+          # start/stop sending traffic to the container, *without*
+          # restarting it. The attester does not accept any traffic as
+          # part of its workflow, which means this isn't very useful.
+          # readinessProbe:
+          # httpGet:
+          #   path: /healthcheck
+          #   port: 3000
+          # failureThreshold: 1
+          # periodSeconds: 10
+          #
+          # Liveness probe - decides restarts for misbehaving
+          # containers
+          livenessProbe:
+            httpGet:
+              path: /healthcheck
+              port: 3000
+            failureThreshold: 1 # If the attester healthcheck fails once,
+            periodSeconds: 10
+
           ports:
             - containerPort: 4343
               name: p2w-attest

solana/pyth2wormhole/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "pyth2wormhole-client"
-version = "1.2.0"
+version = "1.3.0"
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

solana/pyth2wormhole/client/Cargo.toml

@@ -35,6 +35,14 @@ pub struct AttestationConfig {
     #[serde(default = "default_max_msg_accounts")]
     pub max_msg_accounts:          u64,
 
+    /// How many consecutive attestation failures cause the service to
+    /// report as unhealthy.
+    #[serde(default = "default_healthcheck_window_size")]
+    pub healthcheck_window_size: u64,
+
+    #[serde(default = "default_enable_healthcheck")]
+    pub enable_healthcheck: bool,
+
     /// Optionally, we take a mapping account to add remaining symbols from a Pyth deployments.
     /// These symbols are processed under `default_attestation_conditions`.
     #[serde(
@@ -85,7 +93,7 @@ impl AttestationConfig {
 
                     name_to_symbols
                         .entry(name.clone())
-                        .or_insert(vec![])
+                        .or_default()
                         .push(symbol);
                 }
             }
@@ -289,6 +297,14 @@ pub const fn default_min_msg_reuse_interval_ms() -> u64 {
     10_000 // 10s
 }
 
+pub const fn default_healthcheck_window_size() -> u64 {
+    100
+}
+
+pub const fn default_enable_healthcheck() -> bool {
+    true
+}
+
 pub const fn default_mapping_reload_interval_mins() -> u64 {
     15
 }
@@ -473,6 +489,8 @@ mod tests {
         let cfg = AttestationConfig {
             min_msg_reuse_interval_ms:      1000,
             max_msg_accounts:               100_000,
+            enable_healthcheck:             true,
+            healthcheck_window_size:        100,
             min_rpc_interval_ms:            2123,
             mapping_addr:                   None,
             mapping_reload_interval_mins:   42,
@@ -555,6 +573,8 @@ mod tests {
         let cfg = AttestationConfig {
             min_msg_reuse_interval_ms:      1000,
             max_msg_accounts:               100_000,
+            healthcheck_window_size:        100,
+            enable_healthcheck:             true,
             min_rpc_interval_ms:            2123,
             mapping_addr:                   None,
             mapping_reload_interval_mins:   42,

solana/pyth2wormhole/client/src/attestation_cfg.rs

@@ -0,0 +1,52 @@
+use {
+    std::{
+        collections::VecDeque,
+        sync::Arc,
+    },
+    tokio::sync::Mutex,
+};
+
+lazy_static::lazy_static! {
+    pub static ref HEALTHCHECK_STATE: Arc<Mutex<HealthCheckState>> = Arc::new(Mutex::new(HealthCheckState::new(1, false)));
+}
+
+/// Helper structure for deciding service health
+pub struct HealthCheckState {
+    /// Whether to report the healthy/unhealthy status
+    pub enable:          bool,
+    /// Sliding LIFO window over last `max_window_size` attestation results (true = ok, false = error)
+    pub window:          VecDeque<bool>,
+    /// Window size
+    pub max_window_size: usize,
+}
+
+
+impl HealthCheckState {
+    pub fn new(max_window_size: usize, enable: bool) -> Self {
+        Self {
+            enable,
+            window: VecDeque::with_capacity(max_window_size),
+            max_window_size,
+        }
+    }
+    /// Check service health, return None if not enough data is present
+    pub fn is_healthy(&self) -> Option<bool> {
+        if self.window.len() >= self.max_window_size && self.enable {
+            // If all results are false, return false (unhealthy).
+            Some(self.window.iter().any(|entry| *entry))
+        } else {
+            // The window isn't big enough yet or the healthcheck is disabled
+            None
+        }
+    }
+
+    /// Rotate the window
+    pub fn add_result(&mut self, res: bool) {
+        self.window.push_front(res);
+
+        // Trim window back to size if needed. truncate() deletes from
+        // the back and has no effect if new size is greater than
+        // current size.
+        self.window.truncate(self.max_window_size);
+    }
+}

solana/pyth2wormhole/client/src/healthcheck.rs

@@ -1,5 +1,6 @@
 pub mod attestation_cfg;
 pub mod batch_state;
+pub mod healthcheck;
 pub mod message;
 pub mod util;
 
@@ -10,6 +11,10 @@ pub use {
         P2WSymbol,
     },
     batch_state::BatchState,
+    healthcheck::{
+        HealthCheckState,
+        HEALTHCHECK_STATE,
+    },
     message::P2WMessageQueue,
     pyth2wormhole::Pyth2WormholeConfig,
     util::{