Handle SIGHUPs as part of the shutdown manager

Author: sandhose

Cargo.lock

@@ -23,6 +23,7 @@ console = "0.15.10"
 dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
 dotenvy = "0.15.7"
 figment.workspace = true
+futures-util.workspace = true
 http-body-util.workspace = true
 hyper.workspace = true
 ipnetwork = "0.20.0"

crates/cli/Cargo.toml

@@ -27,8 +27,7 @@ use crate::{
     shutdown::ShutdownManager,
     util::{
         database_pool_from_config, mailer_from_config, password_manager_from_config,
-        policy_factory_from_config, register_sighup, site_config_from_config,
-        templates_from_config,
+        policy_factory_from_config, site_config_from_config, templates_from_config,
     },
 };
 
@@ -57,7 +56,7 @@ impl Options {
     #[allow(clippy::too_many_lines)]
     pub async fn run(self, figment: &Figment) -> anyhow::Result<ExitCode> {
         let span = info_span!("cli.run.init").entered();
-        let shutdown = ShutdownManager::new()?;
+        let mut shutdown = ShutdownManager::new()?;
         let config = AppConfig::extract(figment)?;
 
         info!(version = crate::VERSION, "Starting up");
@@ -145,6 +144,7 @@ impl Options {
         // Load and compile the templates
         let templates =
             templates_from_config(&config.templates, &site_config, &url_builder).await?;
+        shutdown.register_reloadable(&templates);
 
         let http_client = mas_http::reqwest_client();
 
@@ -186,6 +186,9 @@ impl Options {
             shutdown.task_tracker(),
             shutdown.soft_shutdown_token(),
         );
+
+        shutdown.register_reloadable(&activity_tracker);
+
         let trusted_proxies = config.http.trusted_proxies.clone();
 
         // Build a rate limiter.
@@ -197,9 +200,6 @@ impl Options {
         // Explicitly the config to properly zeroize secret keys
         drop(config);
 
-        // Listen for SIGHUP
-        register_sighup(&templates, &activity_tracker)?;
-
         limiter.start();
 
         let graphql_schema = mas_handlers::graphql_schema(

crates/cli/src/commands/server.rs

@@ -3,8 +3,11 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 // Please see LICENSE in the repository root for full details.
 
-use std::{process::ExitCode, time::Duration};
+use std::{future::Future, pin::Pin, process::ExitCode, time::Duration};
 
+use futures_util::future::BoxFuture;
+use mas_handlers::ActivityTracker;
+use mas_templates::Templates;
 use tokio::signal::unix::{Signal, SignalKind};
 use tokio_util::{sync::CancellationToken, task::TaskTracker};
 
@@ -28,7 +31,30 @@ pub struct ShutdownManager {
     task_tracker: TaskTracker,
     sigterm: Signal,
     sigint: Signal,
+    sighup: Signal,
     timeout: Duration,
+    reload_handlers: Vec<Box<dyn Fn() -> BoxFuture<'static, ()>>>,
+}
+
+pub trait Reloadable: Clone + Send {
+    fn reload(&self) -> impl Future<Output = ()> + Send;
+}
+
+impl Reloadable for ActivityTracker {
+    async fn reload(&self) {
+        self.flush().await;
+    }
+}
+
+impl Reloadable for Templates {
+    async fn reload(&self) {
+        if let Err(err) = self.reload().await {
+            tracing::error!(
+                error = &err as &dyn std::error::Error,
+                "Failed to reload templates"
+            );
+        }
+    }
 }
 
 impl ShutdownManager {
@@ -42,6 +68,7 @@ impl ShutdownManager {
         let soft_shutdown_token = hard_shutdown_token.child_token();
         let sigterm = tokio::signal::unix::signal(SignalKind::terminate())?;
         let sigint = tokio::signal::unix::signal(SignalKind::interrupt())?;
+        let sighup = tokio::signal::unix::signal(SignalKind::hangup())?;
         let timeout = Duration::from_secs(60);
         let task_tracker = TaskTracker::new();
 
@@ -51,10 +78,21 @@ impl ShutdownManager {
             task_tracker,
             sigterm,
             sigint,
+            sighup,
             timeout,
+            reload_handlers: Vec::new(),
         })
     }
 
+    /// Add a handler to be called when the server gets a SIGHUP
+    pub fn register_reloadable(&mut self, reloadable: &(impl Reloadable + 'static)) {
+        let reloadable = reloadable.clone();
+        self.reload_handlers.push(Box::new(move || {
+            let reloadable = reloadable.clone();
+            Box::pin(async move { reloadable.reload().await })
+        }));
+    }
+
     /// Get a reference to the task tracker
     #[must_use]
     pub fn task_tracker(&self) -> &TaskTracker {
@@ -76,21 +114,36 @@ impl ShutdownManager {
     /// Run until we finish completely shutting down.
     pub async fn run(mut self) -> ExitCode {
         // Wait for a first signal and trigger the soft shutdown
-        let likely_crashed = tokio::select! {
-            () = self.soft_shutdown_token.cancelled() => {
-                tracing::warn!("Another task triggered a shutdown, it likely crashed! Shutting down");
-                true
-            },
-
-            _ = self.sigterm.recv() => {
-                tracing::info!("Shutdown signal received (SIGTERM), shutting down");
-                false
-            },
-
-            _ = self.sigint.recv() => {
-                tracing::info!("Shutdown signal received (SIGINT), shutting down");
-                false
-            },
+        let likely_crashed = loop {
+            tokio::select! {
+                () = self.soft_shutdown_token.cancelled() => {
+                    tracing::warn!("Another task triggered a shutdown, it likely crashed! Shutting down");
+                    break true;
+                },
+
+                _ = self.sigterm.recv() => {
+                    tracing::info!("Shutdown signal received (SIGTERM), shutting down");
+                    break false;
+                },
+
+                _ = self.sigint.recv() => {
+                    tracing::info!("Shutdown signal received (SIGINT), shutting down");
+                    break false;
+                },
+
+                _ = self.sighup.recv() => {
+                    tracing::info!("Reload signal received (SIGHUP), reloading");
+
+                    // XXX: if the handler takes a long time, it will block the
+                    // rest of the shutdown process, which is not ideal. We
+                    // should probably have a timeout here
+                    for handler in &self.reload_handlers {
+                        handler().await;
+                    }
+
+                    tracing::info!("Reloading done");
+                },
+            }
         };
 
         self.soft_shutdown_token.cancel();

crates/cli/src/shutdown.rs

@@ -14,15 +14,15 @@ use mas_config::{
 };
 use mas_data_model::SiteConfig;
 use mas_email::{MailTransport, Mailer};
-use mas_handlers::{passwords::PasswordManager, ActivityTracker};
+use mas_handlers::passwords::PasswordManager;
 use mas_policy::PolicyFactory;
 use mas_router::UrlBuilder;
 use mas_templates::{SiteConfigExt, TemplateLoadingError, Templates};
 use sqlx::{
     postgres::{PgConnectOptions, PgPoolOptions},
     ConnectOptions, PgConnection, PgPool,
 };
-use tracing::{error, info, log::LevelFilter};
+use tracing::log::LevelFilter;
 
 pub async fn password_manager_from_config(
     config: &PasswordsConfig,
@@ -313,37 +313,6 @@ pub async fn database_connection_from_config(
         .context("could not connect to the database")
 }
 
-/// Reload templates on SIGHUP
-pub fn register_sighup(
-    templates: &Templates,
-    activity_tracker: &ActivityTracker,
-) -> anyhow::Result<()> {
-    #[cfg(unix)]
-    {
-        let mut signal = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup())?;
-        let templates = templates.clone();
-        let activity_tracker = activity_tracker.clone();
-
-        tokio::spawn(async move {
-            loop {
-                if signal.recv().await.is_none() {
-                    // No more signals will be received, breaking
-                    break;
-                };
-
-                info!("SIGHUP received, reloading templates & flushing activity tracker");
-
-                activity_tracker.flush().await;
-                templates.clone().reload().await.unwrap_or_else(|err| {
-                    error!(?err, "Error while reloading templates");
-                });
-            }
-        });
-    }
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use rand::SeedableRng;