diff --git a/crates/cli/src/commands/server.rs b/crates/cli/src/commands/server.rs
index 565c6bfda..ae1ceefed 100644
--- a/crates/cli/src/commands/server.rs
+++ b/crates/cli/src/commands/server.rs
@@ -28,6 +28,7 @@ use crate::{
     util::{
         database_pool_from_config, mailer_from_config, password_manager_from_config,
         policy_factory_from_config, site_config_from_config, templates_from_config,
+        test_mailer_in_background,
     },
 };
 
@@ -157,7 +158,7 @@ impl Options {
 
         if !self.no_worker {
             let mailer = mailer_from_config(&config.email, &templates)?;
-            mailer.test_connection().await?;
+            test_mailer_in_background(&mailer, Duration::from_secs(30));
 
             info!("Starting task worker");
             mas_tasks::init(
diff --git a/crates/cli/src/commands/worker.rs b/crates/cli/src/commands/worker.rs
index 33cf52b5b..8d2b4cd33 100644
--- a/crates/cli/src/commands/worker.rs
+++ b/crates/cli/src/commands/worker.rs
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 // Please see LICENSE in the repository root for full details.
 
-use std::process::ExitCode;
+use std::{process::ExitCode, time::Duration};
 
 use clap::Parser;
 use figment::Figment;
@@ -17,7 +17,7 @@ use crate::{
     lifecycle::LifecycleManager,
     util::{
         database_pool_from_config, mailer_from_config, site_config_from_config,
-        templates_from_config,
+        templates_from_config, test_mailer_in_background,
     },
 };
 
@@ -55,7 +55,7 @@ impl Options {
             templates_from_config(&config.templates, &site_config, &url_builder).await?;
 
         let mailer = mailer_from_config(&config.email, &templates)?;
-        mailer.test_connection().await?;
+        test_mailer_in_background(&mailer, Duration::from_secs(30));
 
         let http_client = mas_http::reqwest_client();
         let conn = SynapseConnection::new(
diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs
index b117d0177..02a03b0dc 100644
--- a/crates/cli/src/util.rs
+++ b/crates/cli/src/util.rs
@@ -22,7 +22,7 @@ use sqlx::{
     postgres::{PgConnectOptions, PgPoolOptions},
     ConnectOptions, PgConnection, PgPool,
 };
-use tracing::log::LevelFilter;
+use tracing::{log::LevelFilter, Instrument};
 
 pub async fn password_manager_from_config(
     config: &PasswordsConfig,
@@ -99,6 +99,27 @@ pub fn mailer_from_config(
     Ok(Mailer::new(templates.clone(), transport, from, reply_to))
 }
 
+/// Test the connection to the mailer in a background task
+pub fn test_mailer_in_background(mailer: &Mailer, timeout: Duration) {
+    let mailer = mailer.clone();
+
+    let span = tracing::info_span!("cli.test_mailer");
+    tokio::spawn(async move {
+        match tokio::time::timeout(timeout, mailer.test_connection()).await {
+            Ok(Ok(())) => {}
+            Ok(Err(err)) => {
+                tracing::warn!(
+                    error = &err as &dyn std::error::Error,
+                    "Could not connect to the mail backend, tasks sending mails may fail!"
+                );
+            }
+            Err(_) => {
+                tracing::warn!("Timed out while testing the mail backend connection, tasks sending mails may fail!");
+            }
+        }
+    }.instrument(span));
+}
+
 pub async fn policy_factory_from_config(
     config: &PolicyConfig,
     matrix_config: &MatrixConfig,