fix: retry database connection

Author: Randoooom

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "feedback-fusion"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
 license = "MIT"
 
@@ -46,6 +46,7 @@ tonic-reflection = "0.11.0"
 tonic-web = "0.11.0"
 tokio = { version = "1.37.0", features = ["full"] }
 tower = "0.4.13"
+tokio-retry = "0.3"
 tower-http = { version = "=0.4.4", features = ["trace", "validate-request"] }
 tracing = "0.1.39"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

Cargo.toml

@@ -48,8 +48,8 @@ spec:
             {{- end }}
           livenessProbe:
             {{- toYaml .Values.livenessProbe | nindent 12 }}
-          readinessProbe:
-            {{- toYaml .Values.readinessProbe | nindent 12 }}
+          startupProbe:
+            {{- toYaml .Values.startupProbe | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

charts/feedback-fusion/templates/deployment.yaml

@@ -65,9 +65,12 @@ resources: {}
 livenessProbe:
   grpc:
     port: 8000
-readinessProbe:
+  periodSeconds: 5
+startupProbe:
   grpc:
     port: 8000
+  periodSeconds: 2
+  failureThreshold: 10
 
 # Additional volumes on the output Deployment definition.
 volumes: []

charts/feedback-fusion/values.yaml

@@ -23,6 +23,10 @@
 
 use crate::{database::migration::Migration, prelude::*};
 use rbatis::RBatis;
+use tokio_retry::{
+    strategy::{jitter, FibonacciBackoff},
+    Retry,
+};
 
 pub mod migration;
 pub mod schema;
@@ -105,7 +109,23 @@ macro_rules! database_configuration {
                             #[cfg(feature = $scheme)]
                             Self::$ident(config) => {
                                 let url = config.to_url($scheme);
-                                connection.init($driver {}, url.as_str())?;
+
+                                let retry_strategy = FibonacciBackoff::from_millis(500)
+                                    .map(jitter)
+                                    .take(5);
+
+                                Retry::spawn(retry_strategy.clone(), || async {
+                                    match connection.init($driver {}, url.as_str()) {
+                                        Ok(result) => Ok(result),
+                                        Err(error) => {
+                                            error!("Failed to establish DatabaseConnection, retrying in {}s", retry_strategy.clone().next().unwrap().as_secs());
+                                            error!("{}", error);
+
+                                            Err(error)
+                                        }
+                                    }
+                                }).await?;
+
                                 connection.intercepts.clear();
                                 connection.intercepts.push(std::sync::Arc::new(rbatis::plugin::intercept_log::LogInterceptor::new(log::LevelFilter::Debug)));