fix(cat-gateway): Improve panic logging (#3471)

* Use tracing::error instead of println for logging panic

* Custom middleware for catching panics

* Fix Clippy lints

* Add missing documentation

* More documentation:

* added `/panic` endpoint as exception for `DatabaseConnectionImpl`

* Open ports for the gateway container

* Fix

* Use cargo profile option for stripping the backend binary instead of the strip tool

* fix spellcheck

* Restore args

* Remove stripping from Cargo config

* Update the rust-ci version

---------

Co-authored-by: Mr-Leshiy <[email protected]>

Author: stanislav-tkach

catalyst-gateway/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.5.20 AS rust-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.5.21 AS rust-ci
 IMPORT ../ AS repo-ci
 IMPORT github.com/input-output-hk/catalyst-voices/catalyst-gateway:main AS cat-gateway-main
 

catalyst-gateway/bin/src/service/poem_service.rs

@@ -5,7 +5,7 @@
 use cardano_chain_follower::Network;
 use poem::{
     listener::TcpListener,
-    middleware::{CatchPanic, Compression, Cors, SensitiveHeader},
+    middleware::{Compression, Cors, SensitiveHeader},
     web::CompressionLevel,
     Endpoint, EndpointExt, Route,
 };
@@ -19,8 +19,12 @@ use crate::{
         api::mk_api,
         docs::{docs, favicon},
         utilities::{
-            catch_panic::{panic_endpoint, set_panic_hook, ServicePanicHandler},
-            middleware::{metrics_updater::MetricsUpdaterMiddleware, tracing_mw::Tracing},
+            middleware::{
+                catch_panic::{set_panic_hook, CatchPanicMiddleware},
+                metrics_updater::MetricsUpdaterMiddleware,
+                tracing_mw::Tracing,
+            },
+            panic::panic_endpoint,
         },
     },
     settings::Settings,
@@ -48,7 +52,7 @@ fn mk_app(base_route: Option<Route>) -> impl Endpoint {
         .nest("/favicon.ico", favicon())
         .with(Cors::new())
         .with(Compression::new().with_quality(CompressionLevel::Fastest))
-        .with(CatchPanic::new().with_handler(ServicePanicHandler))
+        .with(CatchPanicMiddleware::new())
         .with(Tracing)
         .with(DatabaseConnectionCheck)
         .with(CatGatewayInfo)

catalyst-gateway/bin/src/service/utilities/catch_panic.rs

@@ -0,0 +1,151 @@
+//! A middleware to catch and log panics.
+
+use std::{any::Any, backtrace::Backtrace, cell::RefCell, panic::AssertUnwindSafe};
+
+use futures::FutureExt;
+use panic_message::panic_message;
+use poem::{
+    http::{HeaderMap, Method, StatusCode, Uri},
+    Endpoint, IntoResponse, Middleware, Request, Response,
+};
+use poem_openapi::payload::Json;
+use tracing::{debug, error};
+
+use crate::{
+    service::{
+        common::responses::code_500_internal_server_error::InternalServerError,
+        utilities::health::{get_live_counter, inc_live_counter, set_not_live},
+    },
+    settings::Settings,
+};
+
+// Allows us to catch the backtrace so we can include it in logs.
+thread_local! {
+    static BACKTRACE: RefCell<Option<String>> = const { RefCell::new(None) };
+    static LOCATION: RefCell<Option<String>> = const { RefCell::new(None) };
+}
+
+/// Sets a custom panic hook to capture the Backtrace and Panic Location for logging
+/// purposes. This hook gets called BEFORE we catch it.  So the thread local variables
+/// stored here are valid when processing the panic capture.
+pub(crate) fn set_panic_hook() {
+    std::panic::set_hook(Box::new(|panic_info| {
+        // Get the backtrace and format it.
+        let raw_trace = Backtrace::force_capture();
+        let trace = format!("{raw_trace}");
+        BACKTRACE.with(move |b| b.borrow_mut().replace(trace));
+
+        // Get the location and format it.
+        let location = match panic_info.location() {
+            Some(location) => format!("{location}"),
+            None => "Unknown".to_string(),
+        };
+        LOCATION.with(move |l| l.borrow_mut().replace(location));
+    }));
+}
+
+/// A middleware for catching and logging panics transforming them to the 500 error
+/// response. The panic will not crash the service, but it becomes not live after
+/// exceeding the `Settings::service_live_counter_threshold()` value. That should cause
+/// Kubernetes to restart the service.
+pub struct CatchPanicMiddleware {}
+
+impl CatchPanicMiddleware {
+    /// Creates a new middleware instance.
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl<E: Endpoint> Middleware<E> for CatchPanicMiddleware {
+    type Output = CatchPanicEndpoint<E>;
+
+    fn transform(
+        &self,
+        ep: E,
+    ) -> Self::Output {
+        CatchPanicEndpoint { inner: ep }
+    }
+}
+
+/// An endpoint for the `CatchPanicMiddleware` middleware.
+pub struct CatchPanicEndpoint<E> {
+    /// An inner endpoint.
+    inner: E,
+}
+
+impl<E: Endpoint> Endpoint for CatchPanicEndpoint<E> {
+    type Output = Response;
+
+    async fn call(
+        &self,
+        req: Request,
+    ) -> poem::Result<Self::Output> {
+        // Preserve all the data that we want to potentially log because a request is consumed.
+        let method = req.method().clone();
+        let uri = req.uri().clone();
+        let headers = req.headers().clone();
+
+        match AssertUnwindSafe(self.inner.call(req)).catch_unwind().await {
+            Ok(resp) => resp.map(IntoResponse::into_response),
+            Err(err) => Ok(panic_response(&err, &method, &uri, &headers)),
+        }
+    }
+}
+
+/// Converts a panic to a response.
+fn panic_response(
+    err: &Box<dyn Any + Send + 'static>,
+    method: &Method,
+    uri: &Uri,
+    headers: &HeaderMap,
+) -> Response {
+    // Increment the counter used for liveness checks.
+    inc_live_counter();
+
+    let current_count = get_live_counter();
+    debug!(
+        live_counter = current_count,
+        "Handling service panic response"
+    );
+
+    // If current count is above the threshold, then flag the system as NOT live.
+    if current_count > Settings::service_live_counter_threshold() {
+        set_not_live();
+    }
+
+    let server_err = InternalServerError::new(None);
+
+    // Get the unique identifier for this panic, so we can find it in the logs.
+    let panic_identifier = server_err.id().to_string();
+
+    // Get the message from the panic as best we can.
+    let err_msg = panic_message(err);
+
+    // This is the location of the panic.
+    let location = match LOCATION.with(|l| l.borrow_mut().take()) {
+        Some(location) => location,
+        None => "Unknown".to_string(),
+    };
+
+    // This is the backtrace of the panic.
+    let backtrace = match BACKTRACE.with(|b| b.borrow_mut().take()) {
+        Some(backtrace) => backtrace,
+        None => "Unknown".to_string(),
+    };
+
+    error!(
+        panic = true,
+        backtrace = backtrace,
+        location = location,
+        id = panic_identifier,
+        message = err_msg,
+        method = ?method,
+        uri = ?uri,
+        headers = ?headers,
+    );
+
+    let mut resp = Json(server_err).into_response();
+    resp.set_status(StatusCode::INTERNAL_SERVER_ERROR);
+    resp
+}

catalyst-gateway/bin/src/service/utilities/middleware/catch_panic.rs

@@ -37,7 +37,7 @@ impl<E: Endpoint> Endpoint for DatabaseConnectionImpl<E> {
 
         // TODO: find a better way to filter URI paths
         let is_health_endpoint = req_path.starts_with("/api/v1/health/");
-        let is_metrics_endpoint = req_path == "/metrics";
+        let is_metrics_endpoint = req_path == "/metrics" || req_path == "/panic";
 
         if !(is_health_endpoint || is_metrics_endpoint) {
             if !event_db_is_live() {

catalyst-gateway/bin/src/service/utilities/middleware/db_check.rs

@@ -1,5 +1,6 @@
 //! Custom POEM Middleware for this service.
 
+pub(crate) mod catch_panic;
 pub(crate) mod db_check;
 pub(crate) mod metrics_updater;
 pub(crate) mod node_info;

catalyst-gateway/bin/src/service/utilities/middleware/mod.rs

@@ -1,10 +1,10 @@
 //! `API` Utility operations
 pub(crate) mod cache;
-pub(crate) mod catch_panic;
 pub(crate) mod convert;
 pub(crate) mod health;
 pub(crate) mod middleware;
 pub(crate) mod net;
+pub(crate) mod panic;
 
 use anyhow::{bail, Result};
 // use pallas::ledger::addresses::Network as PallasNetwork;

catalyst-gateway/bin/src/service/utilities/mod.rs

@@ -0,0 +1,9 @@
+//! An implementation of the panic endpoint.
+
+/// Implementation of the special `/panic` endpoint,
+/// which is used only for testing purposes and enabled for all networks except `Mainnet`
+#[poem::handler]
+#[allow(clippy::panic)]
+pub(crate) fn panic_endpoint() {
+    panic!("Intentional panicking")
+}