Log snapshot import worker dying (#39074)

GitOrigin-RevId: 8de8e08f96c82fbb2e7861d139d079bbbc2657f6

Author: nipunn1313

crates/application/src/snapshot_import/metrics.rs

@@ -1,7 +1,9 @@
 use std::time::Duration;
 
 use metrics::{
+    log_counter,
     log_distribution,
+    register_convex_counter,
     register_convex_histogram,
     StatusTimer,
     STATUS_LABEL,
@@ -23,3 +25,11 @@ register_convex_histogram!(
 pub fn log_snapshot_import_age(age: Duration) {
     log_distribution(&SNAPSHOT_IMPORT_AGE_SECONDS, age.as_secs_f64());
 }
+
+register_convex_counter!(
+    SNAPSHOT_IMPORT_WORKER_DIED_TOTAL,
+    "Number of times the snapshot import worker died",
+);
+pub fn log_snapshot_import_worker_died() {
+    log_counter(&SNAPSHOT_IMPORT_WORKER_DIED_TOTAL, 1);
+}

crates/application/src/snapshot_import/worker.rs

@@ -22,7 +22,10 @@ use usage_tracking::UsageCounter;
 use crate::{
     metrics::log_worker_starting,
     snapshot_import::{
-        metrics::snapshot_import_timer,
+        metrics::{
+            log_snapshot_import_worker_died,
+            snapshot_import_timer,
+        },
         SnapshotImportExecutor,
     },
 };
@@ -51,6 +54,7 @@ impl SnapshotImportWorker {
         async move {
             loop {
                 if let Err(e) = Self::run_once(&mut worker).await {
+                    log_snapshot_import_worker_died();
                     report_error(&mut e.context("SnapshotImportWorker died")).await;
                     let delay = worker.backoff.fail(&mut worker.runtime.rng());
                     worker.runtime.wait(delay).await;