Tag snapshot export/import failure metrics with error type (#39135)

Then we can alert more aggressively on non-operational errors.

Stuff like TableSummaryBootstrapping is operational.

GitOrigin-RevId: 37f4375a801641f398ee04cc3096c877a5d2ffdc

Author: nipunn1313

crates/application/src/exports/metrics.rs

@@ -1,5 +1,6 @@
+use errors::ErrorMetadataAnyhowExt;
 use metrics::{
-    log_counter,
+    log_counter_with_labels,
     register_convex_counter,
     register_convex_histogram,
     StaticMetricLabel,
@@ -23,7 +24,13 @@ pub fn export_timer(instance_name: &str) -> StatusTimer {
 register_convex_counter!(
     SNAPSHOT_EXPORT_FAILED_TOTAL,
     "Number of snapshot export attempts that failed",
+    &["status"]
 );
-pub fn log_export_failed() {
-    log_counter(&SNAPSHOT_EXPORT_FAILED_TOTAL, 1);
+pub fn log_export_failed(e: &anyhow::Error) {
+    let status = e.metric_status_label_value();
+    log_counter_with_labels(
+        &SNAPSHOT_EXPORT_FAILED_TOTAL,
+        1,
+        vec![StaticMetricLabel::new("status", status)],
+    );
 }

crates/application/src/exports/worker.rs

@@ -183,7 +183,7 @@ impl<RT: Runtime> ExportWorker<RT> {
                         tracing::info!("Export {} canceled", export.id());
                         return Ok(());
                     }
-                    log_export_failed();
+                    log_export_failed(&e);
                     report_error(&mut e).await;
                     let delay = self.backoff.fail(&mut self.runtime.rng());
                     tracing::error!("Export failed, retrying in {delay:?}");

crates/application/src/snapshot_import/metrics.rs

@@ -1,10 +1,12 @@
 use std::time::Duration;
 
+use errors::ErrorMetadataAnyhowExt;
 use metrics::{
-    log_counter,
+    log_counter_with_labels,
     log_distribution,
     register_convex_counter,
     register_convex_histogram,
+    StaticMetricLabel,
     StatusTimer,
     STATUS_LABEL,
 };
@@ -27,9 +29,15 @@ pub fn log_snapshot_import_age(age: Duration) {
 }
 
 register_convex_counter!(
-    SNAPSHOT_IMPORT_WORKER_DIED_TOTAL,
+    SNAPSHOT_IMPORT_FAILED_TOTAL,
     "Number of times the snapshot import worker died",
+    &["status"]
 );
-pub fn log_snapshot_import_worker_died() {
-    log_counter(&SNAPSHOT_IMPORT_WORKER_DIED_TOTAL, 1);
+pub fn log_snapshot_import_failed(e: &anyhow::Error) {
+    let status = e.metric_status_label_value();
+    log_counter_with_labels(
+        &SNAPSHOT_IMPORT_FAILED_TOTAL,
+        1,
+        vec![StaticMetricLabel::new("status", status)],
+    );
 }

crates/application/src/snapshot_import/worker.rs

@@ -23,7 +23,7 @@ use crate::{
     metrics::log_worker_starting,
     snapshot_import::{
         metrics::{
-            log_snapshot_import_worker_died,
+            log_snapshot_import_failed,
             snapshot_import_timer,
         },
         SnapshotImportExecutor,
@@ -54,7 +54,7 @@ impl SnapshotImportWorker {
         async move {
             loop {
                 if let Err(e) = Self::run_once(&mut worker).await {
-                    log_snapshot_import_worker_died();
+                    log_snapshot_import_failed(&e);
                     report_error(&mut e.context("SnapshotImportWorker died")).await;
                     let delay = worker.backoff.fail(&mut worker.runtime.rng());
                     worker.runtime.wait(delay).await;