Ignore non-fatal farming errors, emit them as events and log as metrics

Author: nazar-pc

crates/subspace-farmer-components/src/proving.rs

@@ -60,6 +60,20 @@ pub enum ProvingError {
     RecordReadingError(#[from] ReadingError),
 }
 
+impl ProvingError {
+    /// Whether this error is fatal and makes farm unusable
+    pub fn is_fatal(&self) -> bool {
+        match self {
+            ProvingError::InvalidErasureCodingInstance => true,
+            ProvingError::FailedToCreatePolynomialForRecord { .. } => false,
+            ProvingError::FailedToCreateChunkWitness { .. } => false,
+            ProvingError::FailedToDecodeSectorContentsMap(_) => false,
+            ProvingError::Io(_) => true,
+            ProvingError::RecordReadingError(error) => error.is_fatal(),
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 struct WinningChunk {
     /// Chunk offset within s-bucket

crates/subspace-farmer-components/src/reading.rs

@@ -75,6 +75,21 @@ pub enum ReadingError {
     ChecksumMismatch,
 }
 
+impl ReadingError {
+    /// Whether this error is fatal and renders farm unusable
+    pub fn is_fatal(&self) -> bool {
+        match self {
+            ReadingError::FailedToReadChunk { .. } => false,
+            ReadingError::InvalidChunk { .. } => false,
+            ReadingError::FailedToErasureDecodeRecord { .. } => false,
+            ReadingError::WrongRecordSizeAfterDecoding { .. } => false,
+            ReadingError::FailedToDecodeSectorContentsMap(_) => false,
+            ReadingError::Io(_) => true,
+            ReadingError::ChecksumMismatch => false,
+        }
+    }
+}
+
 /// Record contained in the plot
 #[derive(Debug, Clone)]
 pub struct PlotRecord {

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs

@@ -761,6 +761,9 @@ where
                                 proving_details.result,
                             );
                         }
+                        FarmingNotification::NonFatalError(error) => {
+                            farmer_metrics.note_farming_error(&single_disk_farm_id, error);
+                        }
                     }
                 }))
                 .detach();

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm/metrics.rs

@@ -5,12 +5,13 @@ use prometheus_client::registry::{Registry, Unit};
 use std::sync::atomic::AtomicU64;
 use std::time::Duration;
 use subspace_farmer::single_disk_farm::farming::ProvingResult;
-use subspace_farmer::single_disk_farm::SingleDiskFarmId;
+use subspace_farmer::single_disk_farm::{FarmingError, SingleDiskFarmId};
 
 #[derive(Debug, Clone)]
 pub(super) struct FarmerMetrics {
     auditing_time: Family<Vec<(String, String)>, Histogram>,
     proving_time: Family<Vec<(String, String)>, Histogram>,
+    farming_errors: Family<Vec<(String, String)>, Counter<u64, AtomicU64>>,
     sector_downloading_time: Family<Vec<(String, String)>, Histogram>,
     sector_encoding_time: Family<Vec<(String, String)>, Histogram>,
     sector_writing_time: Family<Vec<(String, String)>, Histogram>,
@@ -51,6 +52,14 @@ impl FarmerMetrics {
             proving_time.clone(),
         );
 
+        let farming_errors = Family::<_, _>::new_with_constructor(Counter::<_, _>::default);
+
+        sub_registry.register(
+            "farming_errors",
+            "Non-fatal farming errors",
+            farming_errors.clone(),
+        );
+
         let sector_downloading_time = Family::<_, _>::new_with_constructor(|| {
             Histogram::new(exponential_buckets(0.0001, 2.0, 15))
         });
@@ -170,6 +179,7 @@ impl FarmerMetrics {
         Self {
             auditing_time,
             proving_time,
+            farming_errors,
             sector_downloading_time,
             sector_encoding_time,
             sector_writing_time,
@@ -212,6 +222,19 @@ impl FarmerMetrics {
             .observe(time.as_secs_f64());
     }
 
+    pub(super) fn note_farming_error(
+        &self,
+        single_disk_farm_id: &SingleDiskFarmId,
+        error: &FarmingError,
+    ) {
+        self.farming_errors
+            .get_or_create(&vec![
+                ("farm_id".to_string(), single_disk_farm_id.to_string()),
+                ("error".to_string(), error.str_variant().to_string()),
+            ])
+            .inc();
+    }
+
     pub(super) fn observe_sector_downloading_time(
         &self,
         single_disk_farm_id: &SingleDiskFarmId,