fix(service): Give tolerance in errors - NaNs caught later

Author: devsjc

src/nwp_consumer/internal/repositories/raw_repositories/ecmwf_realtime.py

@@ -303,11 +303,10 @@ def _convert(path: pathlib.Path) -> ResultE[list[xr.DataArray]]:
             )
 
         if len(processed_das) == 0:
-            # Some files do not have data for the relevant area, so don't error here
-            log.warning(
+            return Failure(ValueError(
                 f"Skipped {num_skipped}/{len(dss)} datasets from '{path}'. "
                 "File may not contain the expected parameters and geographic bounds.",
-            )
+            ))
 
         return Success(processed_das)
 

src/nwp_consumer/internal/services/consumer_service.py

@@ -74,20 +74,24 @@ def _fold_dataarrays_generator(
         for value in generator:
             if isinstance(value, Failure):
                 results.extend([value])
+            elif len(value.unwrap()) == 0:
+                continue
             else:
                 results.extend([store.write_to_region(da=da) for da in value.unwrap()])
         successes, failures = partition(results)
         # TODO: Define the failure threshold for number of write attempts properly
+        # ECMWF Realtime for instance needs a bit of tolerance because of some files not containing
+        # data for all geographic regions.
         log.info(f"Processed {len(successes)} DataArrays successfully with {len(failures)} errors.")
-        if len(failures) > 0:
+        if len(failures)/len(results) > 0.06:
             for i, exc in enumerate(failures):
                 if i < 5:
                     log.error(str(exc))
                 else:
                     break
             return Failure(OSError(
                 "Error threshold exceeded: "
-                f"{len(failures)} errors (>0) occurred during processing.",
+                f"{len(failures)/len(results)} errors (>6%) occurred during processing.",
             ))
         else:
             return Success(sum(successes))