improve: handle upload errors (#736)

* refactor: defaults to empty emitter

* refactor chunk_size

* collect stats in upload_finished event

* refactor _session_key

* remove UploadCancelled

* re-raise ExifError from ExifWrite exceptions

* clean up exceptions

* throw EmptyZipFileError

* Capture upload errors and return as results

* Handle errors from result generator

* fix types

* fix types

* validate user_items in upload.py

* introduce uploader.SequenceError and use it to decide if we should
interrupte the overall upload process

* raise InvalidMapillaryZipFileError if the zip file does not contain valid metadata

* improve history emitter setup

* fix tests

* tests

* raise ex from prev ex

Author: ptpt

mapillary_tools/exceptions.py

@@ -1,10 +1,24 @@
+from __future__ import annotations
+
 import typing as T
 
 
 class MapillaryUserError(Exception):
     exit_code: int
 
 
+class MapillaryProcessError(MapillaryUserError):
+    """
+    Base exception for process specific errors
+    """
+
+    exit_code = 6
+
+
+class MapillaryDescriptionError(Exception):
+    pass
+
+
 class MapillaryBadParameterError(MapillaryUserError):
     exit_code = 2
 
@@ -17,44 +31,35 @@ class MapillaryInvalidDescriptionFile(MapillaryUserError):
     exit_code = 4
 
 
-class MapillaryUnknownFileTypeError(MapillaryUserError):
-    exit_code = 5
-
-
-class MapillaryProcessError(MapillaryUserError):
-    exit_code = 6
-
-
 class MapillaryVideoError(MapillaryUserError):
     exit_code = 7
 
 
 class MapillaryFFmpegNotFoundError(MapillaryUserError):
     exit_code = 8
-    help = "https://github.com/mapillary/mapillary_tools#video-support"
 
 
 class MapillaryExiftoolNotFoundError(MapillaryUserError):
     exit_code = 8
 
 
-class MapillaryDescriptionError(Exception):
+class MapillaryGeoTaggingError(MapillaryDescriptionError):
     pass
 
 
-class MapillaryGeoTaggingError(MapillaryDescriptionError):
+class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError):
     pass
 
 
-class MapillaryGPXEmptyError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 9
+class MapillaryGPXEmptyError(MapillaryDescriptionError):
+    pass
 
 
-class MapillaryVideoGPSNotFoundError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 9
+class MapillaryGPSNoiseError(MapillaryDescriptionError):
+    pass
 
 
-class MapillaryGPSNoiseError(MapillaryDescriptionError):
+class MapillaryStationaryVideoError(MapillaryDescriptionError):
     pass
 
 
@@ -68,39 +73,33 @@ def __init__(
         self.gpx_end_time = gpx_end_time
 
 
-class MapillaryStationaryVideoError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 10
-
-
-class MapillaryInvalidBlackVueVideoError(MapillaryDescriptionError, MapillaryUserError):
-    exit_code = 11
-
-
 class MapillaryDuplicationError(MapillaryDescriptionError):
     def __init__(
         self,
         message: str,
         desc: T.Mapping[str, T.Any],
         distance: float,
-        angle_diff: T.Optional[float],
+        angle_diff: float | None,
     ) -> None:
         super().__init__(message)
         self.desc = desc
         self.distance = distance
         self.angle_diff = angle_diff
 
 
-class MapillaryUploadedAlreadyError(MapillaryDescriptionError):
-    def __init__(
-        self,
-        message: str,
-        desc: T.Mapping[str, T.Any],
-    ) -> None:
-        super().__init__(message)
-        self.desc = desc
+class MapillaryEXIFNotFoundError(MapillaryDescriptionError):
+    pass
 
 
-class MapillaryEXIFNotFoundError(MapillaryDescriptionError):
+class MapillaryFileTooLargeError(MapillaryDescriptionError):
+    pass
+
+
+class MapillaryCaptureSpeedTooFastError(MapillaryDescriptionError):
+    pass
+
+
+class MapillaryNullIslandError(MapillaryDescriptionError):
     pass
 
 
@@ -116,17 +115,5 @@ class MapillaryUploadUnauthorizedError(MapillaryUserError):
     exit_code = 14
 
 
-class MapillaryMetadataValidationError(MapillaryUserError, MapillaryDescriptionError):
+class MapillaryMetadataValidationError(MapillaryUserError):
     exit_code = 15
-
-
-class MapillaryFileTooLargeError(MapillaryDescriptionError):
-    pass
-
-
-class MapillaryCaptureSpeedTooFastError(MapillaryDescriptionError):
-    pass
-
-
-class MapillaryNullIslandError(MapillaryDescriptionError):
-    pass

mapillary_tools/mp4/io_utils.py

@@ -3,7 +3,6 @@
 
 
 class ChainedIO(io.IOBase):
-    # is the chained stream seekable?
     _streams: T.Sequence[io.IOBase]
     # the beginning offset of the current stream
     _begin_offset: int

mapillary_tools/process_geotag_properties.py

@@ -562,10 +562,7 @@ def process_finalize(
         # skip all exceptions
         skipped_process_errors = {Exception}
     else:
-        skipped_process_errors = {
-            exceptions.MapillaryDuplicationError,
-            exceptions.MapillaryUploadedAlreadyError,
-        }
+        skipped_process_errors = {exceptions.MapillaryDuplicationError}
     _show_stats(metadatas, skipped_process_errors=skipped_process_errors)
 
     return metadatas

mapillary_tools/types.py

@@ -456,19 +456,19 @@ def validate_image_desc(desc: T.Any) -> None:
         jsonschema.validate(instance=desc, schema=ImageDescriptionFileSchema)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
-        raise exceptions.MapillaryMetadataValidationError(ex.message)
+        raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
     try:
         map_capture_time_to_datetime(desc["MAPCaptureTime"])
     except ValueError as ex:
-        raise exceptions.MapillaryMetadataValidationError(str(ex))
+        raise exceptions.MapillaryMetadataValidationError(str(ex)) from ex
 
 
 def validate_video_desc(desc: T.Any) -> None:
     try:
         jsonschema.validate(instance=desc, schema=VideoDescriptionFileSchema)
     except jsonschema.ValidationError as ex:
         # do not use str(ex) which is more verbose
-        raise exceptions.MapillaryMetadataValidationError(ex.message)
+        raise exceptions.MapillaryMetadataValidationError(ex.message) from ex
 
 
 def datetime_to_map_capture_time(time: T.Union[datetime.datetime, int, float]) -> str: