improve: remove md5sum calculation from processing stage

mapillary_tools/camm/camm_parser.py

@@ -7,9 +7,9 @@
 import logging
 import typing as T
 from enum import Enum
-from typing_extensions import TypeIs
 
 import construct as C
+from typing_extensions import TypeIs
 
 from .. import geo, telemetry
 from ..mp4 import simple_mp4_parser as sparser

mapillary_tools/geotag/geotag_images_from_exif.py

@@ -41,7 +41,6 @@ def build_image_metadata(
 
         image_metadata = types.ImageMetadata(
             filename=image_path,
-            md5sum=None,
             filesize=utils.get_file_size(image_path),
             time=geo.as_unix_time(capture_time),
             lat=lat,

mapillary_tools/geotag/geotag_images_from_exiftool.py

@@ -1,4 +1,3 @@
-import io
 import logging
 import typing as T
 import xml.etree.ElementTree as ET
@@ -36,18 +35,11 @@ def geotag_image(element: ET.Element) -> types.ImageMetadataOrError:
             image_metadata = GeotagImagesFromEXIF.build_image_metadata(
                 image_path, exif, skip_lonlat_error=False
             )
-            # load the image bytes into memory to avoid reading it multiple times
-            with image_path.open("rb") as fp:
-                image_bytesio = io.BytesIO(fp.read())
-            image_bytesio.seek(0, io.SEEK_SET)
         except Exception as ex:
             return types.describe_error_metadata(
                 ex, image_path, filetype=types.FileType.IMAGE
             )
 
-        image_bytesio.seek(0, io.SEEK_SET)
-        image_metadata.update_md5sum(image_bytesio)
-
         return image_metadata
 
     def to_description(self) -> T.List[types.ImageMetadataOrError]:

mapillary_tools/geotag/geotag_videos_from_exiftool_video.py

@@ -68,18 +68,13 @@ def geotag_video(element: ET.Element) -> types.VideoMetadataOrError:
 
             video_metadata = types.VideoMetadata(
                 video_path,
-                md5sum=None,
                 filesize=utils.get_file_size(video_path),
                 filetype=types.FileType.VIDEO,
                 points=points,
                 make=exif.extract_make(),
                 model=exif.extract_model(),
             )
 
-            LOG.debug("Calculating MD5 checksum for %s", str(video_metadata.filename))
-
-            video_metadata.update_md5sum()
-
         except Exception as ex:
             if not isinstance(ex, exceptions.MapillaryDescriptionError):
                 LOG.warning(

mapillary_tools/geotag/geotag_videos_from_video.py

@@ -80,7 +80,6 @@ def _extract_video_metadata(
             if gopro_info is not None:
                 return types.VideoMetadata(
                     filename=video_path,
-                    md5sum=None,
                     filesize=utils.get_file_size(video_path),
                     filetype=types.FileType.GOPRO,
                     points=T.cast(T.List[geo.Point], gopro_info.gps),
@@ -104,7 +103,6 @@ def _extract_video_metadata(
                     make, model = camm_parser.extract_camera_make_and_model(fp)
                     return types.VideoMetadata(
                         filename=video_path,
-                        md5sum=None,
                         filesize=utils.get_file_size(video_path),
                         filetype=types.FileType.CAMM,
                         points=points,
@@ -128,7 +126,6 @@ def _extract_video_metadata(
                     make, model = "BlackVue", blackvue_parser.extract_camera_model(fp)
                     return types.VideoMetadata(
                         filename=video_path,
-                        md5sum=None,
                         filesize=utils.get_file_size(video_path),
                         filetype=types.FileType.BLACKVUE,
                         points=points,

mapillary_tools/types.py

@@ -43,7 +43,7 @@ class FileType(enum.Enum):
 class ImageMetadata(geo.Point):
     filename: Path
     # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
+    md5sum: T.Optional[str] = None
     # filetype: is always FileType.IMAGE
     width: T.Optional[int] = None
     height: T.Optional[int] = None
@@ -78,9 +78,9 @@ def sort_key(self):
 class VideoMetadata:
     filename: Path
     # if None or absent, it will be calculated
-    md5sum: T.Optional[str]
     filetype: FileType
     points: T.Sequence[geo.Point]
+    md5sum: T.Optional[str] = None
     make: T.Optional[str] = None
     model: T.Optional[str] = None
     filesize: T.Optional[int] = None

mapillary_tools/video_data_extraction/extract_video_data.py

@@ -92,13 +92,11 @@ def process_file(self, file: Path) -> VideoMetadataOrError:
             video_metadata = VideoMetadata(
                 filename=file,
                 filetype=FileType.VIDEO,
-                md5sum=None,
                 filesize=utils.get_file_size(file),
                 points=points,
                 make=make,
                 model=model,
             )
-            video_metadata.update_md5sum()
             return video_metadata
         else:
             return ErrorMetadata(

tests/unit/test_camm_parser.py

@@ -72,7 +72,6 @@ def encode_decode_empty_camm_mp4(metadata: types.VideoMetadata) -> types.VideoMe
     # return metadata
     return types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points or [],
         make=make,
@@ -105,7 +104,6 @@ def test_build_and_parse_points():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
     )
@@ -171,7 +169,6 @@ def test_build_and_parse_camm_gps_points():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
     )
@@ -219,7 +216,6 @@ def test_build_and_parse_single_points():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
     )
@@ -238,7 +234,6 @@ def test_build_and_parse_single_point_0():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
     )
@@ -257,7 +252,6 @@ def test_build_and_parse_single_point_neg():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
     )
@@ -273,7 +267,6 @@ def test_build_and_parse_start_early():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
     )
@@ -293,7 +286,6 @@ def test_build_and_parse2():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
         make="test_make汉字",
@@ -314,7 +306,6 @@ def test_build_and_parse9():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
         make="test_make汉字",
@@ -331,7 +322,6 @@ def test_build_and_parse10():
     ]
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
         make="test_make汉字",
@@ -351,7 +341,6 @@ def test_build_and_parse3():
     points = []
     metadata = types.VideoMetadata(
         Path(""),
-        None,
         filetype=types.FileType.CAMM,
         points=points,
         make="test_make汉字",

tests/unit/test_sequence_processing.py

@@ -30,7 +30,6 @@ def _make_image_metadata(
             pass
     return types.ImageMetadata(
         filename=filename,
-        md5sum=None,
         lon=lng,
         lat=lat,
         time=time,
@@ -323,7 +322,6 @@ def test_interpolation(tmpdir: py.path.local):
         ),
         types.VideoMetadata(
             Path("test_video.mp4"),
-            None,
             types.FileType.IMAGE,
             points=[],
             make="hello",
@@ -423,7 +421,6 @@ def test_process_finalize(setup_data):
         _make_image_metadata(Path(corrupt_exif), 1000, 1, 4, angle=22),
         types.VideoMetadata(
             Path(setup_data.join("test_video.mp4")),
-            None,
             types.FileType.IMAGE,
             points=[],
             make="hello",
@@ -544,7 +541,6 @@ def test_video_error(tmpdir: py.path.local):
     sequence: T.List[types.Metadata] = [
         types.VideoMetadata(
             Path(curdir) / Path("test_video_null_island.mp4"),
-            None,
             types.FileType.VIDEO,
             points=[
                 geo.Point(1, -0.00001, -0.00001, 1, angle=None),
@@ -557,7 +553,6 @@ def test_video_error(tmpdir: py.path.local):
         ),
         types.VideoMetadata(
             Path(curdir) / Path("test_video_too_fast.mp4"),
-            None,
             types.FileType.VIDEO,
             points=[
                 geo.Point(1, 1, 1, 1, angle=None),
@@ -570,7 +565,6 @@ def test_video_error(tmpdir: py.path.local):
         ),
         types.VideoMetadata(
             Path(curdir) / Path("test_video_file_too_large.mp4"),
-            None,
             types.FileType.VIDEO,
             points=[geo.Point(1, 1, 1, 1, angle=None)],
             make="hello",
@@ -579,7 +573,6 @@ def test_video_error(tmpdir: py.path.local):
         ),
         types.VideoMetadata(
             Path(curdir) / Path("test_good.mp4"),
-            None,
             types.FileType.VIDEO,
             points=[geo.Point(1, 1, 1, 1, angle=None)],
             make="hello",