Skip to content

Commit f76354e

Browse files
ptptptpt
authored
improve: remove md5sum calculation from processing stage (#725)
* improve: remove md5sum calculation from processing stage * ruff check
1 parent 0325e6e commit f76354e

File tree

9 files changed

+3
-40
lines changed

9 files changed

+3
-40
lines changed

‎mapillary_tools/camm/camm_parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
import logging
88
import typing as T
99
from enum import Enum
10-
from typing_extensions import TypeIs
1110

1211
import construct as C
12+
from typing_extensions import TypeIs
1313

1414
from .. import geo, telemetry
1515
from ..mp4 import simple_mp4_parser as sparser

‎mapillary_tools/geotag/geotag_images_from_exif.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ def build_image_metadata(
4141

4242
image_metadata = types.ImageMetadata(
4343
filename=image_path,
44-
md5sum=None,
4544
filesize=utils.get_file_size(image_path),
4645
time=geo.as_unix_time(capture_time),
4746
lat=lat,

‎mapillary_tools/geotag/geotag_images_from_exiftool.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import io
21
import logging
32
import typing as T
43
import xml.etree.ElementTree as ET
@@ -36,18 +35,11 @@ def geotag_image(element: ET.Element) -> types.ImageMetadataOrError:
3635
image_metadata = GeotagImagesFromEXIF.build_image_metadata(
3736
image_path, exif, skip_lonlat_error=False
3837
)
39-
# load the image bytes into memory to avoid reading it multiple times
40-
with image_path.open("rb") as fp:
41-
image_bytesio = io.BytesIO(fp.read())
42-
image_bytesio.seek(0, io.SEEK_SET)
4338
except Exception as ex:
4439
return types.describe_error_metadata(
4540
ex, image_path, filetype=types.FileType.IMAGE
4641
)
4742

48-
image_bytesio.seek(0, io.SEEK_SET)
49-
image_metadata.update_md5sum(image_bytesio)
50-
5143
return image_metadata
5244

5345
def to_description(self) -> T.List[types.ImageMetadataOrError]:

‎mapillary_tools/geotag/geotag_videos_from_exiftool_video.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,18 +68,13 @@ def geotag_video(element: ET.Element) -> types.VideoMetadataOrError:
6868

6969
video_metadata = types.VideoMetadata(
7070
video_path,
71-
md5sum=None,
7271
filesize=utils.get_file_size(video_path),
7372
filetype=types.FileType.VIDEO,
7473
points=points,
7574
make=exif.extract_make(),
7675
model=exif.extract_model(),
7776
)
7877

79-
LOG.debug("Calculating MD5 checksum for %s", str(video_metadata.filename))
80-
81-
video_metadata.update_md5sum()
82-
8378
except Exception as ex:
8479
if not isinstance(ex, exceptions.MapillaryDescriptionError):
8580
LOG.warning(

‎mapillary_tools/geotag/geotag_videos_from_video.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ def _extract_video_metadata(
8080
if gopro_info is not None:
8181
return types.VideoMetadata(
8282
filename=video_path,
83-
md5sum=None,
8483
filesize=utils.get_file_size(video_path),
8584
filetype=types.FileType.GOPRO,
8685
points=T.cast(T.List[geo.Point], gopro_info.gps),
@@ -104,7 +103,6 @@ def _extract_video_metadata(
104103
make, model = camm_parser.extract_camera_make_and_model(fp)
105104
return types.VideoMetadata(
106105
filename=video_path,
107-
md5sum=None,
108106
filesize=utils.get_file_size(video_path),
109107
filetype=types.FileType.CAMM,
110108
points=points,
@@ -128,7 +126,6 @@ def _extract_video_metadata(
128126
make, model = "BlackVue", blackvue_parser.extract_camera_model(fp)
129127
return types.VideoMetadata(
130128
filename=video_path,
131-
md5sum=None,
132129
filesize=utils.get_file_size(video_path),
133130
filetype=types.FileType.BLACKVUE,
134131
points=points,

‎mapillary_tools/types.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class FileType(enum.Enum):
4343
class ImageMetadata(geo.Point):
4444
filename: Path
4545
# if None or absent, it will be calculated
46-
md5sum: T.Optional[str]
46+
md5sum: T.Optional[str] = None
4747
# filetype: is always FileType.IMAGE
4848
width: T.Optional[int] = None
4949
height: T.Optional[int] = None
@@ -78,9 +78,9 @@ def sort_key(self):
7878
class VideoMetadata:
7979
filename: Path
8080
# if None or absent, it will be calculated
81-
md5sum: T.Optional[str]
8281
filetype: FileType
8382
points: T.Sequence[geo.Point]
83+
md5sum: T.Optional[str] = None
8484
make: T.Optional[str] = None
8585
model: T.Optional[str] = None
8686
filesize: T.Optional[int] = None

‎mapillary_tools/video_data_extraction/extract_video_data.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,13 +92,11 @@ def process_file(self, file: Path) -> VideoMetadataOrError:
9292
video_metadata = VideoMetadata(
9393
filename=file,
9494
filetype=FileType.VIDEO,
95-
md5sum=None,
9695
filesize=utils.get_file_size(file),
9796
points=points,
9897
make=make,
9998
model=model,
10099
)
101-
video_metadata.update_md5sum()
102100
return video_metadata
103101
else:
104102
return ErrorMetadata(

‎tests/unit/test_camm_parser.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ def encode_decode_empty_camm_mp4(metadata: types.VideoMetadata) -> types.VideoMe
7272
# return metadata
7373
return types.VideoMetadata(
7474
Path(""),
75-
None,
7675
filetype=types.FileType.CAMM,
7776
points=points or [],
7877
make=make,
@@ -105,7 +104,6 @@ def test_build_and_parse_points():
105104
]
106105
metadata = types.VideoMetadata(
107106
Path(""),
108-
None,
109107
filetype=types.FileType.CAMM,
110108
points=points,
111109
)
@@ -171,7 +169,6 @@ def test_build_and_parse_camm_gps_points():
171169
]
172170
metadata = types.VideoMetadata(
173171
Path(""),
174-
None,
175172
filetype=types.FileType.CAMM,
176173
points=points,
177174
)
@@ -219,7 +216,6 @@ def test_build_and_parse_single_points():
219216
]
220217
metadata = types.VideoMetadata(
221218
Path(""),
222-
None,
223219
filetype=types.FileType.CAMM,
224220
points=points,
225221
)
@@ -238,7 +234,6 @@ def test_build_and_parse_single_point_0():
238234
]
239235
metadata = types.VideoMetadata(
240236
Path(""),
241-
None,
242237
filetype=types.FileType.CAMM,
243238
points=points,
244239
)
@@ -257,7 +252,6 @@ def test_build_and_parse_single_point_neg():
257252
]
258253
metadata = types.VideoMetadata(
259254
Path(""),
260-
None,
261255
filetype=types.FileType.CAMM,
262256
points=points,
263257
)
@@ -273,7 +267,6 @@ def test_build_and_parse_start_early():
273267
]
274268
metadata = types.VideoMetadata(
275269
Path(""),
276-
None,
277270
filetype=types.FileType.CAMM,
278271
points=points,
279272
)
@@ -293,7 +286,6 @@ def test_build_and_parse2():
293286
]
294287
metadata = types.VideoMetadata(
295288
Path(""),
296-
None,
297289
filetype=types.FileType.CAMM,
298290
points=points,
299291
make="test_make汉字",
@@ -314,7 +306,6 @@ def test_build_and_parse9():
314306
]
315307
metadata = types.VideoMetadata(
316308
Path(""),
317-
None,
318309
filetype=types.FileType.CAMM,
319310
points=points,
320311
make="test_make汉字",
@@ -331,7 +322,6 @@ def test_build_and_parse10():
331322
]
332323
metadata = types.VideoMetadata(
333324
Path(""),
334-
None,
335325
filetype=types.FileType.CAMM,
336326
points=points,
337327
make="test_make汉字",
@@ -351,7 +341,6 @@ def test_build_and_parse3():
351341
points = []
352342
metadata = types.VideoMetadata(
353343
Path(""),
354-
None,
355344
filetype=types.FileType.CAMM,
356345
points=points,
357346
make="test_make汉字",

‎tests/unit/test_sequence_processing.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ def _make_image_metadata(
3030
pass
3131
return types.ImageMetadata(
3232
filename=filename,
33-
md5sum=None,
3433
lon=lng,
3534
lat=lat,
3635
time=time,
@@ -323,7 +322,6 @@ def test_interpolation(tmpdir: py.path.local):
323322
),
324323
types.VideoMetadata(
325324
Path("test_video.mp4"),
326-
None,
327325
types.FileType.IMAGE,
328326
points=[],
329327
make="hello",
@@ -423,7 +421,6 @@ def test_process_finalize(setup_data):
423421
_make_image_metadata(Path(corrupt_exif), 1000, 1, 4, angle=22),
424422
types.VideoMetadata(
425423
Path(setup_data.join("test_video.mp4")),
426-
None,
427424
types.FileType.IMAGE,
428425
points=[],
429426
make="hello",
@@ -544,7 +541,6 @@ def test_video_error(tmpdir: py.path.local):
544541
sequence: T.List[types.Metadata] = [
545542
types.VideoMetadata(
546543
Path(curdir) / Path("test_video_null_island.mp4"),
547-
None,
548544
types.FileType.VIDEO,
549545
points=[
550546
geo.Point(1, -0.00001, -0.00001, 1, angle=None),
@@ -557,7 +553,6 @@ def test_video_error(tmpdir: py.path.local):
557553
),
558554
types.VideoMetadata(
559555
Path(curdir) / Path("test_video_too_fast.mp4"),
560-
None,
561556
types.FileType.VIDEO,
562557
points=[
563558
geo.Point(1, 1, 1, 1, angle=None),
@@ -570,7 +565,6 @@ def test_video_error(tmpdir: py.path.local):
570565
),
571566
types.VideoMetadata(
572567
Path(curdir) / Path("test_video_file_too_large.mp4"),
573-
None,
574568
types.FileType.VIDEO,
575569
points=[geo.Point(1, 1, 1, 1, angle=None)],
576570
make="hello",
@@ -579,7 +573,6 @@ def test_video_error(tmpdir: py.path.local):
579573
),
580574
types.VideoMetadata(
581575
Path(curdir) / Path("test_good.mp4"),
582-
None,
583576
types.FileType.VIDEO,
584577
points=[geo.Point(1, 1, 1, 1, angle=None)],
585578
make="hello",

0 commit comments

Comments
 (0)