Add file sizes to output

Author: nickplesha

mapillary_tools/geotag/geotag_images_from_exif.py

@@ -6,6 +6,7 @@
 
 from tqdm import tqdm
 
+from mapillary_tools import utils
 from .. import exceptions, exif_write, geo, types
 from ..exif_read import ExifRead, ExifReadABC
 from .geotag_from_generic import GeotagImagesFromGeneric
@@ -64,6 +65,7 @@ def build_image_metadata(
         image_metadata = types.ImageMetadata(
             filename=image_path,
             md5sum=None,
+            size=utils.get_file_size(image_path),
             time=geo.as_unix_time(capture_time),
             lat=lat,
             lon=lon,

mapillary_tools/geotag/geotag_videos_from_exiftool_video.py

@@ -6,6 +6,7 @@
 
 from tqdm import tqdm
 
+from mapillary_tools import utils
 from .. import exceptions, exiftool_read, geo, types
 from ..exiftool_read_video import ExifToolReadVideo
 from ..telemetry import GPSPoint
@@ -66,6 +67,7 @@ def geotag_video(element: ET.Element) -> types.VideoMetadataOrError:
             video_metadata = types.VideoMetadata(
                 video_path,
                 md5sum=None,
+                size=utils.get_file_size(video_path),
                 filetype=types.FileType.VIDEO,
                 points=points,
                 make=exif.extract_make(),

mapillary_tools/geotag/geotag_videos_from_video.py

@@ -4,6 +4,8 @@
 from multiprocessing import Pool
 from pathlib import Path
 
+from mapillary_tools import utils
+
 from tqdm import tqdm
 
 from .. import exceptions, geo, types
@@ -82,6 +84,7 @@ def _extract_video_metadata(
                     return types.VideoMetadata(
                         filename=video_path,
                         md5sum=None,
+                        size=utils.get_file_size(video_path),
                         filetype=types.FileType.CAMM,
                         points=points,
                         make=make,
@@ -105,6 +108,7 @@ def _extract_video_metadata(
                     return types.VideoMetadata(
                         filename=video_path,
                         md5sum=None,
+                        size=utils.get_file_size(video_path),
                         filetype=types.FileType.GOPRO,
                         points=T.cast(T.List[geo.Point], points_with_fix),
                         make=make,
@@ -128,6 +132,7 @@ def _extract_video_metadata(
                     return types.VideoMetadata(
                         filename=video_path,
                         md5sum=None,
+                        size=utils.get_file_size(video_path),
                         filetype=types.FileType.BLACKVUE,
                         points=points,
                         make=make,

mapillary_tools/process_geotag_properties.py

@@ -424,19 +424,22 @@ def _show_stats_per_filetype(
     skipped_process_errors: T.Set[T.Type[Exception]],
 ):
     good_metadatas: T.List[T.Union[types.VideoMetadata, types.ImageMetadata]] = []
+    size_to_upload = 0
     error_metadatas: T.List[types.ErrorMetadata] = []
     for metadata in metadatas:
         if isinstance(metadata, types.ErrorMetadata):
             error_metadatas.append(metadata)
         else:
             good_metadatas.append(metadata)
+            size_to_upload += metadata.size
 
     LOG.info("%8d %s(s) read in total", len(metadatas), filetype.value)
     if good_metadatas:
         LOG.info(
-            "\t %8d %s(s) are ready to be uploaded",
+            "\t %8d %s(s) (%s MB) are ready to be uploaded",
             len(good_metadatas),
             filetype.value,
+            round(size_to_upload / 1024 / 1024, 1),
         )
 
     error_counter = collections.Counter(

mapillary_tools/types.py

@@ -56,6 +56,7 @@ class ImageMetadata(geo.Point):
     MAPMetaTags: T.Optional[T.Dict] = None
     # deprecated since v0.10.0; keep here for compatibility
     MAPFilename: T.Optional[str] = None
+    size: T.Optional[int] = None
 
     def update_md5sum(self, image_data: T.Optional[T.BinaryIO] = None) -> None:
         if self.md5sum is None:
@@ -81,6 +82,7 @@ class VideoMetadata:
     points: T.Sequence[geo.Point]
     make: T.Optional[str] = None
     model: T.Optional[str] = None
+    size: T.Optional[int] = None
 
     def update_md5sum(self) -> None:
         if self.md5sum is None:
@@ -143,6 +145,7 @@ class ImageDescription(_SequenceOnly, _Image, MetaProperties, total=True):
     # if None or absent, it will be calculated
     md5sum: T.Optional[str]
     filetype: Literal["image"]
+    size: T.Optional[int]
 
 
 class _VideoDescriptionRequired(TypedDict, total=True):
@@ -156,6 +159,7 @@ class _VideoDescriptionRequired(TypedDict, total=True):
 class VideoDescription(_VideoDescriptionRequired, total=False):
     MAPDeviceMake: str
     MAPDeviceModel: str
+    size: T.Optional[int]
 
 
 class _ErrorDescription(TypedDict, total=False):
@@ -368,6 +372,10 @@ def merge_schema(*schemas: T.Dict) -> T.Dict:
                 "type": ["string", "null"],
                 "description": "MD5 checksum of the image content. If not provided, the uploader will compute it",
             },
+            "size": {
+                "type": ["number", "null"],
+                "description": "File size",
+            },
             "filetype": {
                 "type": "string",
                 "enum": [FileType.IMAGE.value],
@@ -394,6 +402,10 @@ def merge_schema(*schemas: T.Dict) -> T.Dict:
                 "type": ["string", "null"],
                 "description": "MD5 checksum of the video content. If not provided, the uploader will compute it",
             },
+            "size": {
+                "type": ["number", "null"],
+                "description": "File size",
+            },
             "filetype": {
                 "type": "string",
                 "enum": [
@@ -484,6 +496,7 @@ def _as_video_desc(metadata: VideoMetadata) -> VideoDescription:
         "filename": str(metadata.filename.resolve()),
         "md5sum": metadata.md5sum,
         "filetype": metadata.filetype.value,
+        "size": metadata.size,
         "MAPGPSTrack": [_encode_point(p) for p in metadata.points],
     }
     if metadata.make:
@@ -497,6 +510,7 @@ def _as_image_desc(metadata: ImageMetadata) -> ImageDescription:
     desc: ImageDescription = {
         "filename": str(metadata.filename.resolve()),
         "md5sum": metadata.md5sum,
+        "size": metadata.size,
         "filetype": FileType.IMAGE.value,
         "MAPLatitude": round(metadata.lat, _COORDINATES_PRECISION),
         "MAPLongitude": round(metadata.lon, _COORDINATES_PRECISION),
@@ -542,6 +556,7 @@ def _from_image_desc(desc) -> ImageMetadata:
         if k not in [
             "filename",
             "md5sum",
+            "size",
             "filetype",
             "MAPLatitude",
             "MAPLongitude",
@@ -554,6 +569,7 @@ def _from_image_desc(desc) -> ImageMetadata:
     return ImageMetadata(
         filename=Path(desc["filename"]),
         md5sum=desc.get("md5sum"),
+        size=desc.get("size"),
         lat=desc["MAPLatitude"],
         lon=desc["MAPLongitude"],
         alt=desc.get("MAPAltitude"),
@@ -585,6 +601,7 @@ def _from_video_desc(desc: VideoDescription) -> VideoMetadata:
     return VideoMetadata(
         filename=Path(desc["filename"]),
         md5sum=desc["md5sum"],
+        size=desc["size"],
         filetype=FileType(desc["filetype"]),
         points=[_decode_point(entry) for entry in desc["MAPGPSTrack"]],
         make=desc.get("MAPDeviceMake"),

mapillary_tools/utils.py

@@ -201,3 +201,6 @@ def find_xml_files(import_paths: T.Sequence[Path]) -> T.List[Path]:
         else:
             xml_paths.append(path)
     return list(deduplicate_paths(xml_paths))
+
+def get_file_size(path: Path) -> int:
+    return os.path.getsize(path) 

mapillary_tools/video_data_extraction/extract_video_data.py

@@ -94,6 +94,7 @@ def process_file(self, file: Path) -> VideoMetadataOrError:
                 filename=file,
                 filetype=FileType.VIDEO,
                 md5sum=None,
+                size=utils.get_file_size(file),
                 points=points,
                 make=make,
                 model=model,

schema/image_description_schema.json

@@ -66,6 +66,10 @@
                         "video"
                     ],
                     "description": "The video file type"
+                },
+                "size": {
+                    "type": "number",
+                    "description": "File size"
                 }
             },
             "required": [
@@ -160,6 +164,10 @@
                         "image"
                     ],
                     "description": "The image file type"
+                },
+                "size": {
+                    "type": "number",
+                    "description": "File size"
                 }
             },
             "required": [