fix(cli): make file size display consistent (#3111)

Author: Panaetius

poetry.lock

@@ -483,17 +483,17 @@ def confirm_download(files):
         if yes:
             return
 
-        headers = {"checksum": "checksum", "filename": "name", "size_in_mb": "size (b)", "filetype": "type"}
+        headers = {"checksum": "checksum", "filename": "name", "filesize_str": "size", "filetype": "type"}
         communication.echo(tabulate(files, headers=headers, floatfmt=".2f"))
         communication.confirm("Do you wish to download this version?", abort=True, warning=True)
 
     def calculate_total_size(files):
         total_size = 0.0
         for file in files:
-            if file.size_in_mb is not None:
-                total_size += file.size_in_mb
+            if file.filesize is not None:
+                total_size += file.filesize
 
-        return total_size * 2**20
+        return total_size
 
     def remove_files(dataset):
         """Remove files that exist in ``previous_dataset`` but not in ``dataset``.

renku/core/dataset/dataset.py

@@ -39,7 +39,6 @@
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.util import communication
 from renku.core.util.doi import extract_doi, get_doi_url, is_doi
-from renku.core.util.file_size import bytes_to_unit
 from renku.core.util.urls import remove_credentials
 
 if TYPE_CHECKING:
@@ -283,7 +282,7 @@ def fix_data(self, data, **kwargs):
                 source=file.remote_url.geturl(),
                 filename=Path(file.name).name,
                 checksum="",
-                size_in_mb=bytes_to_unit(file.content_size, "mi"),
+                filesize=file.content_size,
                 filetype=file.file_format,
                 path="",
             )

renku/core/dataset/providers/dataverse.py

@@ -20,6 +20,7 @@
 from pathlib import Path
 from typing import Any, List, NamedTuple, Optional, Type
 
+from humanize import naturalsize
 from marshmallow import EXCLUDE
 
 from renku.command.schema.dataset import DatasetSchema
@@ -132,13 +133,16 @@ class ProviderDatasetFile:
     """Store metadata for dataset files that will be downloaded from a provider."""
 
     def __init__(
-        self, source: Optional[str], filename: str, checksum: str, size_in_mb: Optional[float], filetype: str, path: str
+        self, source: Optional[str], filename: str, checksum: str, filesize: Optional[int], filetype: str, path: str
     ):
         self.checksum: str = checksum
         self.filename: str = filename
         self.filetype: str = filetype
         self.path: str = path
-        self.size_in_mb: Optional[float] = size_in_mb
+        self.filesize: Optional[int] = filesize
+        self.filesize_str: Optional[str] = (
+            naturalsize(filesize).upper().replace("BYTES", " B") if filesize is not None else None
+        )
         self.source: Optional[str] = source
 
 

renku/core/dataset/providers/models.py

@@ -33,7 +33,6 @@
 from renku.core.interface.database_dispatcher import IDatabaseDispatcher
 from renku.core.project.project_properties import project_properties
 from renku.core.util import communication
-from renku.core.util.file_size import bytes_to_unit
 from renku.core.util.git import clone_renku_repository, get_cache_directory_for_repository, get_file_size
 from renku.core.util.metadata import is_external_file, make_project_temp_dir
 from renku.core.util.urls import remove_credentials
@@ -523,7 +522,7 @@ def _fetch_dataset(self, client_dispatcher: IClientDispatcher, database_dispatch
                 checksum=file.entity.checksum,
                 filename=Path(file.entity.path).name,
                 filetype=Path(file.entity.path).suffix.replace(".", ""),
-                size_in_mb=bytes_to_unit(get_file_size(self._remote_path, file.entity.path), "mi"),
+                filesize=get_file_size(self._remote_path, file.entity.path),
                 source=file.source,
             )
             for file in dataset.files

renku/core/dataset/providers/renku.py

@@ -30,7 +30,6 @@
 from renku.core.dataset.providers.repository import RepositoryImporter, make_request
 from renku.core.util import communication
 from renku.core.util.doi import is_doi
-from renku.core.util.file_size import bytes_to_unit
 from renku.core.util.urls import remove_credentials
 
 if TYPE_CHECKING:
@@ -219,7 +218,7 @@ def fix_data(self, data, **kwargs):
                 source=file.remote_url.geturl(),
                 filename=Path(file.filename).name,
                 checksum=file.checksum,
-                size_in_mb=bytes_to_unit(file.filesize, "mi"),
+                filesize=file.filesize,
                 filetype=file.type,
                 path="",
             )

renku/core/dataset/providers/zenodo.py

@@ -1405,9 +1405,9 @@ def full_path(self):
         return Path(os.path.abspath(path))
 
     @property
-    def size_in_mb(self):
-        """Return file size in megabytes."""
-        return None if self.filesize is None else self.filesize * 1e-6
+    def filesize(self):
+        """Return file size."""
+        return None if self.filesize is None else self.filesize
 
     def __attrs_post_init__(self):
         """Set the property "name" after initialization."""

renku/core/migration/models/v9.py

@@ -831,7 +831,7 @@ def _clear_line(self):
     return GitProgress()
 
 
-def get_file_size(repository_path: Path, path: str) -> Optional[float]:
+def get_file_size(repository_path: Path, path: str) -> Optional[int]:
     """Return file size for a file inside a git repository."""
     # NOTE: First try to get file size from Git LFS
     try:
@@ -865,11 +865,11 @@ def get_file_size(repository_path: Path, path: str) -> Optional[float]:
             multiplier = conversions.get(unit, None)
             if multiplier is None:
                 continue
-            return size * multiplier
+            return int(size * multiplier)
 
     # Return size of the file on disk
     full_path = repository_path / path
-    return float(os.path.getsize(full_path)) if full_path.exists() else None
+    return os.path.getsize(full_path) if full_path.exists() else None
 
 
 def shorten_message(message: str, line_length: int = 100, body_length: int = 65000) -> str:

renku/core/util/git.py

@@ -378,7 +378,7 @@ def test_dataset_import_renkulab_dataset_with_image(runner, project, client, cli
     assert 0 == result.exit_code, format_result_exception(result)
     assert "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" in result.output
 
-    assert "0.00" in result.output
+    assert "0" in result.output
     assert "OK" in result.output
 
     result = runner.invoke(cli, ["dataset", "ls-files"])