CLI: verdi storage maintain: show a progress bar (aiidateam#6562)

`verdi storage maintain` now  creates and sends over a progress bar to disk-objectstore.
The logger is added to frontend (cmd_storage.py) to respect the same convention as in `verdi archive import/migrate/create`

Co-authored-by: Alexander Goscinski <[email protected]>

Author: khsrali

environment.yml

@@ -11,7 +11,7 @@ dependencies:
 - circus~=0.18.0
 - click-spinner~=0.1.8
 - click~=8.1
-- disk-objectstore~=1.1
+- disk-objectstore~=1.2
 - docstring_parser
 - get-annotations~=0.1
 - python-graphviz~=0.19

pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
   'circus~=0.18.0',
   'click-spinner~=0.1.8',
   'click~=8.1',
-  'disk-objectstore~=1.1',
+  'disk-objectstore~=1.2',
   'docstring-parser',
   'get-annotations~=0.1;python_version<"3.10"',
   'graphviz~=0.19',

requirements/requirements-py-3.10.txt

@@ -42,7 +42,7 @@ debugpy==1.6.7
 decorator==5.1.1
 defusedxml==0.7.1
 deprecation==2.1.0
-disk-objectstore==1.1.0
+disk-objectstore==1.2.0
 docstring-parser==0.15
 docutils==0.20.1
 exceptiongroup==1.1.1

requirements/requirements-py-3.11.txt

@@ -42,7 +42,7 @@ debugpy==1.6.7
 decorator==5.1.1
 defusedxml==0.7.1
 deprecation==2.1.0
-disk-objectstore==1.1.0
+disk-objectstore==1.2.0
 docstring-parser==0.15
 docutils==0.20.1
 executing==1.2.0

requirements/requirements-py-3.12.txt

@@ -42,7 +42,7 @@ debugpy==1.8.0
 decorator==5.1.1
 defusedxml==0.7.1
 deprecation==2.1.0
-disk-objectstore==1.1.0
+disk-objectstore==1.2.0
 docstring-parser==0.15
 docutils==0.20.1
 executing==2.0.0

requirements/requirements-py-3.9.txt

@@ -42,7 +42,7 @@ debugpy==1.6.7
 decorator==5.1.1
 defusedxml==0.7.1
 deprecation==2.1.0
-disk-objectstore==1.1.0
+disk-objectstore==1.2.0
 docstring-parser==0.15
 docutils==0.20.1
 exceptiongroup==1.2.1

src/aiida/cmdline/commands/cmd_storage.py

@@ -8,6 +8,7 @@
 ###########################################################################
 """`verdi storage` commands."""
 
+import logging
 import sys
 
 import click
@@ -153,7 +154,9 @@ def storage_info(detailed):
 def storage_maintain(ctx, full, no_repack, force, dry_run, compress):
     """Performs maintenance tasks on the repository."""
     from aiida.common.exceptions import LockingProfileError
+    from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter
     from aiida.manage.manager import get_manager
+    from aiida.storage.log import STORAGE_LOGGER
 
     manager = get_manager()
     profile = ctx.obj.profile
@@ -184,6 +187,12 @@ def storage_maintain(ctx, full, no_repack, force, dry_run, compress):
     if not dry_run and not force and not click.confirm('Are you sure you want continue in this mode?'):
         return
 
+    if STORAGE_LOGGER.level <= logging.REPORT:
+        # Only keep the first tqdm bar if it is info. To keep the nested bar information one needs report level
+        set_progress_bar_tqdm(leave=STORAGE_LOGGER.level <= logging.INFO)
+    else:
+        set_progress_reporter(None)
+
     try:
         if full and no_repack:
             storage.maintain(full=full, dry_run=dry_run, do_repack=False, compress=compress)

src/aiida/repository/backend/disk_object_store.py

@@ -167,6 +167,8 @@ def maintain(  # type: ignore[override]
         """
         from disk_objectstore import CompressMode
 
+        from aiida.common.progress_reporter import create_callback, get_progress_reporter
+
         if live and (do_repack or clean_storage or do_vacuum):
             overrides = {'do_repack': do_repack, 'clean_storage': clean_storage, 'do_vacuum': do_vacuum}
             keys = ', '.join([key for key, override in overrides.items() if override is True])  # type: ignore
@@ -192,14 +194,18 @@ def maintain(  # type: ignore[override]
                 files_size = container.get_total_size().total_size_loose * BYTES_TO_MB
                 logger.report(f'Packing all loose files ({files_numb} files occupying {files_size} MB) ...')
                 if not dry_run:
-                    container.pack_all_loose(compress=compress)
+                    with get_progress_reporter()(total=1) as progress:
+                        callback = create_callback(progress)
+                        container.pack_all_loose(compress=compress, callback=callback)
 
             if do_repack:
                 files_numb = container.count_objects().packed
                 files_size = container.get_total_size().total_size_packfiles_on_disk * BYTES_TO_MB
                 logger.report(f'Re-packing all pack files ({files_numb} files in packs, occupying {files_size} MB) ...')
                 if not dry_run:
-                    container.repack()
+                    with get_progress_reporter()(total=1) as progress:
+                        callback = create_callback(progress)
+                        container.repack(callback=callback)
 
             if clean_storage:
                 logger.report(f'Cleaning the repository database (with `vacuum={do_vacuum}`) ...')