Merge pull request #1975 from wger-project/feature/ingredient-sync-log

Improve logging for ingredient sync operations

Author: rolandgeider

wger/nutrition/management/commands/sync-ingredients.py

@@ -50,11 +50,16 @@ def handle(self, **options):
 
         check_min_server_version(remote_url)
         try:
-            self.languages = languages
-            if self.languages is not None:
-                for language in self.languages.split(','):
+            if languages is not None:
+                for language in languages.split(','):
                     validate_language_code(language)
         except ValidationError as e:
             raise CommandError('\n'.join([str(arg) for arg in e.args if arg is not None]))
 
-        sync_ingredients(self.stdout.write, self.remote_url, self.languages, self.style.SUCCESS)
+        sync_ingredients(
+            self.stdout.write,
+            self.remote_url,
+            languages,
+            self.style.SUCCESS,
+            show_progress_bar=True
+        )

wger/nutrition/sync.py

@@ -31,6 +31,7 @@
     AWS_S3_BASE_URL,
     generate_image_path,
 )
+from tqdm import tqdm
 
 # wger
 from wger.core.models.language import Language
@@ -247,54 +248,71 @@ def sync_ingredients(
     remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],
     language_codes: Optional[str] = None,
     style_fn=lambda x: x,
+    show_progress_bar: bool = False,
 ):
     """Synchronize the ingredients from the remote server"""
-
-    def _sync_ingredients(language_codes: Optional[int] = None):
-        if language_codes is not None:
-            url = make_uri(
-                INGREDIENTS_ENDPOINT,
-                server_url=remote_url,
-                query={'limit': API_MAX_ITEMS, 'language__in': language_codes},
-            )
-        else:
-            url = make_uri(
-                INGREDIENTS_ENDPOINT,
-                server_url=remote_url,
-                query={'limit': API_MAX_ITEMS},
-            )
-        for data in get_paginated(url, headers=wger_headers()):
-            uuid = data['uuid']
-            name = data['name']
-
-            ingredient_data = extract_info_from_wger_api(data).dict()
-            ingredient_data['uuid'] = uuid
-
-            ingredient, created = Ingredient.objects.update_or_create(
-                uuid=uuid, defaults=ingredient_data
-            )
-
-            print_fn(f'{"created" if created else "updated"} ingredient {uuid} - {name}')
-
     print_fn('*** Synchronizing ingredients...')
 
+    language_ids: List[str] | None = None
     if language_codes is not None:
-        language_ids: List[int] = []
+        language_ids = []
         for code in language_codes.split(','):
             # Leaving the try except in here even though we've already validated on the sync-ingredients command itself.
             # This is in case we ever want to re-use this function for anything else where user can input language codes.
             try:
                 lang = load_language(code, default_to_english=False)
-                language_ids.append(lang.id)
+                language_ids.append(str(lang.id))
             except Language.DoesNotExist as e:
                 print_fn(
                     f'Error: The language code you provided ("{code}") does not exist in this database. Please try again.'
                 )
                 return 0
 
-        for language_id in language_ids:
-            _sync_ingredients(language_id)
-    else:
-        _sync_ingredients()
+    query: dict[str, str | int] = {'limit': API_MAX_ITEMS}
+    if language_ids is not None:
+        query['language__in'] = ','.join(language_ids)
+
+    url = make_uri(
+        INGREDIENTS_ENDPOINT,
+        server_url=remote_url,
+        query=query,
+    )
+
+    # Fetch once to retrieve the number of results
+    response = requests.get(url, headers=wger_headers()).json()
+    total_ingredients = response['count']
+    total_pages = total_ingredients // API_MAX_ITEMS
+
+    ingredient_nr = 1
+    page_nr = 1
+    pbar = tqdm(
+        total=total_ingredients,
+        unit='ingredients',
+        desc='Syncing progress',
+        unit_scale=True
+    )
+    for data in get_paginated(url, headers=wger_headers()):
+        uuid = data['uuid']
+
+        ingredient_data = extract_info_from_wger_api(data).dict()
+        ingredient_data['uuid'] = uuid
+
+        Ingredient.objects.update_or_create(
+            uuid=uuid,
+            defaults=ingredient_data
+        )
+
+        if show_progress_bar:
+            pbar.update(1)
+        else:
+            # Note that get_paginated returns the individual result entries from the pages.
+            # To get the current page, we need to calculate this ourselves.
+            ingredient_nr += 1
+            if ingredient_nr % API_MAX_ITEMS == 0:
+                page_nr += 1
+                print_fn(f'Processing ingredients, page {page_nr: >4} of {total_pages}')
+
+    pbar.close()
 
     print_fn(style_fn('done!\n'))
+    return None

wger/nutrition/tasks.py

@@ -19,13 +19,12 @@
     randint,
 )
 
-# Django
-from django.conf import settings
-from django.core.management import call_command
-
 # Third Party
 from celery import shared_task
 from celery.schedules import crontab
+# Django
+from django.conf import settings
+from django.core.management import call_command
 
 # wger
 from wger.celery_configuration import app
@@ -35,7 +34,6 @@
     sync_ingredients,
 )
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -64,7 +62,10 @@ def sync_all_ingredients_task():
     """
     Fetches the current ingredients from the default wger instance
     """
-    sync_ingredients(logger.info)
+    sync_ingredients(
+        logger.info,
+        show_progress_bar=False
+    )
 
 
 @app.task