Improve Source.index_media function

tubesync/sync/models/source.py

@@ -1,6 +1,7 @@
 import os
 import re
 import uuid
+from collections import deque as queue
 from pathlib import Path
 from django import db
 from django.conf import settings
@@ -527,23 +528,31 @@ def get_index(self, type):
             days = timezone.timedelta(seconds=self.download_cap).days
         response = indexer(self.get_index_url(type=type), days=days)
         if not isinstance(response, dict):
-            return []
-        entries = response.get('entries', []) 
+            return list()
+        entries = response.get('entries', list()) 
         return entries
 
     def index_media(self):
         '''
-            Index the media source returning a list of media metadata as dicts.
+            Index the media source returning a queue of media metadata as dicts.
         '''
-        entries = list()
+        entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None)
         if self.index_videos:
-            entries += self.get_index('videos')
+            entries.extend(reversed(self.get_index('videos')))
+
         # Playlists do something different that I have yet to figure out
         if not self.is_playlist:
             if self.index_streams:
-                entries += self.get_index('streams')
+                streams = self.get_index('streams')
+                if entries.maxlen is None or 0 == len(entries):
+                    entries.extend(reversed(streams))
+                else:
+                    # share the queue between streams and videos
+                    allowed_streams = max(
+                        entries.maxlen // 2,
+                        entries.maxlen - len(entries),
+                    )
+                    entries.extend(reversed(streams[-1 * allowed_streams :]))
 
-        if settings.MAX_ENTRIES_PROCESSING:
-            entries = entries[:settings.MAX_ENTRIES_PROCESSING]
         return entries
 

tubesync/sync/tasks.py

@@ -255,7 +255,7 @@ def cleanup_old_media():
     schedule_media_servers_update()
 
 
-def cleanup_removed_media(source, videos):
+def cleanup_removed_media(source, video_keys):
     if not source.delete_removed_media:
         return
     log.info(f'Cleaning up media no longer in source: {source}')
@@ -265,8 +265,7 @@ def cleanup_removed_media(source, videos):
         source=source,
     )
     for media in qs_gen(mqs):
-        matching_source_item = [video['id'] for video in videos if video['id'] == media.key]
-        if not matching_source_item:
+        if media.key not in video_keys:
             log.info(f'{media.name} is no longer in source, removing')
             with atomic():
                 media.delete()
@@ -316,12 +315,17 @@ def index_source_task(source_id):
             end=task.verbose_name.find('Index'),
         )
     tvn_format = '{:,}' + f'/{num_videos:,}'
-    for vn, video in enumerate(videos, start=1):
+    vn = 0
+    video_keys = set()
+    while len(videos) > 0:
+        vn += 1
+        video = videos.popleft()
         # Create or update each video as a Media object
         key = video.get(source.key_field, None)
         if not key:
             # Video has no unique key (ID), it can't be indexed
             continue
+        video_keys.add(key)
         update_task_status(task, tvn_format.format(vn))
         # media, new_media = Media.objects.get_or_create(key=key, source=source)
         try:
@@ -376,7 +380,7 @@ def index_source_task(source_id):
     # Reset task.verbose_name to the saved value
     update_task_status(task, None)
     # Cleanup of media no longer available from the source
-    cleanup_removed_media(source, videos)
+    cleanup_removed_media(source, video_keys)
     videos = video = None
 
 

tubesync/sync/youtube.py

@@ -326,13 +326,10 @@ def download_media(
             final_path = Path(output_file).resolve(strict=False)
         expected_file = shell_quote(str(final_path))
         cmds = pp_opts.exec_cmd.get('after_move', list())
-        # It is important that we use a tuple for strings.
-        # Otherwise, list adds each character instead.
-        # That last comma is really necessary!
-        cmds += (
+        cmds.append(
             f'test -f {expected_file} || '
             'mv -T -u -- %(filepath,_filename|)q '
-            f'{expected_file}',
+            f'{expected_file}'
         )
         # assignment is the quickest way to cover both 'get' cases
         pp_opts.exec_cmd['after_move'] = cmds
@@ -387,7 +384,7 @@ def download_media(
     youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict())
     formats_list = youtube_ea_dict.get('formats', list())
     if 'missing_pot' not in formats_list:
-        formats_list += ('missing_pot',)
+        formats_list.append('missing_pot')
         youtube_ea_dict.update({
             'formats': formats_list,
         })