Merge pull request #162 from NotJoeMartinez/failed_db_add_large_channel

Failed db add large channel

Author: NotJoeMartinez

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yt-fts"
-version = "0.1.54"
+version = "0.1.55"
 description = "Search all of a YouTube channel from the command line"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -23,11 +23,11 @@ dependencies = [
     "click==8.1.7",
     "openai==1.35.3",
     "chromadb==0.5.2",
-    "requests==2.31.0",
+    "requests>=2.32.2,<3",
     "rich==13.7.1",
     "sqlite-utils==3.36",
     "beautifulsoup4==4.12.3",
-    "yt-dlp==2024.5.27",
+    "yt-dlp==2024.7.16",
     "webvtt-py==0.5.1"
 ]
 

requirements.txt

@@ -148,24 +148,31 @@ def download_vtts(number_of_jobs, video_ids, language, tmp_dir):  # download, up
 
 def quiet_progress_hook(d):  # download
     if d['status'] == 'finished':
-        filename = Path(d['filename']).name
-        print(f" -> {filename}")
+        # TODO: Add verbose flag to show full path
+        print(f" -> {d['filename']}")
+        # filename = Path(d['filename']).name
+        # print(f" -> {filename}")
 
 
 def get_vtt(tmp_dir, video_url, language):  # download
-    ydl_opts = {
-        'outtmpl': f'{tmp_dir}/%(id)s',
-        'writeinfojson': True,
-        'writeautomaticsub': True,
-        'subtitlesformat': 'vtt',
-        'skip_download': True,
-        'subtitleslangs': [language, '-live_chat'],
-        'quiet': True,
-        'progress_hooks': [quiet_progress_hook]
-    }
 
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        ydl.download([video_url])
+    try: 
+        ydl_opts = {
+            'outtmpl': f'{tmp_dir}/%(id)s',
+            'writeinfojson': True,
+            'writeautomaticsub': True,
+            'subtitlesformat': 'vtt',
+            'skip_download': True,
+            'subtitleslangs': [language, '-live_chat'],
+            'quiet': True,
+            'progress_hooks': [quiet_progress_hook]
+        }
+
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            ydl.download([video_url])
+
+    except Exception as e:
+        console.print(f"Failed to get: {video_url}\n{e}")
 
 
 def vtt_to_db(dir_path):  # download, update
@@ -264,6 +271,7 @@ def download_channel(channel_id, channel_name, language, number_of_jobs, s):  #
     import tempfile
     from yt_fts.db_utils import add_channel_info
 
+    # TODO: Allow for partial downloads
     with tempfile.TemporaryDirectory() as tmp_dir:
         channel_url = f"https://www.youtube.com/channel/{channel_id}/videos"
         list_of_videos_urls = get_videos_list(channel_url)
@@ -275,6 +283,7 @@ def download_channel(channel_id, channel_name, language, number_of_jobs, s):  #
     return True
 
 
+# TODO: Allow for partial downloads
 def download_playlist(playlist_url, s, language=None, number_of_jobs=None):  # yt-fts
     """
         Downloads all subtitles from playlist, making new channels where needed

yt_fts/download.py

@@ -26,7 +26,7 @@
     show_message
 )
 
-YT_FTS_VERSION = "0.1.54"
+YT_FTS_VERSION = "0.1.55"
 console = Console()