Merge pull request #540 from arabcoders/dev

feat: support using output template for metadata generation.

Author: arabcoders

app/library/Download.py

@@ -189,7 +189,7 @@ def _postprocessor_hook(self, data: dict) -> None:
         dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
         self.status_queue.put({"id": self.id, "action": "postprocessing", **dataDict, "status": "postprocessing"})
 
-    def post_hooks(self, filename: str | None = None) -> None:
+    def _post_hooks(self, filename: str | None = None) -> None:
         if not filename:
             return
 
@@ -227,7 +227,7 @@ def _download(self) -> None:
                 {
                     "progress_hooks": [self._progress_hook],
                     "postprocessor_hooks": [self._postprocessor_hook],
-                    "post_hooks": [self.post_hooks],
+                    "post_hooks": [self._post_hooks],
                 }
             )
 
@@ -330,6 +330,8 @@ def mark_cancelled(*_) -> None:
 
                 signal.signal(signal.SIGUSR1, mark_cancelled)
 
+            self.status_queue.put({"id": self.id, "status": "downloading"})
+
             if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
                 self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
                 _dct: dict = self.info_dict.copy()

app/library/Tasks.py

@@ -77,6 +77,9 @@ def get_ytdlp_opts(self) -> YTDLPOpts:
         if self.cli:
             params = params.add_cli(self.cli, from_user=True)
 
+        if self.template:
+            params = params.add({"outtmpl": {"default": self.template}}, from_user=False)
+
         return params
 
     def mark(self) -> tuple[bool, str]:
@@ -140,7 +143,9 @@ def fetch_metadata(self, full: bool = False) -> tuple[dict[str, Any] | None, boo
 
         params = params.get_all()
 
-        ie_info: dict | None = extract_info(params, self.url, no_archive=True, follow_redirect=False, cache=True)
+        ie_info: dict | None = extract_info(
+            params, self.url, no_archive=True, follow_redirect=False, sanitize_info=True
+        )
         if not ie_info or not isinstance(ie_info, dict):
             return ({}, False, "Failed to extract information from URL.")
 

app/library/Utils.py

@@ -65,7 +65,7 @@
 ]
 "Keys to remove from yt-dlp options at various levels."
 
-YTDLP_INFO_CLS: YTDLP = None
+YTDLP_INFO_CLS: YTDLP | None = None
 "Cached YTDLP info class."
 
 ALLOWED_SUBS_EXTENSIONS: set[str] = {".srt", ".vtt", ".ass"}
@@ -96,6 +96,32 @@ def formatTime(self, record, datefmt=None):  # noqa: ARG002, N802
         return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
 
 
+def get_static_ytdlp(reload: bool = False) -> YTDLP:
+    """
+    Get a static YTDLP instance for info extraction.
+
+    Args:
+        reload (bool): If True, forces re-creation of the instance.
+
+    Returns:
+        YTDLP: A static YTDLP instance.
+
+    """
+    global YTDLP_INFO_CLS  # noqa: PLW0603
+    if YTDLP_INFO_CLS is None or reload:
+        YTDLP_INFO_CLS = YTDLP(
+            params={
+                "color": "no_color",
+                "extract_flat": True,
+                "skip_download": True,
+                "ignoreerrors": True,
+                "ignore_no_formats_error": True,
+                "quiet": True,
+            }
+        )
+    return YTDLP_INFO_CLS
+
+
 def patch_metadataparser() -> None:
     """
     Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues.
@@ -382,7 +408,7 @@ def extract_info(
     if not data["is_premiere"]:
         data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
 
-    return YTDLP.sanitize_info(data) if sanitize_info else data
+    return YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
 
 
 def _is_safe_key(key: any) -> bool:
@@ -1406,27 +1432,13 @@ def get_archive_id(url: str) -> dict[str, str | None]:
         }
 
     """
-    global YTDLP_INFO_CLS  # noqa: PLW0603
-
     idDict: dict[str, None] = {
         "id": None,
         "ie_key": None,
         "archive_id": None,
     }
 
-    if YTDLP_INFO_CLS is None:
-        YTDLP_INFO_CLS = YTDLP(
-            params={
-                "color": "no_color",
-                "extract_flat": True,
-                "skip_download": True,
-                "ignoreerrors": True,
-                "ignore_no_formats_error": True,
-                "quiet": True,
-            }
-        )
-
-    for key, _ie in YTDLP_INFO_CLS._ies.items():
+    for key, _ie in get_static_ytdlp()._ies.items():
         try:
             if not _ie.suitable(url):
                 continue
@@ -1923,3 +1935,18 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
         extras["thumbnail"] = "https://img.youtube.com/vi/{id}/maxresdefault.jpg".format(**entry)
 
     return extras
+
+
+def parse_outtmpl(output_template: str, info_dict: dict) -> str:
+    """
+    Parse yt-dlp output template with given info_dict.
+
+    Args:
+        output_template (str): The output template string.
+        info_dict (dict): The info dictionary from yt-dlp.
+
+    Returns:
+        str: The parsed output string.
+
+    """
+    return get_static_ytdlp().prepare_filename(info_dict=info_dict, outtmpl=output_template)

app/routes/api/tasks.py

@@ -1,3 +1,5 @@
+import asyncio
+import functools
 import logging
 import uuid
 from typing import TYPE_CHECKING, Any
@@ -10,7 +12,7 @@
 from app.library.encoder import Encoder
 from app.library.router import route
 from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks
-from app.library.Utils import get_channel_images, get_file, init_class, validate_url, validate_uuid
+from app.library.Utils import get_channel_images, get_file, init_class, parse_outtmpl, validate_url, validate_uuid
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -252,10 +254,38 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
         if not save_path.exists():
             save_path.mkdir(parents=True, exist_ok=True)
 
-        metadata, status, message = task.fetch_metadata(full=False)
+        metadata, status, message = await asyncio.wait_for(
+            fut=asyncio.get_running_loop().run_in_executor(
+                None,
+                functools.partial(task.fetch_metadata, full=False),
+            ),
+            timeout=120,
+        )
         if not status:
             return web.json_response(data={"error": message}, status=web.HTTPBadRequest.status_code)
 
+        if not task.folder:
+            try:
+                outtmpl = parse_outtmpl(
+                    output_template=task.get_ytdlp_opts().get_all().get("outtmpl", {}).get("default", "{title} [{id}]"),
+                    info_dict=metadata,
+                )
+                if outtmpl:
+                    _path = save_path / outtmpl
+                    if not _path.is_dir():
+                        _path = _path.parent
+
+                    (save_path, _) = get_file(config.download_path, _path.relative_to(config.download_path))
+                    if not str(save_path or "").startswith(str(config.download_path)):
+                        return web.json_response(
+                            data={"error": "Invalid final path folder."}, status=web.HTTPBadRequest.status_code
+                        )
+
+                    if not save_path.exists():
+                        save_path.mkdir(parents=True, exist_ok=True)
+            except Exception as e:
+                LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'")
+
         info = {
             "id": ag(metadata, ["id", "channel_id"]),
             "id_type": metadata.get("extractor", "").split(":")[0].lower() if metadata.get("extractor") else None,
@@ -277,12 +307,12 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
         from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
 
         title: str = sanitize_filename(info.get("title"))
-        filename: Path = save_path / f"{title} [{info.get('id')}].info.json"
-        filename.write_text(encoder.encode(metadata), encoding="utf-8")
-        info["json_file"] = str(filename.relative_to(config.download_path))
+        info_file: Path = save_path / f"{title} [{info.get('id')}].info.json"
+        info_file.write_text(encoder.encode(metadata), encoding="utf-8")
+        info["json_file"] = str(info_file.relative_to(config.download_path))
 
-        filename = save_path / "tvshow.nfo"
-        info["nfo_file"] = f"{save_path}/{filename}"
+        xml_file: Path = save_path / "tvshow.nfo"
+        info["nfo_file"] = str(xml_file.relative_to(config.download_path))
 
         xml_content = "<tvshow>\n"
         xml_content += f"  <title>{NFOMakerPP._escape_text(info.get('title'))}</title>\n"
@@ -303,8 +333,7 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
             xml_content += f"  <year>{info.get('year')}</year>\n"
         xml_content += "  <status>Continuing</status>\n"
         xml_content += "</tvshow>\n"
-
-        filename.write_text(xml_content, encoding="utf-8")
+        xml_file.write_text(xml_content, encoding="utf-8")
 
         try:
             from yt_dlp.utils.networking import random_user_agent
@@ -348,9 +377,9 @@ async def task_metadata(request: Request, config: Config, encoder: Encoder) -> R
 
                         resp = await client.request(method="GET", url=url, follow_redirects=True)
 
-                        filename = save_path / f"{key}.jpg"
-                        filename.write_bytes(resp.content)
-                        info["thumbnails"][key] = str(filename.relative_to(config.download_path))
+                        img_file = save_path / f"{key}.jpg"
+                        img_file.write_bytes(resp.content)
+                        info["thumbnails"][key] = str(img_file.relative_to(config.download_path))
                     except Exception as e:
                         LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url}'. '{e!s}'")
                         continue

app/tests/test_download.py

@@ -152,9 +152,9 @@ def test_post_hooks_pushes_filename(self) -> None:
         d = Download(make_item())
         q = DummyQueue()
         d.status_queue = q
-        d.post_hooks(None)
+        d._post_hooks(None)
         assert len(q.items) == 0
-        d.post_hooks("name.ext")
+        d._post_hooks("name.ext")
         assert len(q.items) == 1
         assert q.items[0]["filename"] == "name.ext"