Deprecate web-ui repo (#2463)

This removed the need for a second repo for the web content by making
all HTML SSG using Jinja and Datastar.

The old API routes are not being used now but I left them in place since
they are quite useful.

After merging and releasing, the old https://github.com/spotDL/web-ui/
can be archived.

Author: Silverarmor

.vscode/settings.json

@@ -1,13 +1,7 @@
 {
-    "cSpell.words": [
-        "isrc",
-        "pdoc",
-        "spotdl"
-    ],
+    "cSpell.words": ["datastar", "isrc", "pdoc", "spotdl"],
     "python.formatting.provider": "black",
-    "python.testing.pytestArgs": [
-        "tests"
-    ],
+    "python.testing.pytestArgs": ["tests"],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true
-}
+}

docs/usage.md

@@ -338,9 +338,6 @@ If you don't want the config to load automatically, change the `load_config` opt
     "ca_file": null,
     "allowed_origins": null,
     "keep_sessions": false,
-    "force_update_gui": false,
-    "web_gui_repo": null,
-    "web_gui_location": null
 }
 ```
 
@@ -435,7 +432,7 @@ Spotify options:
   --max-retries MAX_RETRIES
                         The maximum number of retries to perform when getting metadata.
   --headless            Run in headless mode.
-  --use-cache-file      Use the cache file to get metadata. It's located under C:\Users\user\.spotdl\.spotify_cache or ~/.spotdl/.spotify_cache under linux. It only caches tracks and
+  --use-cache-file      Use the cache file to get metadata. It's located under C:\Users\<user>\.spotdl\.spotify_cache or ~/.spotdl/.spotify_cache under linux. It only caches tracks and
                         gets updated whenever spotDL gets metadata from Spotify. (It may provide outdated metadata use with caution)
 
 FFmpeg options:
@@ -510,11 +507,6 @@ Web options:
                         The allowed origins for the web server.
   --web-use-output-dir  Use the output directory instead of the session directory for downloads. (This might cause issues if you have multiple users using the web-ui at the same time)
   --keep-sessions       Keep the session directory after the web server is closed.
-  --force-update-gui    Refresh the web server directory with a fresh git checkout
-  --web-gui-repo WEB_GUI_REPO
-                        Custom web gui repo to use for the web server. Example: https://github.com/spotdl/web-ui/tree/master/dist
-  --web-gui-location WEB_GUI_LOCATION
-                        Path to the web gui directory to use for the web server.
   --enable-tls          Enable TLS on the web server.
   --cert-file CERT_FILE
                         File Path to the TLS Certificate (PEM format).

pyproject.toml

@@ -46,6 +46,9 @@ dependencies = [
     "syncedlyrics>=1.0.1,<2",
     "soundcloud-v2>=1.6.0,<2",
     "websockets~=14.1",
+    "datastar-py>=0.6.3",
+    "jinja2>=3.1.5",
+    "python-multipart>=0.0.20",
 ]
 
 [project.urls]

spotdl/console/web.py

@@ -9,7 +9,6 @@
 import signal
 import sys
 import webbrowser
-from pathlib import Path
 
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -26,8 +25,8 @@
     app_state,
     fix_mime_types,
     get_current_state,
-    router,
 )
+from spotdl.web import api, routes
 
 __all__ = ["web"]
 
@@ -66,41 +65,15 @@ def web(web_settings: WebOptions, downloader_settings: DownloaderOptions):
 
     downloader_settings["simple_tui"] = True
 
-    # Download web app from GitHub if not already downloaded or force flag set
-    web_app_dir = get_web_ui_path()
-    dist_dir = web_app_dir / "dist"
-    if (not dist_dir.exists() or web_settings["force_update_gui"]) and web_settings[
-        "web_gui_location"
-    ] is None:
-        if web_settings["web_gui_repo"] is None:
-            gui_repo = "https://github.com/spotdl/web-ui/tree/master/dist"
-        else:
-            gui_repo = web_settings["web_gui_repo"]
-
-        logger.info("Updating web app from %s", gui_repo)
-
-        download_github_dir(
-            gui_repo,
-            output_dir=str(web_app_dir),
-        )
-        web_app_dir = Path(os.path.join(web_app_dir, "dist")).resolve()
-    elif web_settings["web_gui_location"]:
-        web_app_dir = Path(web_settings["web_gui_location"]).resolve()
-        logger.info("Using custom web app location: %s", web_app_dir)
-    else:
-        logger.info(
-            "Using cached web app. To update use the `--force-update-gui` flag."
-        )
-        web_app_dir = Path(os.path.join(web_app_dir, "dist")).resolve()
-
     app_state.api = FastAPI(
         title="spotDL",
         description="Download music from Spotify",
         version=__version__,
         dependencies=[Depends(get_current_state)],
     )
 
-    app_state.api.include_router(router)
+    app_state.api.include_router(api.router)
+    app_state.api.include_router(routes.router)
 
     # Add the CORS middleware
     app_state.api.add_middleware(
@@ -116,17 +89,18 @@ def web(web_settings: WebOptions, downloader_settings: DownloaderOptions):
     )
 
     # Add the static files
+    web_app_dir = get_web_ui_path()
     app_state.api.mount(
-        "/",
-        SPAStaticFiles(directory=web_app_dir, html=True),
+        "/assets",
+        SPAStaticFiles(directory=web_app_dir / "assets", html=True),
         name="static",
     )
     protocol = "http"
     config = Config(
         app=app_state.api,
         host=web_settings["host"],
         port=web_settings["port"],
-        workers=1,
+        # workers=1,
         log_level=NAME_TO_LEVEL[downloader_settings["log_level"]],
         loop=app_state.loop,  # type: ignore
     )

spotdl/download/downloader.py

@@ -838,6 +838,7 @@ def search_and_download(  # pylint: disable=R0911
                 generate_lrc(song, output_file)
 
             display_progress_tracker.notify_complete()
+            display_progress_tracker.set_path(str(output_file))
 
             # Add the song to the known songs
             self.known_songs.get(song.url, []).append(output_file)

spotdl/download/progress_handler.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional
 
 from rich import get_console
@@ -99,6 +100,64 @@ def render(self, task: Task) -> Text:
         return text
 
 
+@dataclass
+class ClientSongDownload:
+    """
+    Represents the download progress of a single song for a client.
+    """
+
+    song: Song
+    progress: int
+    message: str
+    path: Optional[str] = None
+
+
+class ProgressTracker:
+    """
+    Tracks the progress of each song download.
+    Similar to Rich Progress but without any TUI elements.
+    """
+
+    songs: Dict[str, ClientSongDownload] = {}
+
+    def add(self, song: Song):
+        """
+        Add a song to the progress tracker.
+        """
+        # check if exists
+        if song.url in self.songs:
+            return
+        self.songs[song.url] = ClientSongDownload(
+            song=song, progress=0, message="Processing"
+        )
+
+    def update(self, song: Song, progress: int, message: str):
+        """
+        Update the progress of a song in the progress tracker.
+        """
+        if song.url in self.songs:
+            self.songs[song.url].progress = progress
+            self.songs[song.url].message = message
+        else:
+            self.songs[song.url] = ClientSongDownload(
+                song=song, progress=progress, message=message
+            )
+
+    def remove(self, song: Song):
+        """
+        Remove a song from the progress tracker.
+        """
+        if song.url in self.songs:
+            del self.songs[song.url]
+
+    def set_path(self, song: Song, path: str):
+        """
+        Set the download path for a song.
+        """
+        if song.url in self.songs:
+            self.songs[song.url].path = path
+
+
 class ProgressHandler:
     """
     Class for handling the progress of a download, including the progress bar.
@@ -131,6 +190,8 @@ def __init__(
         self.quiet = logger.getEffectiveLevel() < 10
         self.overall_task_id: Optional[TaskID] = None
 
+        self.progress_tracker = ProgressTracker()
+
         if not self.simple_tui:
             console = get_console()
 
@@ -274,7 +335,9 @@ def __init__(self, parent, song: Song) -> None:
         self.progress: int = 0
         self.old_progress: int = 0
         self.status = ""
+        self.path: Optional[str] = None
 
+        self.parent.progress_tracker.add(self.song)
         if not self.parent.simple_tui:
             self.task_id = self.parent.rich_progress_bar.add_task(
                 description=escape(self.song_name),
@@ -299,9 +362,14 @@ def update(self, message=""):
         # The change in progress since last update
         delta = self.progress - self.old_progress
 
+        self.parent.progress_tracker.update(self.song, self.progress, message)
+        if self.progress == 100 or message == "Error":
+            if not self.parent.web_ui:
+                self.parent.progress_tracker.remove(self.song)
+
         if not self.parent.simple_tui:
             # Update the progress bar
-            # `start_task` called everytime to ensure progress is remove from indeterminate state
+            # `start_task` called every time to ensure progress is remove from indeterminate state
             self.parent.rich_progress_bar.start_task(self.task_id)
             self.parent.rich_progress_bar.update(
                 self.task_id,
@@ -310,6 +378,9 @@ def update(self, message=""):
                 completed=self.progress,
             )
 
+            # Refresh the progress bar to show the changes before it gets removed in case of 100%
+            self.parent.rich_progress_bar.refresh()
+
             # If task is complete
             if self.progress == 100 or message == "Error":
                 self.parent.overall_completed_tasks += 1
@@ -438,3 +509,14 @@ def yt_dlp_progress_hook(self, data: Dict[str, Any]) -> None:
                 self.progress = downloaded_bytes / file_bytes * 50
 
             self.update("Downloading")
+
+    def set_path(self, path: str) -> None:
+        """
+        Sets the path of the song.
+
+        ### Arguments
+        - path: The path to set.
+        """
+
+        self.path = path
+        self.parent.progress_tracker.set_path(self.song, path)

spotdl/types/options.py

@@ -106,9 +106,6 @@ class WebOptions(TypedDict):
     ca_file: Optional[str]
     allowed_origins: Optional[List[str]]
     keep_sessions: bool
-    force_update_gui: bool
-    web_gui_repo: Optional[str]
-    web_gui_location: Optional[str]
 
 
 class SpotDLOptions(SpotifyOptions, DownloaderOptions, WebOptions):
@@ -202,9 +199,6 @@ class WebOptionalOptions(TypedDict, total=False):
     ca_file: Optional[str]
     allowed_origins: Optional[str]
     keep_sessions: bool
-    force_update_gui: bool
-    web_gui_repo: Optional[str]
-    web_gui_location: Optional[str]
 
 
 class SpotDLOptionalOptions(

spotdl/types/song.py

@@ -278,6 +278,15 @@ def json(self) -> Dict[str, Any]:
 
         return asdict(self)
 
+    # def __json__(self) -> str:
+    #     """
+    #     Returns a JSON string of the song's data.
+
+    #     ### Returns
+    #     - The JSON string.
+    #     """
+    #     return json.dumps(self.json)
+
 
 @dataclass(frozen=True)
 class SongList:

spotdl/utils/arguments.py

@@ -684,32 +684,6 @@ def parse_web_options(parser: _ArgumentGroup):
         help="Keep the session directory after the web server is closed.",
     )
 
-    # Add keep sessions argument
-    parser.add_argument(
-        "--force-update-gui",
-        action="store_const",
-        const=True,
-        default=False,
-        help="Refresh the web server directory with a fresh git checkout",
-    )
-
-    # Add custom web gui repo
-    parser.add_argument(
-        "--web-gui-repo",
-        type=str,
-        help=(
-            "Custom web gui repo to use for the web server. "
-            "Example: https://github.com/spotdl/web-ui/tree/master/dist"
-        ),
-    )
-
-    # Add custom web gui repo
-    parser.add_argument(
-        "--web-gui-location",
-        type=str,
-        help="Path to the web gui directory to use for the web server.",
-    )
-
     # Enable TLS for the web server
     parser.add_argument(
         "--enable-tls",

spotdl/utils/config.py

@@ -162,7 +162,10 @@ def get_web_ui_path() -> Path:
     - If the web-ui directory does not exist, it will be created.
     """
 
-    web_ui_path = get_spotdl_path() / "web-ui"
+    # web_ui_path = get_spotdl_path() / "web-ui"
+    # web_ui_path = get_spotdl_path() / "src" / "spotdl" / "web" / "static"
+    web_ui_path = Path(__file__).parent.parent / "web" / "static"
+    print("Web UI path:", web_ui_path)
 
     if not web_ui_path.exists():
         os.mkdir(web_ui_path)
@@ -378,9 +381,6 @@ def get_parameter(cls, key):
     "ca_file": None,
     "allowed_origins": None,
     "keep_sessions": False,
-    "force_update_gui": False,
-    "web_gui_repo": None,
-    "web_gui_location": None,
 }
 
 # Type: ignore because of the issues above