bookmarkmgr: Fix URLs with outdated subdomains

Author: lubo

bookmarkmgr/bookmarkmgr/checks/link_status.py

@@ -1,10 +1,14 @@
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable
 from enum import IntEnum, unique
+from functools import partial
 from http import HTTPStatus
 import itertools
 import re
+from typing import Any, Protocol
 from urllib.parse import ParseResult, quote, urlparse
 
+from tld import get_fld
+
 from bookmarkmgr.cronet import RequestError, Response
 from bookmarkmgr.scraper import Page
 
@@ -20,6 +24,12 @@
     HTTPStatus.GONE.value,
 }
 
+_get_fld_lax = partial(
+    get_fld,
+    fail_silently=True,
+    fix_protocol=True,
+)
+
 
 @unique
 class LinkStatus(IntEnum):
@@ -78,21 +88,54 @@ async def check_link_status(
     return link_status, error
 
 
-def _fix_url_quoting(url: ParseResult) -> ParseResult:
+def _fix_url_quoting(
+    url: ParseResult,
+    **_: Any,
+) -> ParseResult:
     return url._replace(path=quote(url.path))
 
 
-def _fix_url_trailing_slash(url: ParseResult) -> ParseResult:
+def _fix_url_subdomain(
+    url: ParseResult,
+    redirect_url: ParseResult,
+) -> ParseResult:
+    if (
+        url.hostname is None
+        or redirect_url.hostname is None
+        or url.hostname == redirect_url.hostname
+    ):
+        return url
+
+    if _get_fld_lax(url.hostname) == _get_fld_lax(redirect_url.hostname):
+        return url._replace(netloc=redirect_url.netloc)
+
+    return url
+
+
+def _fix_url_trailing_slash(
+    url: ParseResult,
+    **_: Any,
+) -> ParseResult:
     return url._replace(
         path=(
             url.path.rstrip("/") if url.path.endswith("/") else f"{url.path}/"
         ),
     )
 
 
-_URL_FIXERS: list[Callable[[ParseResult], ParseResult]] = [
+class _FixerCallable(Protocol):
+    def __call__(
+        self,
+        url: ParseResult,
+        *,
+        redirect_url: ParseResult,
+    ) -> ParseResult: ...
+
+
+_URL_FIXERS: list[_FixerCallable] = [
     _fix_url_quoting,
     _fix_url_trailing_slash,
+    _fix_url_subdomain,
 ]
 
 
@@ -113,6 +156,7 @@ def get_fixed_url(response: Response, url: str) -> None | str:
             for fixer in fixer_combination:
                 fixed_parsed_url = fixer(
                     fixed_parsed_url,
+                    redirect_url=parsed_redirect_url,
                 )
 
             if fixed_parsed_url == parsed_redirect_url:

bookmarkmgr/poetry.lock

@@ -37,6 +37,7 @@ cffi = "^1.17.1"
 enlighten = "^1.13.0"
 overrides = "^7.7.0"
 python = "^3.12"
+tld = "^0.13"
 yarl = "^1.18.3"
 
 [tool.poetry.group.dev.dependencies]