feat: use repo as source of truth (#317)

Author: sdn4z

README.md

@@ -6,6 +6,7 @@
 [![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue?logo=python&logoColor=yellow)](https://pypi.org/project/twyn/)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![License](https://img.shields.io/github/license/elementsinteractive/twyn)](LICENSE)
+
 ## Table of Contents
 
 - [Overview](#overview)
@@ -171,18 +172,14 @@ allowlist=["my_package"]
 source="https://mirror-with-trusted-dependencies.com/file.json"
 ```
 
-> [!WARNING]
-> `twyn` will have a default reference URL for every source of trusted packages that is configurable.
-> If you want to protect yourself against spoofing attacks, it is recommended to set your own
-> reference url.
-
 The file format for each reference is as follows:
 
-- **PyPI reference**:
-
-```ts
+```jsonc
 {
-    rows: {project: string}[]
+  "date": "string (ISO 8601 format, e.g. 2025-09-10T14:23:00+00)",
+  "packages": [
+    { "name": "string" }
+  ]
 }
 ```
 

src/twyn/trusted_packages/references/base.py

@@ -7,6 +7,7 @@
 
 from twyn.trusted_packages.cache_handler import CacheEntry, CacheHandler
 from twyn.trusted_packages.exceptions import (
+    EmptyPackagesListError,
     InvalidJSONError,
 )
 
@@ -28,26 +29,19 @@ def __init__(self, source: Optional[str] = None, cache_handler: Union[CacheHandl
         self.source = source or self.DEFAULT_SOURCE
         self.cache_handler = cache_handler
 
-    @staticmethod
-    @abstractmethod
-    def _parse(packages_json: dict[str, Any]) -> set[str]:
-        """Parse and retrieve the packages within the given json structure."""
-
     @staticmethod
     @abstractmethod
     def normalize_packages(packages: set[str]) -> set[str]:
         """Normalize package names to make sure they're valid within the package manager context."""
 
     def _download(self) -> dict[str, Any]:
-        packages = requests.get(self.source)
-        packages.raise_for_status()
+        response = requests.get(self.source)
+        response.raise_for_status()
+
         try:
-            packages_json: dict[str, Any] = packages.json()
+            return response.json()
         except requests.exceptions.JSONDecodeError as err:
             raise InvalidJSONError from err
-        else:
-            logger.debug("Successfully downloaded trusted packages list from %s", self.source)
-            return packages_json
 
     def _save_trusted_packages_to_cache_if_enabled(self, packages: set[str]) -> None:
         """Save trusted packages using CacheHandler."""
@@ -69,18 +63,24 @@ def _get_packages_from_cache_if_enabled(self) -> set[str]:
         return cache_entry.packages
 
     def get_packages(self) -> set[str]:
-        """Download and parse online source of top Python Package Index packages."""
-        packages_to_use = set()
-        packages_to_use = self._get_packages_from_cache_if_enabled()
+        """Download and parse online source of top packages from the package ecosystem."""
+        packages = self._get_packages_from_cache_if_enabled()
         # we don't save the cache here, we keep it as it is so the date remains the original one.
-
-        if not packages_to_use:
+        if not packages:
             # no cache usage, no cache hit (non-existent or outdated) or cache was empty.
             logger.info("Fetching trusted packages from trusted packages reference...")
-            packages_to_use = self._parse(self._download())
+            data = self._download()
+            try:
+                packages = set(data["packages"])
+            except KeyError as err:
+                raise InvalidJSONError("`packages` key not in JSON.") from err
+
+            logger.debug("Successfully downloaded trusted packages list from %s", self.source)
+            if not packages:
+                raise EmptyPackagesListError
 
             # New packages were downloaded, we create a new entry updating all values.
-            self._save_trusted_packages_to_cache_if_enabled(packages_to_use)
+            self._save_trusted_packages_to_cache_if_enabled(packages)
 
-        normalized_packages = self.normalize_packages(packages_to_use)
+        normalized_packages = self.normalize_packages(packages)
         return normalized_packages

src/twyn/trusted_packages/references/top_npm_reference.py

@@ -1,12 +1,9 @@
 import logging
 import re
-from typing import Any
 
 from typing_extensions import override
 
 from twyn.trusted_packages.exceptions import (
-    EmptyPackagesListError,
-    InvalidReferenceFormatError,
     PackageNormalizingError,
 )
 from twyn.trusted_packages.references.base import AbstractPackageReference
@@ -17,22 +14,9 @@
 class TopNpmReference(AbstractPackageReference):
     """Top npm packages retrieved from an online source."""
 
-    DEFAULT_SOURCE: str = "https://www.npmleaderboard.org/api/packages"
-
-    @override
-    @staticmethod
-    def _parse(packages_info: dict[str, Any]) -> set[str]:
-        try:
-            names = {pkg["name"] for pkg in packages_info["packages"]}
-
-        except KeyError as err:
-            raise InvalidReferenceFormatError from err
-
-        if not names:
-            raise EmptyPackagesListError
-
-        logger.debug("Successfully parsed trusted packages list")
-        return names
+    DEFAULT_SOURCE: str = (
+        "https://raw.githubusercontent.com/elementsinteractive/twyn/refs/heads/main/dependencies/npm.json"
+    )
 
     @override
     @staticmethod

src/twyn/trusted_packages/references/top_pypi_reference.py

@@ -1,12 +1,9 @@
 import logging
 import re
-from typing import Any
 
 from typing_extensions import override
 
 from twyn.trusted_packages.exceptions import (
-    EmptyPackagesListError,
-    InvalidReferenceFormatError,
     PackageNormalizingError,
 )
 from twyn.trusted_packages.references.base import AbstractPackageReference
@@ -17,21 +14,9 @@
 class TopPyPiReference(AbstractPackageReference):
     """Top PyPi packages retrieved from an online source."""
 
-    DEFAULT_SOURCE: str = "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json"
-
-    @override
-    @staticmethod
-    def _parse(packages_info: dict[str, Any]) -> set[str]:
-        try:
-            names = {row["project"] for row in packages_info["rows"]}
-        except KeyError as err:
-            raise InvalidReferenceFormatError from err
-
-        if not names:
-            raise EmptyPackagesListError
-
-        logger.debug("Successfully parsed trusted packages list")
-        return names
+    DEFAULT_SOURCE: str = (
+        "https://raw.githubusercontent.com/elementsinteractive/twyn/refs/heads/main/dependencies/pypi.json"
+    )
 
     @override
     @staticmethod

tests/conftest.py

@@ -1,4 +1,5 @@
-from collections.abc import Iterable, Iterator
+import datetime
+from collections.abc import Iterator
 from contextlib import contextmanager
 from pathlib import Path
 from unittest import mock
@@ -14,12 +15,12 @@ def create_tmp_file(path: Path, data: str) -> Iterator[Path]:
 
 
 @contextmanager
-def patch_pypi_packages_download(packages: Iterable[str]) -> Iterator[mock.Mock]:
+def patch_pypi_packages_download(packages: list[str]) -> Iterator[mock.Mock]:
     """Patcher of `requests.get` for Top PyPi list.
 
     Replaces call with the output you would get from downloading the top PyPi packages list.
     """
-    json_response = {"rows": [{"project": name} for name in packages]}
+    json_response = {"packages": packages, "date": datetime.datetime.now().isoformat()}
 
     with mock.patch("twyn.trusted_packages.TopPyPiReference._download") as mock_download:
         mock_download.return_value = json_response
@@ -28,12 +29,12 @@ def patch_pypi_packages_download(packages: Iterable[str]) -> Iterator[mock.Mock]
 
 
 @contextmanager
-def patch_npm_packages_download(packages: Iterable[str]) -> Iterator[mock.Mock]:
+def patch_npm_packages_download(packages: list[str]) -> Iterator[mock.Mock]:
     """Patcher of `requests.get` for Top Npm list.
 
     Replaces call with the output you would get from downloading the top Npm packages list.
     """
-    json_response = {"packages": [{"name": name} for name in packages]}
+    json_response = {"packages": packages, "date": datetime.datetime.now().isoformat()}
 
     with mock.patch("twyn.trusted_packages.TopNpmReference._download") as mock_download:
         mock_download.return_value = json_response

tests/main/test_main.py

@@ -286,32 +286,6 @@ def test_check_dependencies_ignores_package_in_allowlist(
 
         assert error == TyposquatCheckResultList(errors=[])
 
-    @pytest.mark.parametrize(
-        "package_name",
-        [
-            "my.package",
-            "my-package",
-            "my_package",
-            "My.Package",
-        ],
-    )
-    @patch("twyn.trusted_packages.TopPyPiReference._get_packages_from_cache_if_enabled")
-    def test_normalize_package(self, mock_get_packages_from_cache: Mock, package_name: Mock) -> None:
-        mock_get_packages_from_cache.return_value = {"requests", "mypackage"}
-        error = check_dependencies(
-            config_file=None,
-            dependency_file=None,
-            dependencies={package_name},
-            selector_method="first-letter",
-            package_ecosystem="pypi",
-        )
-
-        assert error == TyposquatCheckResultList(
-            errors=[
-                TyposquatCheckResult(dependency="my-package", similars=["mypackage"]),
-            ]
-        )
-
     @patch("twyn.trusted_packages.TopPyPiReference.get_packages")
     def test_check_dependencies_does_not_error_on_same_package(
         self, mock_get_packages: Mock, uv_lock_file_with_typo: Path