feat: support multiple dependency files in config and cli

Author: sdn4z

src/twyn/cli.py

@@ -47,6 +47,7 @@ def entry_point() -> None:
 @click.option(
     "--dependency-file",
     type=str,
+    multiple=True,
     help=(
         "Dependency file to analyze. By default, twyn will search in the current directory "
         "for supported files, but this option will override that behavior."
@@ -112,7 +113,7 @@ def entry_point() -> None:
 )
 def run(  # noqa: C901
     config: str,
-    dependency_file: Optional[str],
+    dependency_file: tuple[str],
     dependency: tuple[str],
     selector_method: str,
     v: bool,
@@ -133,15 +134,16 @@ def run(  # noqa: C901
             "Only one of --dependency or --dependency-file can be set at a time.", ctx=click.get_current_context()
         )
 
-    if dependency_file and not any(dependency_file.endswith(key) for key in DEPENDENCY_FILE_MAPPING):
-        raise click.UsageError("Dependency file name not supported.", ctx=click.get_current_context())
+    for dep_file in dependency_file:
+        if dep_file and not any(dep_file.endswith(key) for key in DEPENDENCY_FILE_MAPPING):
+            raise click.UsageError(f"Dependency file name {dep_file} not supported.", ctx=click.get_current_context())
 
     try:
         possible_typos = check_dependencies(
             selector_method=selector_method,
             dependencies=set(dependency) or None,
             config_file=config,
-            dependency_file=dependency_file,
+            dependency_files=set(dependency_file) or None,
             use_cache=not no_cache if no_cache is not None else no_cache,
             show_progress_bar=False if json else not no_track,
             load_config_from_file=True,

src/twyn/config/config_handler.py

@@ -32,7 +32,7 @@
 class TwynConfiguration:
     """Fully resolved configuration for Twyn."""
 
-    dependency_file: Optional[str]
+    dependency_files: set[str]
     selector_method: str
     allowlist: set[str]
     source: Optional[str]
@@ -45,7 +45,7 @@ class TwynConfiguration:
 class ReadTwynConfiguration:
     """Configuration for twyn as set by the user. It may have None values."""
 
-    dependency_file: Optional[str] = None
+    dependency_files: Optional[set[str]] = field(default_factory=set)
     selector_method: Optional[str] = None
     allowlist: set[str] = field(default_factory=set)
     source: Optional[str] = None
@@ -63,7 +63,7 @@ def __init__(self, file_handler: Optional[FileHandler] = None) -> None:
     def resolve_config(
         self,
         selector_method: Optional[str] = None,
-        dependency_file: Optional[str] = None,
+        dependency_files: Optional[set[str]] = None,
         use_cache: Optional[bool] = None,
         package_ecosystem: Optional[PackageEcosystems] = None,
         recursive: Optional[bool] = None,
@@ -107,7 +107,7 @@ def resolve_config(
             final_recursive = DEFAULT_RECURSIVE
 
         return TwynConfiguration(
-            dependency_file=dependency_file or read_config.dependency_file,
+            dependency_files=dependency_files or read_config.dependency_files or set(),
             selector_method=final_selector_method,
             allowlist=read_config.allowlist,
             source=read_config.source,
@@ -141,12 +141,27 @@ def remove_package_from_allowlist(self, package_name: str) -> None:
     def _get_read_config(self, toml: TOMLDocument) -> ReadTwynConfiguration:
         """Read the twyn configuration from a provided toml document."""
         twyn_config_data = toml.get("tool", {}).get("twyn", {})
+
+        dependency_file = twyn_config_data.get("dependency_file", set())
+        if isinstance(dependency_file, str):
+            dependency_file = {dependency_file}
+        elif isinstance(dependency_file, list):
+            dependency_file = set(dependency_file)
+
+        allowlist = twyn_config_data.get("allowlist", set())
+        if isinstance(allowlist, str):
+            allowlist = {allowlist}
+        elif isinstance(allowlist, list):
+            allowlist = set(allowlist)
+
         return ReadTwynConfiguration(
-            dependency_file=twyn_config_data.get("dependency_file"),
+            dependency_files=dependency_file,
             selector_method=twyn_config_data.get("selector_method"),
-            allowlist=set(twyn_config_data.get("allowlist", set())),
+            allowlist=allowlist,
             source=twyn_config_data.get("source"),
             use_cache=twyn_config_data.get("use_cache"),
+            package_ecosystem=twyn_config_data.get("package_ecosystem"),
+            recursive=twyn_config_data.get("recursive"),
         )
 
     def _write_config(self, toml: TOMLDocument, config: ReadTwynConfiguration) -> None:
@@ -155,12 +170,12 @@ def _write_config(self, toml: TOMLDocument, config: ReadTwynConfiguration) -> No
         All null values are simply omitted from the toml file.
         """
         twyn_toml_data = asdict(config, dict_factory=lambda x: _serialize_config(x))
+
         if "tool" not in toml:
             toml.add("tool", table())
         if "twyn" not in toml["tool"]:  # type: ignore[operator]
             toml["tool"]["twyn"] = {}  # type: ignore[index]
         toml["tool"]["twyn"] = twyn_toml_data  # type: ignore[index]
-
         self._write_toml(toml)
 
     def _write_toml(self, toml: TOMLDocument) -> None:

src/twyn/dependency_parser/dependency_selector.py

@@ -12,8 +12,8 @@
 
 
 class DependencySelector:
-    def __init__(self, dependency_file: Optional[str] = None, root_path: str = ".") -> None:
-        self.dependency_file = dependency_file or ""
+    def __init__(self, dependency_files: Optional[set[str]] = None, root_path: str = ".") -> None:
+        self.dependency_files = dependency_files or set()
         self.root_path = root_path
 
     def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
@@ -38,18 +38,18 @@ def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
 
     def get_dependency_file_parsers_from_file_name(self) -> list[AbstractParser]:
         parsers = []
-        for known_dependency_file_name in DEPENDENCY_FILE_MAPPING:
-            if self.dependency_file.endswith(known_dependency_file_name):
-                file_parser = DEPENDENCY_FILE_MAPPING[known_dependency_file_name](self.dependency_file)
-                parsers.append(file_parser)
-
+        for dependency_file in self.dependency_files:
+            for known_dependency_file_name in DEPENDENCY_FILE_MAPPING:
+                if dependency_file.endswith(known_dependency_file_name):
+                    file_parser = DEPENDENCY_FILE_MAPPING[known_dependency_file_name](dependency_file)
+                    parsers.append(file_parser)
         if not parsers:
             raise NoMatchingParserError
 
         return parsers
 
     def get_dependency_parsers(self) -> list[AbstractParser]:
-        if self.dependency_file:
+        if self.dependency_files:
             logger.debug("Dependency file provided. Assigning a parser.")
             return self.get_dependency_file_parsers_from_file_name()
 

src/twyn/main.py

@@ -37,7 +37,7 @@
 def check_dependencies(
     selector_method: Union[SelectorMethod, None] = None,
     config_file: Optional[str] = None,
-    dependency_file: Optional[str] = None,
+    dependency_files: Optional[set[str]] = None,
     dependencies: Optional[set[str]] = None,
     use_cache: Optional[bool] = True,
     show_progress_bar: bool = False,
@@ -68,7 +68,7 @@ def check_dependencies(
         load_config_from_file=load_config_from_file,
         config_file=config_file,
         selector_method=selector_method,
-        dependency_file=dependency_file,
+        dependency_files=dependency_files,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
         recursive=recursive,
@@ -93,7 +93,7 @@ def check_dependencies(
     if config.package_ecosystem:
         logger.warning("`package_ecosystem` is not supported when reading dependencies from files. It will be ignored.")
 
-    if config.dependency_file and config.recursive:
+    if config.dependency_files and config.recursive:
         logger.warning(
             "`--recursive` has been set together with `--dependency-file`. `--dependency-file` will take precedence."
         )
@@ -104,7 +104,7 @@ def check_dependencies(
         maybe_cache_handler=maybe_cache_handler,
         allowlist=config.allowlist,
         show_progress_bar=show_progress_bar,
-        dependency_file=config.dependency_file,
+        dependency_files=config.dependency_files,
     )
 
 
@@ -153,16 +153,17 @@ def _analyze_packages_from_source(
     allowlist: set[str],
     selector_method: SelectorMethod,
     show_progress_bar: bool,
-    dependency_file: Optional[str],
+    dependency_files: Optional[set[str]],
     source: Optional[str],
     maybe_cache_handler: Optional[CacheHandler],
 ) -> TyposquatCheckResults:
     """Analyze dependencies from a dependencies file.
 
     It will return a list of the possible typos grouped by source, each source being a dependency file.
     """
-    dependency_managers = _get_dependency_managers_and_parsers_mapping(dependency_file)
     typos_by_file = TyposquatCheckResults()
+
+    dependency_managers = _get_dependency_managers_and_parsers_mapping(dependency_files)
     for dependency_manager, parsers in dependency_managers.items():
         top_package_reference = dependency_manager.trusted_packages_source(source, maybe_cache_handler)
 
@@ -174,7 +175,6 @@ def _analyze_packages_from_source(
             threshold_class=SimilarityThreshold,
         )
         results: list[TyposquatCheckResultFromSource] = []
-
         for parser in parsers:
             analyzed_dependencies = _analyze_dependencies(
                 top_package_reference, trusted_packages, parser.parse(), allowlist, show_progress_bar, parser.file_path
@@ -252,13 +252,13 @@ def _get_selector_method(selector_method: str) -> SelectorMethod:
 
 
 def _get_dependency_managers_and_parsers_mapping(
-    dependency_file: Optional[str],
+    dependency_files: Optional[set[str]],
 ) -> dict[type[BaseDependencyManager], list[AbstractParser]]:
     """Return a dictionary, grouping all files to parse by their DependencyManager."""
     dependency_managers: dict[type[BaseDependencyManager], list[AbstractParser]] = {}
 
     # No dependencies introduced via the CLI, so the dependecy file was either given or will be auto-detected
-    dependency_selector = DependencySelector(dependency_file)
+    dependency_selector = DependencySelector(dependency_files)
     dependency_parsers = dependency_selector.get_dependency_parsers()
 
     for parser in dependency_parsers:
@@ -274,7 +274,7 @@ def _get_config(
     load_config_from_file: bool,
     config_file: Optional[str],
     selector_method: Union[SelectorMethod, None],
-    dependency_file: Optional[str],
+    dependency_files: Optional[set[str]],
     use_cache: Optional[bool],
     package_ecosystem: Optional[PackageEcosystems],
     recursive: Optional[bool],
@@ -286,7 +286,7 @@ def _get_config(
         config_file_handler = None
     return ConfigHandler(config_file_handler).resolve_config(
         selector_method=selector_method,
-        dependency_file=dependency_file,
+        dependency_files=dependency_files,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
         recursive=recursive,

tests/config/test_config_handler.py

@@ -1,7 +1,6 @@
 import dataclasses
 from copy import deepcopy
 from pathlib import Path
-from typing import NoReturn
 from unittest.mock import Mock, patch
 
 import pytest
@@ -26,17 +25,14 @@
 
 
 class TestConfigHandler:
-    def throw_exception(self) -> NoReturn:
-        raise PathNotFoundError
-
     @patch("twyn.file_handler.file_handler.FileHandler.read")
     def test_no_enforce_file_on_non_existent_file(self, mock_is_file: Mock) -> None:
         """Resolving the config without enforcing the file to be present gives you defaults."""
-        mock_is_file.side_effect = self.throw_exception
+        mock_is_file.side_effect = PathNotFoundError()
         config = ConfigHandler(FileHandler(DEFAULT_PROJECT_TOML_FILE)).resolve_config()
 
         assert config == TwynConfiguration(
-            dependency_file=None,
+            dependency_files=set(),
             selector_method="all",
             allowlist=set(),
             source=None,
@@ -51,7 +47,7 @@ def test_config_raises_for_unknown_file(self) -> None:
 
     def test_read_config_values(self, pyproject_toml_file: Path) -> None:
         config = ConfigHandler(file_handler=FileHandler(pyproject_toml_file)).resolve_config()
-        assert config.dependency_file == "my_file.txt"
+        assert config.dependency_files == {"my_file.txt", "my_other_file.txt"}
         assert config.selector_method == "all"
         assert config.allowlist == {"boto4", "boto2"}
         assert config.use_cache is False
@@ -62,7 +58,7 @@ def test_get_twyn_data_from_file(self, pyproject_toml_file: Path) -> None:
         toml = handler._read_toml()
         twyn_data = ConfigHandler(FileHandler(pyproject_toml_file))._get_read_config(toml)
         assert twyn_data == ReadTwynConfiguration(
-            dependency_file="my_file.txt",
+            dependency_files={"my_file.txt", "my_other_file.txt"},
             selector_method="all",
             allowlist={"boto4", "boto2"},
             source=None,
@@ -75,7 +71,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
 
         initial_config = handler.resolve_config()
         to_write = deepcopy(initial_config)
-        to_write = dataclasses.replace(to_write, allowlist={})
+        to_write = dataclasses.replace(to_write, allowlist=set(), dependency_files=set())
 
         handler._write_config(toml, to_write)
 
@@ -100,9 +96,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
                     "scripts": {"twyn": "twyn.cli:entry_point"},
                 },
                 "twyn": {
-                    "dependency_file": "my_file.txt",
                     "selector_method": "all",
-                    "allowlist": {},
                     "use_cache": False,
                     "recursive": False,
                 },
@@ -141,7 +135,7 @@ def test_no_load_config_from_cache(self, pyproject_toml_file: Path) -> None:
         config = ConfigHandler().resolve_config()
 
         assert config.allowlist == set()
-        assert config.dependency_file is None
+        assert config.dependency_files == set()
         assert config.use_cache is True
         assert config.selector_method == DEFAULT_SELECTOR_METHOD
         assert config.source is None
@@ -260,3 +254,15 @@ def test_invalid_selector_method_from_config_file(self, tmp_path: Path) -> None:
         error_message = str(exc_info.value)
         assert "Invalid selector_method 'invalid-selector'" in error_message
         assert "Must be one of: all, first-letter, nearby-letter" in error_message
+
+    def test_load_single_dependency_file(self, tmp_path: Path) -> None:
+        pyproject_toml = tmp_path / "pyproject.toml"
+        data = """
+        [tool.twyn]
+        dependency_file="requirements.txt"
+        """
+        pyproject_toml.write_text(data)
+
+        config = ConfigHandler(FileHandler(str(pyproject_toml))).resolve_config()
+
+        assert config.dependency_files == {"requirements.txt"}

tests/conftest.py

@@ -237,7 +237,7 @@ def pyproject_toml_file(tmp_path: Path) -> Iterator[Path]:
     twyn = "twyn.cli:entry_point"
 
     [tool.twyn]
-    dependency_file="my_file.txt"
+    dependency_file=["my_file.txt", "my_other_file.txt"]
     selector_method="all"
     logging_level="debug"
     allowlist=["boto4", "boto2"]

tests/dependency_parser/test_dependency_selector.py

@@ -28,14 +28,16 @@ class TestDependencySelector:
             ("/some/path/package-lock.json", PackageLockJsonParser),
         ],
     )
-    def test_get_dependency_parser(self, file_name: str, parser_class: type[AbstractParser]):
-        parser = DependencySelector(file_name).get_dependency_parsers()
+    def test_get_dependency_parser(self, file_name: str, parser_class: type[AbstractParser]) -> None:
+        parser = DependencySelector({file_name}).get_dependency_parsers()
         assert len(parser) == 1
 
         assert isinstance(parser[0], parser_class)
         assert str(parser[0].file_handler.file_path).endswith(file_name)
 
-    def test_get_dependency_parser_auto_detect_requirements_file(self, requirements_txt_file: Path, tmp_path: Path):
+    def test_get_dependency_parser_auto_detect_requirements_file(
+        self, requirements_txt_file: Path, tmp_path: Path
+    ) -> None:
         parser = DependencySelector("", root_path=str(tmp_path)).get_dependency_parsers()
         assert isinstance(parser[0], RequirementsTxtParser)
 

tests/main/conftest.py

@@ -8,6 +8,6 @@
 @pytest.fixture(scope="module")
 def disable_track() -> Generator[None, Any, None]:
     """Disables the track UI for running tests."""
-    with patch("rich.progress.track") as m_track:
+    with patch("rich.progress.track") as m_track, patch("click.echo"):
         m_track.side_effect = lambda iterable, description: iterable
         yield