feat: support multiple dependency files in config and cli

Author: sdn4z

src/twyn/cli.py

@@ -47,6 +47,7 @@ def entry_point() -> None:
 @click.option(
     "--dependency-file",
     type=str,
+    multiple=True,
     help=(
         "Dependency file to analyze. By default, twyn will search in the current directory "
         "for supported files, but this option will override that behavior."
@@ -112,7 +113,7 @@ def entry_point() -> None:
 )
 def run(  # noqa: C901
     config: str,
-    dependency_file: Optional[str],
+    dependency_file: tuple[str],
     dependency: tuple[str],
     selector_method: str,
     v: bool,
@@ -133,15 +134,16 @@ def run(  # noqa: C901
             "Only one of --dependency or --dependency-file can be set at a time.", ctx=click.get_current_context()
         )
 
-    if dependency_file and not any(dependency_file.endswith(key) for key in DEPENDENCY_FILE_MAPPING):
-        raise click.UsageError("Dependency file name not supported.", ctx=click.get_current_context())
+    for dep_file in dependency_file:
+        if dep_file and not any(dep_file.endswith(key) for key in DEPENDENCY_FILE_MAPPING):
+            raise click.UsageError(f"Dependency file name {dep_file} not supported.", ctx=click.get_current_context())
 
     try:
         possible_typos = check_dependencies(
             selector_method=selector_method,
             dependencies=set(dependency) or None,
             config_file=config,
-            dependency_file=dependency_file,
+            dependency_files=set(dependency_file) or None,
             use_cache=not no_cache if no_cache is not None else no_cache,
             show_progress_bar=False if json else not no_track,
             load_config_from_file=True,

src/twyn/config/config_handler.py

@@ -3,6 +3,7 @@
 from enum import Enum
 from pathlib import Path
 from typing import Any, Optional, Union
+from warnings import warn
 
 from tomlkit import TOMLDocument, dumps, load, table
 
@@ -32,7 +33,7 @@
 class TwynConfiguration:
     """Fully resolved configuration for Twyn."""
 
-    dependency_file: Optional[str]
+    dependency_files: set[str]
     selector_method: str
     allowlist: set[str]
     source: Optional[str]
@@ -45,7 +46,7 @@ class TwynConfiguration:
 class ReadTwynConfiguration:
     """Configuration for twyn as set by the user. It may have None values."""
 
-    dependency_file: Optional[str] = None
+    dependency_files: Optional[set[str]] = field(default_factory=set)
     selector_method: Optional[str] = None
     allowlist: set[str] = field(default_factory=set)
     source: Optional[str] = None
@@ -63,7 +64,7 @@ def __init__(self, file_handler: Optional[FileHandler] = None) -> None:
     def resolve_config(
         self,
         selector_method: Optional[str] = None,
-        dependency_file: Optional[str] = None,
+        dependency_files: Optional[set[str]] = None,
         use_cache: Optional[bool] = None,
         package_ecosystem: Optional[PackageEcosystems] = None,
         recursive: Optional[bool] = None,
@@ -107,7 +108,7 @@ def resolve_config(
             final_recursive = DEFAULT_RECURSIVE
 
         return TwynConfiguration(
-            dependency_file=dependency_file or read_config.dependency_file,
+            dependency_files=dependency_files or read_config.dependency_files or set(),
             selector_method=final_selector_method,
             allowlist=read_config.allowlist,
             source=read_config.source,
@@ -141,12 +142,39 @@ def remove_package_from_allowlist(self, package_name: str) -> None:
     def _get_read_config(self, toml: TOMLDocument) -> ReadTwynConfiguration:
         """Read the twyn configuration from a provided toml document."""
         twyn_config_data = toml.get("tool", {}).get("twyn", {})
+        dependency_file = twyn_config_data.get("dependency_file", set())
+        dependency_files = twyn_config_data.get("dependency_files", set())
+
+        if dependency_file:
+            if dependency_files:
+                raise TOMLError(
+                    "Found both `dependency_file` and `dependency_files` in your config file. Please, use only `dependency_files`."
+                )
+
+            warn(
+                "Using `dependency_file` in your twyn configuration is deprecated, this field will be removed in an upcoming release. Use `dependency_files` instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+            dependency_files = {dependency_file}
+        elif dependency_files:
+            dependency_files = set(dependency_files)
+
+        allowlist = twyn_config_data.get("allowlist", set())
+        if isinstance(allowlist, str):
+            allowlist = {allowlist}
+        elif isinstance(allowlist, list):
+            allowlist = set(allowlist)
+
         return ReadTwynConfiguration(
-            dependency_file=twyn_config_data.get("dependency_file"),
+            dependency_files=dependency_files,
             selector_method=twyn_config_data.get("selector_method"),
-            allowlist=set(twyn_config_data.get("allowlist", set())),
+            allowlist=allowlist,
             source=twyn_config_data.get("source"),
             use_cache=twyn_config_data.get("use_cache"),
+            package_ecosystem=twyn_config_data.get("package_ecosystem"),
+            recursive=twyn_config_data.get("recursive"),
         )
 
     def _write_config(self, toml: TOMLDocument, config: ReadTwynConfiguration) -> None:
@@ -155,12 +183,12 @@ def _write_config(self, toml: TOMLDocument, config: ReadTwynConfiguration) -> No
         All null values are simply omitted from the toml file.
         """
         twyn_toml_data = asdict(config, dict_factory=lambda x: _serialize_config(x))
+
         if "tool" not in toml:
             toml.add("tool", table())
         if "twyn" not in toml["tool"]:  # type: ignore[operator]
             toml["tool"]["twyn"] = {}  # type: ignore[index]
         toml["tool"]["twyn"] = twyn_toml_data  # type: ignore[index]
-
         self._write_toml(toml)
 
     def _write_toml(self, toml: TOMLDocument) -> None:

src/twyn/dependency_parser/dependency_selector.py

@@ -12,8 +12,8 @@
 
 
 class DependencySelector:
-    def __init__(self, dependency_file: Optional[str] = None, root_path: str = ".") -> None:
-        self.dependency_file = dependency_file or ""
+    def __init__(self, dependency_files: Optional[set[str]] = None, root_path: str = ".") -> None:
+        self.dependency_files = dependency_files or set()
         self.root_path = root_path
 
     def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
@@ -38,18 +38,18 @@ def auto_detect_dependency_file_parser(self) -> list[AbstractParser]:
 
     def get_dependency_file_parsers_from_file_name(self) -> list[AbstractParser]:
         parsers = []
-        for known_dependency_file_name in DEPENDENCY_FILE_MAPPING:
-            if self.dependency_file.endswith(known_dependency_file_name):
-                file_parser = DEPENDENCY_FILE_MAPPING[known_dependency_file_name](self.dependency_file)
-                parsers.append(file_parser)
-
+        for dependency_file in self.dependency_files:
+            for known_dependency_file_name in DEPENDENCY_FILE_MAPPING:
+                if dependency_file.endswith(known_dependency_file_name):
+                    file_parser = DEPENDENCY_FILE_MAPPING[known_dependency_file_name](dependency_file)
+                    parsers.append(file_parser)
         if not parsers:
             raise NoMatchingParserError
 
         return parsers
 
     def get_dependency_parsers(self) -> list[AbstractParser]:
-        if self.dependency_file:
+        if self.dependency_files:
             logger.debug("Dependency file provided. Assigning a parser.")
             return self.get_dependency_file_parsers_from_file_name()
 

src/twyn/main.py

@@ -32,12 +32,13 @@
 
 logger = logging.getLogger("twyn")
 logger.addHandler(logging.NullHandler())
+logging.captureWarnings(True)
 
 
 def check_dependencies(
     selector_method: Union[SelectorMethod, None] = None,
     config_file: Optional[str] = None,
-    dependency_file: Optional[str] = None,
+    dependency_files: Optional[set[str]] = None,
     dependencies: Optional[set[str]] = None,
     use_cache: Optional[bool] = True,
     show_progress_bar: bool = False,
@@ -68,7 +69,7 @@ def check_dependencies(
         load_config_from_file=load_config_from_file,
         config_file=config_file,
         selector_method=selector_method,
-        dependency_file=dependency_file,
+        dependency_files=dependency_files,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
         recursive=recursive,
@@ -93,7 +94,7 @@ def check_dependencies(
     if config.package_ecosystem:
         logger.warning("`package_ecosystem` is not supported when reading dependencies from files. It will be ignored.")
 
-    if config.dependency_file and config.recursive:
+    if config.dependency_files and config.recursive:
         logger.warning(
             "`--recursive` has been set together with `--dependency-file`. `--dependency-file` will take precedence."
         )
@@ -104,7 +105,7 @@ def check_dependencies(
         maybe_cache_handler=maybe_cache_handler,
         allowlist=config.allowlist,
         show_progress_bar=show_progress_bar,
-        dependency_file=config.dependency_file,
+        dependency_files=config.dependency_files,
     )
 
 
@@ -153,16 +154,17 @@ def _analyze_packages_from_source(
     allowlist: set[str],
     selector_method: SelectorMethod,
     show_progress_bar: bool,
-    dependency_file: Optional[str],
+    dependency_files: Optional[set[str]],
     source: Optional[str],
     maybe_cache_handler: Optional[CacheHandler],
 ) -> TyposquatCheckResults:
     """Analyze dependencies from a dependencies file.
 
     It will return a list of the possible typos grouped by source, each source being a dependency file.
     """
-    dependency_managers = _get_dependency_managers_and_parsers_mapping(dependency_file)
     typos_by_file = TyposquatCheckResults()
+
+    dependency_managers = _get_dependency_managers_and_parsers_mapping(dependency_files)
     for dependency_manager, parsers in dependency_managers.items():
         top_package_reference = dependency_manager.trusted_packages_source(source, maybe_cache_handler)
 
@@ -174,7 +176,6 @@ def _analyze_packages_from_source(
             threshold_class=SimilarityThreshold,
         )
         results: list[TyposquatCheckResultFromSource] = []
-
         for parser in parsers:
             analyzed_dependencies = _analyze_dependencies(
                 top_package_reference, trusted_packages, parser.parse(), allowlist, show_progress_bar, parser.file_path
@@ -252,13 +253,13 @@ def _get_selector_method(selector_method: str) -> SelectorMethod:
 
 
 def _get_dependency_managers_and_parsers_mapping(
-    dependency_file: Optional[str],
+    dependency_files: Optional[set[str]],
 ) -> dict[type[BaseDependencyManager], list[AbstractParser]]:
     """Return a dictionary, grouping all files to parse by their DependencyManager."""
     dependency_managers: dict[type[BaseDependencyManager], list[AbstractParser]] = {}
 
     # No dependencies introduced via the CLI, so the dependecy file was either given or will be auto-detected
-    dependency_selector = DependencySelector(dependency_file)
+    dependency_selector = DependencySelector(dependency_files)
     dependency_parsers = dependency_selector.get_dependency_parsers()
 
     for parser in dependency_parsers:
@@ -274,7 +275,7 @@ def _get_config(
     load_config_from_file: bool,
     config_file: Optional[str],
     selector_method: Union[SelectorMethod, None],
-    dependency_file: Optional[str],
+    dependency_files: Optional[set[str]],
     use_cache: Optional[bool],
     package_ecosystem: Optional[PackageEcosystems],
     recursive: Optional[bool],
@@ -286,7 +287,7 @@ def _get_config(
         config_file_handler = None
     return ConfigHandler(config_file_handler).resolve_config(
         selector_method=selector_method,
-        dependency_file=dependency_file,
+        dependency_files=dependency_files,
         use_cache=use_cache,
         package_ecosystem=package_ecosystem,
         recursive=recursive,

tests/config/test_config_handler.py

@@ -1,7 +1,6 @@
 import dataclasses
 from copy import deepcopy
 from pathlib import Path
-from typing import NoReturn
 from unittest.mock import Mock, patch
 
 import pytest
@@ -26,17 +25,14 @@
 
 
 class TestConfigHandler:
-    def throw_exception(self) -> NoReturn:
-        raise PathNotFoundError
-
     @patch("twyn.file_handler.file_handler.FileHandler.read")
     def test_no_enforce_file_on_non_existent_file(self, mock_is_file: Mock) -> None:
         """Resolving the config without enforcing the file to be present gives you defaults."""
-        mock_is_file.side_effect = self.throw_exception
+        mock_is_file.side_effect = PathNotFoundError()
         config = ConfigHandler(FileHandler(DEFAULT_PROJECT_TOML_FILE)).resolve_config()
 
         assert config == TwynConfiguration(
-            dependency_file=None,
+            dependency_files=set(),
             selector_method="all",
             allowlist=set(),
             source=None,
@@ -51,7 +47,7 @@ def test_config_raises_for_unknown_file(self) -> None:
 
     def test_read_config_values(self, pyproject_toml_file: Path) -> None:
         config = ConfigHandler(file_handler=FileHandler(pyproject_toml_file)).resolve_config()
-        assert config.dependency_file == "my_file.txt"
+        assert config.dependency_files == {"my_file.txt", "my_other_file.txt"}
         assert config.selector_method == "all"
         assert config.allowlist == {"boto4", "boto2"}
         assert config.use_cache is False
@@ -62,7 +58,7 @@ def test_get_twyn_data_from_file(self, pyproject_toml_file: Path) -> None:
         toml = handler._read_toml()
         twyn_data = ConfigHandler(FileHandler(pyproject_toml_file))._get_read_config(toml)
         assert twyn_data == ReadTwynConfiguration(
-            dependency_file="my_file.txt",
+            dependency_files={"my_file.txt", "my_other_file.txt"},
             selector_method="all",
             allowlist={"boto4", "boto2"},
             source=None,
@@ -75,7 +71,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
 
         initial_config = handler.resolve_config()
         to_write = deepcopy(initial_config)
-        to_write = dataclasses.replace(to_write, allowlist={})
+        to_write = dataclasses.replace(to_write, allowlist=set(), dependency_files=set())
 
         handler._write_config(toml, to_write)
 
@@ -100,9 +96,7 @@ def test_write_toml(self, pyproject_toml_file: Path) -> None:
                     "scripts": {"twyn": "twyn.cli:entry_point"},
                 },
                 "twyn": {
-                    "dependency_file": "my_file.txt",
                     "selector_method": "all",
-                    "allowlist": {},
                     "use_cache": False,
                     "recursive": False,
                 },
@@ -141,7 +135,7 @@ def test_no_load_config_from_cache(self, pyproject_toml_file: Path) -> None:
         config = ConfigHandler().resolve_config()
 
         assert config.allowlist == set()
-        assert config.dependency_file is None
+        assert config.dependency_files == set()
         assert config.use_cache is True
         assert config.selector_method == DEFAULT_SELECTOR_METHOD
         assert config.source is None
@@ -260,3 +254,27 @@ def test_invalid_selector_method_from_config_file(self, tmp_path: Path) -> None:
         error_message = str(exc_info.value)
         assert "Invalid selector_method 'invalid-selector'" in error_message
         assert "Must be one of: all, first-letter, nearby-letter" in error_message
+
+    def test_load_single_dependency_file(self, tmp_path: Path) -> None:
+        pyproject_toml = tmp_path / "pyproject.toml"
+        data = """
+        [tool.twyn]
+        dependency_file="requirements.txt"
+        """
+        pyproject_toml.write_text(data)
+
+        config = ConfigHandler(FileHandler(str(pyproject_toml))).resolve_config()
+
+        assert config.dependency_files == {"requirements.txt"}
+
+    def test_both_dependency_files_args_in_config_raise_error(self, tmp_path: Path) -> None:
+        pyproject_toml = tmp_path / "pyproject.toml"
+        data = """
+        [tool.twyn]
+        dependency_file="requirements.txt"
+        dependency_files=["requirements.txt"]
+        """
+        pyproject_toml.write_text(data)
+
+        with pytest.raises(TOMLError):
+            ConfigHandler(FileHandler(str(pyproject_toml))).resolve_config()

tests/conftest.py

@@ -237,7 +237,7 @@ def pyproject_toml_file(tmp_path: Path) -> Iterator[Path]:
     twyn = "twyn.cli:entry_point"
 
     [tool.twyn]
-    dependency_file="my_file.txt"
+    dependency_files=["my_file.txt", "my_other_file.txt"]
     selector_method="all"
     logging_level="debug"
     allowlist=["boto4", "boto2"]