Handle requirements.txt file with -r line (#416)

be able to handle requirements.txt file with -r line

Author: clavedeluna

src/codemodder/project_analysis/file_parsers/requirements_txt_file_parser.py

@@ -29,9 +29,7 @@ def _parse_file(self, file: Path) -> PackageStore | None:
             logger.debug("Unknown encoding for file: %s", file)
             return None
 
-        dependencies = set(
-            line.split("#")[0].strip() for line in lines if not line.startswith("#")
-        )
+        dependencies = self._clean_lines(lines)
 
         return PackageStore(
             type=self.file_type,
@@ -42,3 +40,13 @@ def _parse_file(self, file: Path) -> PackageStore | None:
             # and extracting py versions from them.
             py_versions=[],
         )
+
+    def _clean_lines(self, lines: list[str]) -> set[str]:
+        """Return a set of dependency `lines` excluding any lines
+        that may be comments or may be pointers to other requirement files (-r ..._
+        """
+        return set(
+            line.split("#")[0].strip()
+            for line in lines
+            if not (line.startswith("#") or line.startswith("-r "))
+        )

tests/conftest.py

@@ -45,6 +45,17 @@ def pkg_with_reqs_txt(tmp_path_factory):
     return base_dir
 
 
+@pytest.fixture(scope="module")
+def pkg_with_reqs_r_line(tmp_path_factory):
+    base_dir = tmp_path_factory.mktemp("foo")
+    req_file = base_dir / "requirements.txt"
+    second_req_file = base_dir / "more_requirements.txt"
+    second_req_file.write_text("django<5")
+    reqs = "# comment\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n-r more_requirements.txt\n"
+    req_file.write_text(reqs)
+    return base_dir
+
+
 @pytest.fixture(scope="module")
 def pkg_with_reqs_txt_and_comments(tmp_path_factory):
     base_dir = tmp_path_factory.mktemp("foo")

tests/project_analysis/file_parsers/test_requirements_txt_file_parser.py

@@ -47,3 +47,13 @@ def test_trailing_comments(self, pkg_with_reqs_txt_and_comments):
         assert store.file == pkg_with_reqs_txt_and_comments / parser.file_type.value
         assert store.py_versions == []
         assert len(store.dependencies) == 4
+
+    def test_parse_with_r_line(self, pkg_with_reqs_r_line):
+        parser = RequirementsTxtParser(pkg_with_reqs_r_line)
+        found = parser.parse()
+        assert len(found) == 1
+        store = found[0]
+        assert store.type.value == "requirements.txt"
+        assert store.file == pkg_with_reqs_r_line / parser.file_type.value
+        assert store.py_versions == []
+        assert len(store.dependencies) == 4