Synchronise master with upstream

openstack_requirements/check.py

@@ -20,6 +20,7 @@
 
 from packaging import markers
 
+from openstack_requirements.project import Project
 from openstack_requirements import requirement
 
 MIN_PY_VERSION = '3.5'
@@ -32,23 +33,27 @@
 
 
 class RequirementsList:
-    def __init__(self, name, project):
+    def __init__(self, name: str, project: Project) -> None:
         self.name = name
-        self.reqs_by_file = {}
+        self.reqs_by_file: dict[str, dict[str, set[str]]] = {}
         self.project = project
         self.failed = False
 
     @property
-    def reqs(self):
+    def reqs(self) -> dict[str, set[str]]:
+        """Flattens the list of per-file reqs."""
         return {k: v for d in self.reqs_by_file.values() for k, v in d.items()}
 
-    def extract_reqs(self, content, strict):
+    def extract_reqs(
+        self, content: list[str], strict: bool
+    ) -> dict[str, set[str]]:
         reqs = collections.defaultdict(set)
-        parsed = requirement.parse(content)
+        parsed = requirement.parse_lines(content)
         for name, entries in parsed.items():
             if not name:
                 # Comments and other unprocessed lines
                 continue
+
             list_reqs = [r for (r, line) in entries]
             # Strip the comments out before checking if there are duplicates
             list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
@@ -64,7 +69,7 @@ def extract_reqs(self, content, strict):
             reqs[name].update(list_reqs)
         return reqs
 
-    def process(self, strict=True):
+    def process(self, strict: bool = True) -> None:
         """Convert the project into ready to use data.
 
         - an iterable of requirement sets to check
@@ -99,18 +104,17 @@ def process(self, strict=True):
                 )
 
             print(f"Processing {fname} (requirements)")
-            if strict and not content.endswith('\n'):
-                print(
-                    f"Requirements file {fname} does not end with a newline.",
-                    file=sys.stderr,
-                )
-            self.reqs_by_file[fname] = self.extract_reqs(content, strict)
+            self.reqs_by_file[f'{fname} (dependencies)'] = self.extract_reqs(
+                content, strict
+            )
 
         for fname, extras in self.project['extras'].items():
             print(f"Processing {fname} (extras)")
             for name, content in extras.items():
-                print(f"Processing .[{name}]")
-                self.reqs_by_file[name] = self.extract_reqs(content, strict)
+                print(f"  Processing .[{name}]")
+                self.reqs_by_file[f'{fname} (.[{name}] extra)'] = (
+                    self.extract_reqs(content, strict)
+                )
 
 
 def _get_exclusions(req):
@@ -191,20 +195,19 @@ def _is_requirement_in_global_reqs(
             difference = req_exclusions - global_exclusions
             print(
                 f"ERROR: Requirement for package {local_req.package} "
-                "excludes a version not excluded in the "
-                "global list.\n"
-                f"  Local settings : {req_exclusions}\n"
-                f"  Global settings: {global_exclusions}\n"
-                f"  Unexpected     : {difference}"
+                f"excludes a version not excluded in the "
+                f"global list.\n"
+                f"  Local settings : {list(req_exclusions)}\n"
+                f"  Global settings: {list(global_exclusions)}\n"
+                f"  Unexpected     : {list(difference)}"
             )
             return False
 
     print(
-        "ERROR: "
-        f"Could not find a global requirements entry to match package {local_req.package}. "
-        "If the package is already included in the global list, "
-        "the name or platform markers there may not match the local "
-        "settings."
+        f"ERROR: Could not find a global requirements entry to match package "
+        f"{local_req.package}. If the package is already included in the "
+        f"global list, the name or platform markers there may not match the "
+        f"local settings."
     )
     return False
 

openstack_requirements/cmds/check_exists.py

@@ -67,7 +67,7 @@ def main(args=None):
         print(
             f'\nComparing {require_file} with global-requirements and upper-constraints'
         )
-        requirements = requirement.parse(data)
+        requirements = requirement.parse_lines(data)
         for name, spec_list in requirements.items():
             if not name or name in denylist:
                 continue

openstack_requirements/project.py

@@ -19,6 +19,9 @@
 import errno
 import io
 import os
+import sys
+from typing import Any
+from typing import TypedDict
 
 try:
     # Python 3.11+
@@ -28,41 +31,73 @@
     import tomli as tomllib  # type: ignore
 
 
-def _read_pyproject_toml(root):
+def _read_raw(root: str, filename: str) -> str | None:
+    try:
+        path = os.path.join(root, filename)
+        with open(path, encoding="utf-8") as f:
+            data = f.read()
+            if not data.endswith('\n'):
+                print(
+                    f"Requirements file {filename} does not end with a "
+                    f"newline.",
+                    file=sys.stderr,
+                )
+            return data
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return None
+
+        raise
+
+
+def _read_pyproject_toml(root: str) -> dict[str, Any] | None:
     data = _read_raw(root, 'pyproject.toml')
     if data is None:
         return None
 
     return tomllib.loads(data)
 
 
-def _read_pyproject_toml_requirements(root):
+def _read_requirements_txt(root: str, filename: str) -> list[str] | None:
+    data = _read_raw(root, filename)
+    if data is None:
+        return None
+
+    result = []
+    for line in data.splitlines():
+        # we only ignore comments and empty lines: everything else is
+        # handled later
+        line = line.strip()
+
+        if line.startswith('#') or not line:
+            continue
+
+        result.append(line)
+
+    return result
+
+
+def _read_pyproject_toml_requirements(root: str) -> list[str] | None:
     data = _read_pyproject_toml(root) or {}
 
     # projects may not have PEP-621 project metadata
     if 'project' not in data:
         return None
 
-    # FIXME(stephenfin): We should not be doing this, but the fix requires a
-    # larger change to do normalization here.
-    return '\n'.join(data['project'].get('dependencies', []))
+    return data['project'].get('dependencies', [])
 
 
-def _read_pyproject_toml_extras(root):
+def _read_pyproject_toml_extras(root: str) -> dict[str, list[str]] | None:
     data = _read_pyproject_toml(root) or {}
 
     # projects may not have PEP-621 project metadata
     if 'project' not in data:
         return None
 
-    # FIXME(stephenfin): As above, we should not be doing this.
-    return {
-        k: '\n'.join(v) for k, v in
-        data['project'].get('optional-dependencies', {}).items()
-    }
+    return data['project'].get('optional-dependencies', {})
 
 
-def _read_setup_cfg_extras(root):
+def _read_setup_cfg_extras(root: str) -> dict[str, list[str]] | None:
     data = _read_raw(root, 'setup.cfg')
     if data is None:
         return None
@@ -72,20 +107,32 @@ def _read_setup_cfg_extras(root):
     if not c.has_section('extras'):
         return None
 
-    return dict(c.items('extras'))
+    result: dict[str, list[str]] = {}
+    for extra, deps in c.items('extras'):
+        result[extra] = []
+        for line in deps.splitlines():
+            # we only ignore comments and empty lines: everything else is
+            # handled later
+            line = line.strip()
 
+            if line.startswith('#') or not line:
+                continue
 
-def _read_raw(root, filename):
-    try:
-        path = os.path.join(root, filename)
-        with open(path, encoding="utf-8") as f:
-            return f.read()
-    except OSError as e:
-        if e.errno != errno.ENOENT:
-            raise
+            result[extra].append(line)
+
+    return result
+
+
+class Project(TypedDict):
+    # The root directory path
+    root: str
+    # A mapping of filename to the contents of that file
+    requirements: dict[str, list[str]]
+    # A mapping of filename to extras from that file
+    extras: dict[str, dict[str, list[str]]]
 
 
-def read(root):
+def read(root: str) -> Project:
     """Read into memory the packaging data for the project at root.
 
     :param root: A directory path.
@@ -96,11 +143,13 @@ def read(root):
           requirements
     """
     # Store root directory and installer-related files for later processing
-    result = {'root': root}
+    result: Project = {
+        'root': root,
+        'requirements': {},
+        'extras': {},
+    }
 
     # Store requirements
-    result['requirements'] = {}
-
     if (data := _read_pyproject_toml_requirements(root)) is not None:
         result['requirements']['pyproject.toml'] = data
 
@@ -116,12 +165,10 @@ def read(root):
         'test-requirements-py2.txt',
         'test-requirements-py3.txt',
     ]:
-        if (data := _read_raw(root, filename)) is not None:
+        if (data := _read_requirements_txt(root, filename)) is not None:
             result['requirements'][filename] = data
 
     # Store extras
-    result['extras'] = {}
-
     if (data := _read_setup_cfg_extras(root)) is not None:
         result['extras']['setup.cfg'] = data
 

openstack_requirements/requirement.py

@@ -99,6 +99,10 @@ def parse(content, permit_urls=False):
     return to_dict(to_reqs(content, permit_urls=permit_urls))
 
 
+def parse_lines(lines, permit_urls=False):
+    return to_dict(to_req(line, permit_urls=permit_urls) for line in lines)
+
+
 def parse_line(req_line, permit_urls=False):
     """Parse a single line of a requirements file.
 
@@ -201,6 +205,13 @@ def _pass_through(req_line, permit_urls=False):
         )
 
 
+def to_req(line, permit_urls=False):
+    if _pass_through(line, permit_urls=permit_urls):
+        return (None, line)
+    else:
+        return (parse_line(line, permit_urls=permit_urls), line)
+
+
 def to_reqs(content, permit_urls=False):
     for content_line in content.splitlines(True):
         req_line = content_line.strip()
@@ -209,10 +220,7 @@ def to_reqs(content, permit_urls=False):
         if req_line.startswith('#') or not req_line:
             continue
 
-        if _pass_through(req_line, permit_urls=permit_urls):
-            yield None, content_line
-        else:
-            yield parse_line(req_line, permit_urls=permit_urls), content_line
+        yield to_req(req_line, permit_urls)
 
 
 def check_reqs_bounds_policy(global_reqs):

openstack_requirements/tests/test_check.py

@@ -30,24 +30,24 @@ def test_extras__setup_cfg(self):
         project_data = {
             'root': '/fake/root',
             'requirements': {
-                'requirements.txt': 'requests>=2.0.0\n'
+                'requirements.txt': ['requests>=2.0.0'],
             },
             'extras': {
                 'setup.cfg': {
-                    'test': 'pytest>=6.0.0\nflake8>=3.8.0\n',
-                    'dev': 'black>=24.0.0\nmypy>=0.900\n'
+                    'test': ['pytest>=6.0.0', 'flake8>=3.8.0'],
+                    'dev': ['black>=24.0.0', 'mypy>=0.900'],
                 }
-            }
+            },
         }
 
         req_list = check.RequirementsList('test-project', project_data)
         req_list.process(strict=False)
 
-        self.assertIn('test', req_list.reqs_by_file)
-        self.assertIn('dev', req_list.reqs_by_file)
+        self.assertIn('setup.cfg (.[test] extra)', req_list.reqs_by_file)
+        self.assertIn('setup.cfg (.[dev] extra)', req_list.reqs_by_file)
 
-        test_reqs = req_list.reqs_by_file['test']
-        dev_reqs = req_list.reqs_by_file['dev']
+        test_reqs = req_list.reqs_by_file['setup.cfg (.[test] extra)']
+        dev_reqs = req_list.reqs_by_file['setup.cfg (.[dev] extra)']
 
         self.assertEqual(len(test_reqs), 2)
         self.assertIn('pytest', test_reqs)

openstack_requirements/tests/test_check_constraints.py

@@ -137,16 +137,14 @@ def remove_req_read_reqs_file(filename):
 
             return orig_mocked_read_req(filename)
 
-        new_reqs = '>1.10.0\nsomerandommodule\n'
-
         # lets change the six requirement not include the u-c version
         proj_read = project.read(common.project_fixture.root)
-        proj_read['requirements']['requirements.txt'] = (
-            proj_read['requirements']['requirements.txt'][:-1] + new_reqs
+        proj_read['requirements']['requirements.txt'][-1] = 'six>1.10.0'
+        proj_read['requirements']['requirements.txt'].append(
+            'somerandommodule'
         )
-        proj_read['requirements']['test-requirements.txt'] = (
-            proj_read['requirements']['test-requirements.txt']
-            + 'anotherrandommodule\n'
+        proj_read['requirements']['test-requirements.txt'].append(
+            'anotherrandommodule'
         )
 
         expected_outs = [
@@ -182,9 +180,7 @@ def test_project_req_bigger_then_uc(self):
 
         # lets change the six requirement not include the u-c version
         proj_read = project.read(common.project_fixture.root)
-        proj_read['requirements']['requirements.txt'] = (
-            proj_read['requirements']['requirements.txt'][:-1] + '>1.10.0\n'
-        )
+        proj_read['requirements']['requirements.txt'][-1] = 'six>1.10.0'
         expected_out = (
             'six must be <= 1.10.0 from upper-constraints and '
             'include the upper-constraints version'
@@ -211,9 +207,8 @@ def test_project_req_not_include_uc_version(self):
 
         # lets change the six requirement not include the u-c version
         proj_read = project.read(common.project_fixture.root)
-        proj_read['requirements']['requirements.txt'] = (
-            proj_read['requirements']['requirements.txt'][:-1]
-            + '<1.10.0,>1.10.0\n'
+        proj_read['requirements']['requirements.txt'][-1] = (
+            'six<1.10.0,>1.10.0'
         )
         expected_out = (
             'six must be <= 1.10.0 from upper-constraints and '