diff --git a/openstack_requirements/check.py b/openstack_requirements/check.py index 34ded6df5..7b6711b24 100644 --- a/openstack_requirements/check.py +++ b/openstack_requirements/check.py @@ -20,6 +20,7 @@ from packaging import markers +from openstack_requirements.project import Project from openstack_requirements import requirement MIN_PY_VERSION = '3.5' @@ -32,23 +33,27 @@ class RequirementsList: - def __init__(self, name, project): + def __init__(self, name: str, project: Project) -> None: self.name = name - self.reqs_by_file = {} + self.reqs_by_file: dict[str, dict[str, set[str]]] = {} self.project = project self.failed = False @property - def reqs(self): + def reqs(self) -> dict[str, set[str]]: + """Flattens the list of per-file reqs.""" return {k: v for d in self.reqs_by_file.values() for k, v in d.items()} - def extract_reqs(self, content, strict): + def extract_reqs( + self, content: list[str], strict: bool + ) -> dict[str, set[str]]: reqs = collections.defaultdict(set) - parsed = requirement.parse(content) + parsed = requirement.parse_lines(content) for name, entries in parsed.items(): if not name: # Comments and other unprocessed lines continue + list_reqs = [r for (r, line) in entries] # Strip the comments out before checking if there are duplicates list_reqs_stripped = [r._replace(comment='') for r in list_reqs] @@ -64,7 +69,7 @@ def extract_reqs(self, content, strict): reqs[name].update(list_reqs) return reqs - def process(self, strict=True): + def process(self, strict: bool = True) -> None: """Convert the project into ready to use data. - an iterable of requirement sets to check @@ -99,18 +104,17 @@ def process(self, strict=True): ) print(f"Processing {fname} (requirements)") - if strict and not content.endswith('\n'): - print( - f"Requirements file {fname} does not end with a newline.", - file=sys.stderr, - ) - self.reqs_by_file[fname] = self.extract_reqs(content, strict) + self.reqs_by_file[f'{fname} (dependencies)'] = self.extract_reqs( + content, strict + ) for fname, extras in self.project['extras'].items(): print(f"Processing {fname} (extras)") for name, content in extras.items(): - print(f"Processing .[{name}]") - self.reqs_by_file[name] = self.extract_reqs(content, strict) + print(f" Processing .[{name}]") + self.reqs_by_file[f'{fname} (.[{name}] extra)'] = ( + self.extract_reqs(content, strict) + ) def _get_exclusions(req): @@ -191,20 +195,19 @@ def _is_requirement_in_global_reqs( difference = req_exclusions - global_exclusions print( f"ERROR: Requirement for package {local_req.package} " - "excludes a version not excluded in the " - "global list.\n" - f" Local settings : {req_exclusions}\n" - f" Global settings: {global_exclusions}\n" - f" Unexpected : {difference}" + f"excludes a version not excluded in the " + f"global list.\n" + f" Local settings : {list(req_exclusions)}\n" + f" Global settings: {list(global_exclusions)}\n" + f" Unexpected : {list(difference)}" ) return False print( - "ERROR: " - f"Could not find a global requirements entry to match package {local_req.package}. " - "If the package is already included in the global list, " - "the name or platform markers there may not match the local " - "settings." + f"ERROR: Could not find a global requirements entry to match package " + f"{local_req.package}. If the package is already included in the " + f"global list, the name or platform markers there may not match the " + f"local settings." ) return False diff --git a/openstack_requirements/cmds/check_exists.py b/openstack_requirements/cmds/check_exists.py index 685b3a259..dd2d5c680 100644 --- a/openstack_requirements/cmds/check_exists.py +++ b/openstack_requirements/cmds/check_exists.py @@ -67,7 +67,7 @@ def main(args=None): print( f'\nComparing {require_file} with global-requirements and upper-constraints' ) - requirements = requirement.parse(data) + requirements = requirement.parse_lines(data) for name, spec_list in requirements.items(): if not name or name in denylist: continue diff --git a/openstack_requirements/project.py b/openstack_requirements/project.py index 213d42834..a83035f9f 100644 --- a/openstack_requirements/project.py +++ b/openstack_requirements/project.py @@ -19,6 +19,9 @@ import errno import io import os +import sys +from typing import Any +from typing import TypedDict try: # Python 3.11+ @@ -28,7 +31,26 @@ import tomli as tomllib # type: ignore -def _read_pyproject_toml(root): +def _read_raw(root: str, filename: str) -> str | None: + try: + path = os.path.join(root, filename) + with open(path, encoding="utf-8") as f: + data = f.read() + if not data.endswith('\n'): + print( + f"Requirements file {filename} does not end with a " + f"newline.", + file=sys.stderr, + ) + return data + except OSError as e: + if e.errno == errno.ENOENT: + return None + + raise + + +def _read_pyproject_toml(root: str) -> dict[str, Any] | None: data = _read_raw(root, 'pyproject.toml') if data is None: return None @@ -36,33 +58,46 @@ def _read_pyproject_toml(root): return tomllib.loads(data) -def _read_pyproject_toml_requirements(root): +def _read_requirements_txt(root: str, filename: str) -> list[str] | None: + data = _read_raw(root, filename) + if data is None: + return None + + result = [] + for line in data.splitlines(): + # we only ignore comments and empty lines: everything else is + # handled later + line = line.strip() + + if line.startswith('#') or not line: + continue + + result.append(line) + + return result + + +def _read_pyproject_toml_requirements(root: str) -> list[str] | None: data = _read_pyproject_toml(root) or {} # projects may not have PEP-621 project metadata if 'project' not in data: return None - # FIXME(stephenfin): We should not be doing this, but the fix requires a - # larger change to do normalization here. - return '\n'.join(data['project'].get('dependencies', [])) + return data['project'].get('dependencies', []) -def _read_pyproject_toml_extras(root): +def _read_pyproject_toml_extras(root: str) -> dict[str, list[str]] | None: data = _read_pyproject_toml(root) or {} # projects may not have PEP-621 project metadata if 'project' not in data: return None - # FIXME(stephenfin): As above, we should not be doing this. - return { - k: '\n'.join(v) for k, v in - data['project'].get('optional-dependencies', {}).items() - } + return data['project'].get('optional-dependencies', {}) -def _read_setup_cfg_extras(root): +def _read_setup_cfg_extras(root: str) -> dict[str, list[str]] | None: data = _read_raw(root, 'setup.cfg') if data is None: return None @@ -72,20 +107,32 @@ def _read_setup_cfg_extras(root): if not c.has_section('extras'): return None - return dict(c.items('extras')) + result: dict[str, list[str]] = {} + for extra, deps in c.items('extras'): + result[extra] = [] + for line in deps.splitlines(): + # we only ignore comments and empty lines: everything else is + # handled later + line = line.strip() + if line.startswith('#') or not line: + continue -def _read_raw(root, filename): - try: - path = os.path.join(root, filename) - with open(path, encoding="utf-8") as f: - return f.read() - except OSError as e: - if e.errno != errno.ENOENT: - raise + result[extra].append(line) + + return result + + +class Project(TypedDict): + # The root directory path + root: str + # A mapping of filename to the contents of that file + requirements: dict[str, list[str]] + # A mapping of filename to extras from that file + extras: dict[str, dict[str, list[str]]] -def read(root): +def read(root: str) -> Project: """Read into memory the packaging data for the project at root. :param root: A directory path. @@ -96,11 +143,13 @@ def read(root): requirements """ # Store root directory and installer-related files for later processing - result = {'root': root} + result: Project = { + 'root': root, + 'requirements': {}, + 'extras': {}, + } # Store requirements - result['requirements'] = {} - if (data := _read_pyproject_toml_requirements(root)) is not None: result['requirements']['pyproject.toml'] = data @@ -116,12 +165,10 @@ def read(root): 'test-requirements-py2.txt', 'test-requirements-py3.txt', ]: - if (data := _read_raw(root, filename)) is not None: + if (data := _read_requirements_txt(root, filename)) is not None: result['requirements'][filename] = data # Store extras - result['extras'] = {} - if (data := _read_setup_cfg_extras(root)) is not None: result['extras']['setup.cfg'] = data diff --git a/openstack_requirements/requirement.py b/openstack_requirements/requirement.py index 8782583f9..40e4ea325 100644 --- a/openstack_requirements/requirement.py +++ b/openstack_requirements/requirement.py @@ -99,6 +99,10 @@ def parse(content, permit_urls=False): return to_dict(to_reqs(content, permit_urls=permit_urls)) +def parse_lines(lines, permit_urls=False): + return to_dict(to_req(line, permit_urls=permit_urls) for line in lines) + + def parse_line(req_line, permit_urls=False): """Parse a single line of a requirements file. @@ -201,6 +205,13 @@ def _pass_through(req_line, permit_urls=False): ) +def to_req(line, permit_urls=False): + if _pass_through(line, permit_urls=permit_urls): + return (None, line) + else: + return (parse_line(line, permit_urls=permit_urls), line) + + def to_reqs(content, permit_urls=False): for content_line in content.splitlines(True): req_line = content_line.strip() @@ -209,10 +220,7 @@ def to_reqs(content, permit_urls=False): if req_line.startswith('#') or not req_line: continue - if _pass_through(req_line, permit_urls=permit_urls): - yield None, content_line - else: - yield parse_line(req_line, permit_urls=permit_urls), content_line + yield to_req(req_line, permit_urls) def check_reqs_bounds_policy(global_reqs): diff --git a/openstack_requirements/tests/test_check.py b/openstack_requirements/tests/test_check.py index 414d2c67d..ef34a74e9 100644 --- a/openstack_requirements/tests/test_check.py +++ b/openstack_requirements/tests/test_check.py @@ -30,24 +30,24 @@ def test_extras__setup_cfg(self): project_data = { 'root': '/fake/root', 'requirements': { - 'requirements.txt': 'requests>=2.0.0\n' + 'requirements.txt': ['requests>=2.0.0'], }, 'extras': { 'setup.cfg': { - 'test': 'pytest>=6.0.0\nflake8>=3.8.0\n', - 'dev': 'black>=24.0.0\nmypy>=0.900\n' + 'test': ['pytest>=6.0.0', 'flake8>=3.8.0'], + 'dev': ['black>=24.0.0', 'mypy>=0.900'], } - } + }, } req_list = check.RequirementsList('test-project', project_data) req_list.process(strict=False) - self.assertIn('test', req_list.reqs_by_file) - self.assertIn('dev', req_list.reqs_by_file) + self.assertIn('setup.cfg (.[test] extra)', req_list.reqs_by_file) + self.assertIn('setup.cfg (.[dev] extra)', req_list.reqs_by_file) - test_reqs = req_list.reqs_by_file['test'] - dev_reqs = req_list.reqs_by_file['dev'] + test_reqs = req_list.reqs_by_file['setup.cfg (.[test] extra)'] + dev_reqs = req_list.reqs_by_file['setup.cfg (.[dev] extra)'] self.assertEqual(len(test_reqs), 2) self.assertIn('pytest', test_reqs) diff --git a/openstack_requirements/tests/test_check_constraints.py b/openstack_requirements/tests/test_check_constraints.py index 354f33c7e..f8d8d6da1 100644 --- a/openstack_requirements/tests/test_check_constraints.py +++ b/openstack_requirements/tests/test_check_constraints.py @@ -137,16 +137,14 @@ def remove_req_read_reqs_file(filename): return orig_mocked_read_req(filename) - new_reqs = '>1.10.0\nsomerandommodule\n' - # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) - proj_read['requirements']['requirements.txt'] = ( - proj_read['requirements']['requirements.txt'][:-1] + new_reqs + proj_read['requirements']['requirements.txt'][-1] = 'six>1.10.0' + proj_read['requirements']['requirements.txt'].append( + 'somerandommodule' ) - proj_read['requirements']['test-requirements.txt'] = ( - proj_read['requirements']['test-requirements.txt'] - + 'anotherrandommodule\n' + proj_read['requirements']['test-requirements.txt'].append( + 'anotherrandommodule' ) expected_outs = [ @@ -182,9 +180,7 @@ def test_project_req_bigger_then_uc(self): # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) - proj_read['requirements']['requirements.txt'] = ( - proj_read['requirements']['requirements.txt'][:-1] + '>1.10.0\n' - ) + proj_read['requirements']['requirements.txt'][-1] = 'six>1.10.0' expected_out = ( 'six must be <= 1.10.0 from upper-constraints and ' 'include the upper-constraints version' @@ -211,9 +207,8 @@ def test_project_req_not_include_uc_version(self): # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) - proj_read['requirements']['requirements.txt'] = ( - proj_read['requirements']['requirements.txt'][:-1] - + '<1.10.0,>1.10.0\n' + proj_read['requirements']['requirements.txt'][-1] = ( + 'six<1.10.0,>1.10.0' ) expected_out = ( 'six must be <= 1.10.0 from upper-constraints and ' diff --git a/openstack_requirements/tests/test_project.py b/openstack_requirements/tests/test_project.py index da672fe8e..d61831bdd 100644 --- a/openstack_requirements/tests/test_project.py +++ b/openstack_requirements/tests/test_project.py @@ -72,7 +72,7 @@ def test_pyproject_toml(self): ] """) ) - expected = {'1': 'foo', '2': 'foo\nbar'} + expected = {'1': ['foo'], '2': ['foo', 'bar']} self.assertEqual(expected, project._read_pyproject_toml_extras(root)) def test_setup_cfg(self): @@ -88,7 +88,7 @@ def test_setup_cfg(self): bar """) ) - expected = {'1': '\nfoo', '2': '\nfoo # fred\nbar'} + expected = {'1': ['foo'], '2': ['foo # fred', 'bar']} self.assertEqual(expected, project._read_setup_cfg_extras(root)) def test_none(self): diff --git a/playbooks/files/project-requirements-change.py b/playbooks/files/project-requirements-change.py index 09da5c6ca..1a997e486 100755 --- a/playbooks/files/project-requirements-change.py +++ b/playbooks/files/project-requirements-change.py @@ -103,6 +103,8 @@ def main(): print('selecting default requirements directory for normal mode') reqdir = _DEFAULT_REQS_DIR + reqdir = os.path.abspath(reqdir) + print(f'Branch: {branch}') print(f'Source: {args.src_dir}') print(f'Requirements: {reqdir}') @@ -114,14 +116,19 @@ def main(): # build a list of requirements from the global list in the # openstack/requirements project so we can match them to the changes with tempdir(): - with open(reqdir + '/global-requirements.txt') as f: - global_reqs = check.get_global_reqs(f.read()) - denylist = requirement.parse(open(reqdir + '/denylist.txt').read()) - backports_file = reqdir + '/backports.txt' + with open(os.path.join(reqdir, 'global-requirements.txt')) as fh: + global_reqs = check.get_global_reqs(fh.read()) + + with open(os.path.join(reqdir, 'denylist.txt')) as fh: + denylist = requirement.parse(fh.read()) + + backports_file = os.path.join(reqdir, 'backports.txt') if os.path.exists(backports_file): - backports = requirement.parse(open(backports_file).read()) + with open(backports_file) as fh: + backports = requirement.parse(fh.read()) else: backports = {} + cwd = os.getcwd() # build a list of requirements in the proposed change, # and check them for style violations while doing so @@ -154,6 +161,7 @@ def main(): print("*** Incompatible requirement found!") print("*** See https://docs.openstack.org/requirements/latest/") sys.exit(1) + print("Updated requirements match openstack/requirements.") diff --git a/pyproject.toml b/pyproject.toml index 0d31dcc54..13374bbe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ quote-style = "preserve" docstring-code-format = true [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "U"] +select = ["E4", "E7", "E9", "F", "UP"] # [tool.ruff.lint.per-file-ignores] # "openstack/tests/*" = ["S"] diff --git a/upper-constraints.txt b/upper-constraints.txt index 36e52d6d6..ba10b4b11 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -40,24 +40,24 @@ setproctitle===1.3.7 pytest===8.4.2 python-slugify===8.0.4 cursive===0.2.3 -oslo.service===4.3.0 +oslo.service===4.4.1 django-appconf===1.1.0 ntc_templates===8.1.0 sphinxcontrib-nwdiag===2.0.0 rbd-iscsi-client===0.1.8 requests-aws===0.1.8 alabaster===1.0.0 -pbr===7.0.2 +pbr===7.0.3 munch===4.0.0 waiting===1.5.0 attrs===25.4.0 microversion-parse===2.0.0 jwcrypto===1.5.6 Pint===0.24.4 -oslo.i18n===6.6.0 +oslo.i18n===6.7.1 jsonpath-rw-ext===1.2.2 -python-mistralclient===6.0.0 -oslo.context===6.1.0 +python-mistralclient===6.1.0 +oslo.context===6.2.0 rcssmin===1.1.2 pycadf===4.0.1 grpcio===1.76.0 @@ -66,7 +66,7 @@ fixtures===4.2.6 neutron-lib===3.22.0 XStatic-FileSaver===1.3.2.0 jaraco.functools===4.3.0 -oslo.metrics===0.13.0 +oslo.metrics===0.14.0 storage-interfaces===1.0.5 pydantic===2.12.3 persist-queue===1.0.0 @@ -99,7 +99,7 @@ google-api-core===2.27.0 requests-toolbelt===1.0.0 simplejson===3.20.2 types-paramiko===4.0.0.20250822 -python-swiftclient===4.8.0 +python-swiftclient===4.9.0 pyOpenSSL===24.2.1 typing-inspection===0.4.2 monasca-common===3.8.0 @@ -128,7 +128,7 @@ pymongo===4.15.3 python-cloudkittyclient===5.4.0 soupsieve===2.8 sqlparse===0.5.3 -oslotest===5.0.1 +oslotest===6.0.0 jsonpointer===3.0.0 defusedxml===0.7.1 opentelemetry-sdk===1.38.0 @@ -208,7 +208,7 @@ h11===0.16.0 Pygments===2.19.2 XStatic-Hogan===2.0.0.3 XStatic-objectpath===1.2.1.0 -python-manilaclient===5.6.0 +python-manilaclient===5.7.0 sphinxcontrib-serializinghtml===2.0.0 requests===2.32.5 snowballstemmer===3.0.1 @@ -235,17 +235,17 @@ daiquiri===3.4.0 influxdb===5.3.2 funcparserlib===2.0.0a0 passlib===1.7.4 -cliff===4.11.0 +cliff===4.12.0 os-brick===6.13.0 valkey===6.1.1 scp===0.15.0 lark===1.3.0 -python-zaqarclient===4.1.0 +python-zaqarclient===4.2.0 ldappool===3.0.0 hpack===4.1.0 joblib===1.5.2 google-api-python-client===2.185.0 -castellan===5.4.1 +castellan===5.5.0 oslo.versionedobjects===3.8.0 enmerkar===0.7.1 webcolors===24.11.1 @@ -276,7 +276,7 @@ gossip===2.5.0 suds-community===1.2.0 os_vif===4.2.1 qrcode===8.2 -oslo.middleware===6.6.0 +oslo.middleware===7.0.0 XStatic-mdi===1.6.50.2 pydantic_core===2.41.4 uritemplate===4.2.0 @@ -286,7 +286,7 @@ os-ken===4.0.0 ujson===5.11.0 selenium===3.141.0 pytest-subtests===0.15.0 -mistral-lib===3.4.0 +mistral-lib===3.5.0 dogtag-pki===11.2.1 XStatic-Angular-UUID===0.0.4.0 sphinxcontrib-seqdiag===3.0.0 @@ -321,7 +321,7 @@ python-monascaclient===2.8.0 opentelemetry-api===1.38.0 automaton===3.2.0 types-urllib3===1.26.25.14 -os-service-types===1.8.1 +os-service-types===1.8.2 keyring===25.6.0 elementpath===4.8.0 wsgi_intercept===1.13.1 @@ -360,11 +360,11 @@ tooz===7.0.0 idna===3.11 yamlloader===1.5.2 protobuf===6.33.0 -sushy===5.7.1 +sushy===5.8.0 python-neutronclient===11.6.0 types-setuptools===80.9.0.20250822 pika===1.3.2 -oslo.cache===3.12.0 +oslo.cache===4.0.0 WebTest===3.0.7 os-collect-config===14.0.1 edgegrid-python===2.0.2 @@ -374,7 +374,7 @@ requests-oauthlib===2.0.0 oslo.reports===3.6.0 pysnmp-lextudio===6.1.2 bitmath===1.3.3.1 -ceilometermiddleware===3.8.0 +ceilometermiddleware===3.9.0 testrepository===0.0.21 sympy===1.14.0 Logbook===1.8.2 @@ -386,7 +386,7 @@ seqdiag===3.0.0 numpy===2.2.6 msgpack===1.1.2 Sphinx===8.1.3 -oslo.config===10.0.0 +oslo.config===10.1.0 openstackdocstheme===3.5.0 osc-placement===4.7.0 rpds-py===0.28.0 @@ -418,11 +418,11 @@ WSME===0.12.1 tomli===2.3.0 oslo.upgradecheck===2.6.0 sherlock===0.4.1 -stevedore===5.5.0 +stevedore===5.6.0 botocore===1.35.99 xmltodict===1.0.2 pyasn1===0.6.0 -oslo.rootwrap===7.7.0 +oslo.rootwrap===7.8.0 Django===4.2.25 pexpect===4.9.0 cmd2===2.7.0 @@ -440,7 +440,7 @@ cotyledon===2.1.0 xattr===1.3.0 systemd-python===235 python-memcached===1.62 -openstacksdk===4.7.1 +openstacksdk===4.8.0 infi.dtypes.nqn===0.1.0 six===1.17.0 h2===4.3.0 @@ -459,7 +459,7 @@ tomlkit===0.13.3 etcd3gw===2.4.2 Flask-RESTful===0.3.10 GitPython===3.1.45 -python-ironicclient===5.13.0 +python-ironicclient===5.14.0 babel===2.17.0 XStatic===1.0.3 XStatic-Angular-FileUpload===12.2.13.0 @@ -474,7 +474,7 @@ pymemcache===4.0.0 wrapt===2.0.0 oslo.privsep===3.8.0 sphinxcontrib-apidoc===0.6.0 -oslo.policy===4.6.0 +oslo.policy===4.7.0 hvac===2.3.0 pyeclib===1.7.0 repoze.lru===0.7 @@ -512,7 +512,7 @@ docker===7.1.0 storops===1.2.11 anyio===4.11.0 XStatic-Angular-lrdragndrop===1.0.2.6 -ovsdbapp===2.13.0 +ovsdbapp===2.14.0 aniso8601===10.0.1 rjsmin===1.2.2 icalendar===6.3.1 @@ -554,7 +554,7 @@ python-linstor===1.25.3 filelock===3.20.0 python-tackerclient===2.4.0 python-heatclient===4.3.0 -oslo.utils===9.1.0 +oslo.utils===9.2.0 requests-kerberos===0.15.0 itsdangerous===2.2.0 XStatic-jquery-ui===1.13.0.1