Improve min_deps_check.py script, invoke automatically

The min_deps_check.py script now acts on its own advice and will update
requirements-minimum.txt automatically by invoking `pip-compile`. The
update_pinned_dependencies.sh script now invokes the min_deps_check.py
script.

Author: mx-moth

scripts/min_deps_check.py

@@ -1,21 +1,29 @@
 #!/usr/bin/env python
 """
-Fetch from conda database all available versions of the emsarray dependencies and their
-publication date. Compare it against continuous-integration/min-deps.yaml to verify the
-policy on obsolete dependencies is being followed. Print a pretty report :)
+Fetch from PyPI all available versions of the emsarray dependencies and their
+publication date. Compare it against continuous-integration/requirements-minimum.txt
+to verify the policy on obsolete dependencies is being followed.
+Update the pinned dependencies using `pip-compile`.
 
 Based heavily on `min_deps_check.py` from xarray
 but rewritten to pull requirements from a Python requirements.txt and available versions from PyPI.
 
 Needs the following extra deps installed:
 
-    $ pip3 install packaging requests python-dateutil
+    $ pip3 install packaging requests python-dateutil pip-tools
+
+This is automatically run as part of the `scripts/update_pinned_dependencies.sh` script.
 
 See also
 ========
 https://github.com/pydata/xarray/blob/v2024.06.0/ci/min_deps_check.py
 """
+import dataclasses
 import datetime
+import enum
+import itertools
+import shlex
+import subprocess
 import sys
 from collections.abc import Iterator
 
@@ -24,6 +32,23 @@
 import requests
 from dateutil.relativedelta import relativedelta
 
+
+class VersionStatus(enum.StrEnum):
+    older = '<'
+    current = '~='
+    newer = '>'
+
+
+@dataclasses.dataclass
+class Dependency:
+    package_name: str
+    requirements_version: packaging.version.Version
+    requirements_date: datetime.date | None
+    policy_version: packaging.version.Version
+    policy_date: datetime.date | None
+    status: VersionStatus
+
+
 IGNORE_DEPS: set[str] = {
     'certifi',
     'pytz',
@@ -49,13 +74,12 @@ def warning(msg: str) -> None:
 
 
 def parse_requirements(
-    fname: str
+    filename: str
 ) -> Iterator[tuple[str, packaging.specifiers.Specifier, packaging.version.Version]]:
-    """Load requirements/min-all-deps.yml
-
-    Yield (package name, major version, minor version, patch version)
     """
-    for line_number, line in enumerate(open(fname), start=1):
+    Parse a requirements file, yield (package name, specifier, version) for each requirement.
+    """
+    for line_number, line in enumerate(open(filename), start=1):
         if '#' in line:
             line = line[:line.index('#')]
         line = line.strip()
@@ -71,9 +95,6 @@ def parse_requirements(
             continue
         specifier = next(iter(requirement.specifier))
 
-        if specifier.operator != '~=':
-            error(f"Specificity for dependency {requirement.name} should be '~='")
-
         version = packaging.version.parse(specifier.version)
         if version.micro is None:
             warning(
@@ -110,7 +131,7 @@ def process_pkg(
     pkg: str,
     specifier: packaging.specifiers.Specifier,
     version: packaging.version.Version,
-) -> tuple[str, str, str, str, str, str]:
+) -> Dependency:
     """Compare package version from requirements file to available versions in conda.
     Return row to build pandas dataframe:
 
@@ -148,7 +169,7 @@ def process_pkg(
     policy_version = packaging.version.parse(policy_specifier.version)
 
     # Find the release date of the policy version
-    policy_release_date = min(
+    policy_date = min(
         (
             release_date
             for release_version, release_date
@@ -158,17 +179,17 @@ def process_pkg(
     )
 
     if version in policy_specifier:
-        status = "~="
+        status = VersionStatus.current
     else:
         if version < policy_version:
-            status = '<'
+            status = VersionStatus.older
             warning(
                 f"Requirement {pkg} {version} was published on {req_published:%Y-%m-%d} "
                 f"which is older than the required {policy_months} months of support. "
                 f"Minimum policy supported version is {pkg}{policy_specifier}."
             )
         elif version > policy_version:
-            status = '> (!)'
+            status = VersionStatus.newer
             if req_published is None:
                 error(
                     f"Package version is newer than policy version. "
@@ -186,26 +207,85 @@ def process_pkg(
                     f"Update requirement to {pkg}{policy_specifier}."
                 )
 
-    return (
-        pkg,
-        str(version),
-        req_published.strftime("%Y-%m-%d") if req_published else "-",
-        str(policy_version),
-        policy_release_date.strftime("%Y-%m-%d") if policy_release_date else "-",
-        status,
+    return Dependency(
+        package_name=pkg,
+        requirements_version=version,
+        requirements_date=req_published,
+        policy_version=policy_version,
+        policy_date=policy_date,
+        status=status,
     )
 
 
 def main() -> None:
+    if len(sys.argv) < 2:
+        print(f"Usage: {sys.argv[0]} continuous-integration/requirements-minimum.txt")
+        sys.exit(1)
+
     requirements_file = sys.argv[1]
-    rows = [process_pkg(pkg, specifier, version) for pkg, specifier, version in parse_requirements(requirements_file)]
+    dependencies = [
+        process_pkg(pkg, specifier, version)
+        for pkg, specifier, version in parse_requirements(requirements_file)
+    ]
 
     print()
     print("Package              Required                Status Policy                 ")
     print("-------------------- ----------------------- ------ -----------------------")
     fmt = "{0:20} {1:10} ({2:10}) {5:^6} {3:10} ({4:10})"
-    for row in rows:
-        print(fmt.format(*row))
+    for d in dependencies:
+        requirements_date = (
+            d.requirements_date.strftime("%Y-%m-%d")
+            if d.requirements_date is not None
+            else "-"
+        )
+        policy_date = (
+            d.policy_date.strftime("%Y-%m-%d")
+            if d.policy_date is not None
+            else "-"
+        )
+        print(
+            f"{d.package_name:20} {d.requirements_version!s:10} ({requirements_date:10}) "
+            f"{d.status:^6} {d.policy_version!s:10} ({policy_date:10})"
+        )
+
+    upgrade_args = list(itertools.chain.from_iterable(
+        ['--upgrade-package', f'{d.package_name}~={d.policy_version}']
+        for d in dependencies
+        if d.status is VersionStatus.older
+    ))
+    maintain_args = list(itertools.chain.from_iterable(
+        ['--upgrade-package', f'{d.package_name}~={d.requirements_version}']
+        for d in dependencies
+        if d.status in {VersionStatus.current, VersionStatus.newer}
+    ))
+    if upgrade_args:
+        ignored_args = list(itertools.chain.from_iterable(
+            ['--unsafe-package', ignored]
+            for ignored in IGNORE_DEPS
+        ))
+        cmd = [
+            'pip-compile',
+            '--quiet',
+            '--extra', 'complete',
+            '--strip-extras',
+            '--unsafe-package', 'emsarray',
+            '--no-allow-unsafe',
+            # '--no-header',
+            # '--no-annotate',
+            '--output-file', requirements_file,
+        ] + upgrade_args + maintain_args + ignored_args + [
+            'pyproject.toml',
+        ]
+        print('$', shlex.join(cmd))
+        subprocess.check_call(cmd)
+        cmd = [
+            'sed',
+            '-i',
+            's/==/~=/',
+            requirements_file,
+        ]
+        print('$', shlex.join(cmd))
+        subprocess.check_call(cmd)
 
     if errors:
         print("\nErrors:")

scripts/update_pinned_dependencies.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-PYTHON_VERSIONS=('3.10' '3.11' '3.12')
+PYTHON_VERSIONS=('3.11' '3.12' '3.13')
 HERE="$( cd -- "$( realpath -- "$( dirname -- "$0" )" )" && pwd )"
 PROJECT_ROOT="$( dirname "$HERE" )"
 
@@ -11,28 +11,53 @@ cd "$PROJECT_ROOT"
 conda_venv_root=$( mktemp -d emsarray-conda-environments.XXXXXXX )
 echo "Working in ${conda_venv_root}"
 
-for version in "${PYTHON_VERSIONS[@]}" ; do
-	requirements_file="./continuous-integration/requirements-${version}.txt"
-	echo "Updating $requirements_file"
-
-	conda_prefix="${conda_venv_root}/py${version}"
-	conda create \
-		--yes --quiet \
-		--prefix="${conda_prefix}" \
-		--no-default-packages
-	conda install \
-		--yes \
-		--prefix="${conda_prefix}" \
-		"python=${version}" \
-		pip-tools
-	conda run \
-		--prefix="${conda_prefix}" \
-		pip-compile \
-			--upgrade \
-			--extra="testing" \
-			--output-file="${requirements_file}" \
-			setup.cfg
-	conda env remove --yes --prefix="${conda_prefix}"
-done
+version="${PYTHON_VERSIONS[0]}"
+requirements_file="./continuous-integration/requirements-minimum.txt"
+echo "Updating $requirements_file"
+conda_prefix="${conda_venv_root}/py-min"
+conda create \
+	--yes --quiet \
+	--prefix="${conda_prefix}" \
+	--no-default-packages
+conda install \
+	--yes \
+	--prefix="${conda_prefix}" \
+	--channel conda-forge \
+	"python=${version}" pip
+conda run \
+	--prefix="${conda_prefix}" \
+	pip install pip-tools packaging requests python-dateutil pip-tools
+conda run \
+	--prefix="${conda_prefix}" \
+	python3 ./scripts/min_deps_check.py "$requirements_file"
+conda env remove --yes --prefix="${conda_prefix}"
+
+# for version in "${PYTHON_VERSIONS[@]}" ; do
+# 	requirements_file="./continuous-integration/requirements-${version}.txt"
+# 	echo "Updating $requirements_file"
+# 
+# 	conda_prefix="${conda_venv_root}/py${version}"
+# 	conda create \
+# 		--yes --quiet \
+# 		--prefix="${conda_prefix}" \
+# 		--no-default-packages
+# 	conda install \
+# 		--yes \
+# 		--prefix="${conda_prefix}" \
+# 		--channel conda-forge \
+# 		"python=${version}" \
+# 		pip-tools
+# 	conda run \
+# 		--prefix="${conda_prefix}" \
+# 		pip-compile \
+# 			--upgrade \
+# 			--extra="testing" \
+# 			--output-file="${requirements_file}" \
+# 			--unsafe-package emsarray \
+# 			--no-allow-unsafe \
+# 			pyproject.toml
+# 	conda env remove --yes --prefix="${conda_prefix}"
+# done
+
 
 echo rm -rf "$conda_venv_root"