diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f31f242..0d1e9e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,9 @@ jobs: id: action-run continue-on-error: true with: + policy: policy.yaml environment-paths: ${{ matrix.env-paths }} + today: 2024-12-20 - name: detect outcome if: always() shell: bash -l {0} diff --git a/.gitignore b/.gitignore index 5b48d8a..644c670 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .prettier_cache __pycache__/ +.coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a63125..354db93 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -9,7 +9,7 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.7 + rev: v0.12.11 hooks: - id: ruff args: ["--fix"] diff --git a/README.md b/README.md index 0939659..69ac401 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,35 @@ Check that the minimum dependency versions follow `xarray`'s policy. ## Usage -To use the `minimum-dependency-versions` action in workflows, simply add a new step: +To use the `minimum-dependency-versions` action in workflows, create a policy file (`policy.yaml`): + +```yaml +channels: + - conda-forge +platforms: + - noarch + - linux-64 +policy: + # policy in months + # Example is xarray's values + packages: + python: 30 + numpy: 18 + default: 12 + overrides: + # override the policy for specific packages + package3: 0.3.1 + # these packages are completely ignored + exclude: + - package1 + - package2 + - ... + # these packages don't fail the CI, but will be printed in the report as a warning + ignored_violations: + - package4 +``` + +then add a new step to CI: ```yaml jobs: @@ -14,6 +42,7 @@ jobs: ... - uses: xarray-contrib/minimum-dependency-versions@version with: + policy: policy.yaml environment-paths: path/to/env.yaml ``` diff --git a/action.yaml b/action.yaml index 96916b8..36af420 100644 --- a/action.yaml +++ b/action.yaml @@ -2,11 +2,21 @@ name: "minimum-dependency-versions" description: >- Check that the minimum dependency versions follow `xarray`'s policy. inputs: + policy: + description: >- + The path to the policy to follow + required: true + type: string environment-paths: description: >- The paths to the environment files - required: True + required: true type: list + today: + description: >- + Time machine for testing + required: false + type: string outputs: {} runs: @@ -16,12 +26,16 @@ runs: - name: install dependencies shell: bash -l {0} run: | + echo "::group::Install dependencies" python -m pip install -r ${{ github.action_path }}/requirements.txt + echo "::endgroup::" - name: analyze environments shell: bash -l {0} env: COLUMNS: 120 FORCE_COLOR: 3 - INPUT: ${{ inputs.environment-paths }} + POLICY_PATH: ${{ inputs.policy }} + ENVIRONMENT_PATHS: ${{ inputs.environment-paths }} + TODAY: ${{ inputs.today }} run: | - python ${{ github.action_path }}/minimum_versions.py $(echo $INPUT) + python minimum_versions.py --today="$TODAY" --policy="$POLICY_PATH" $ENVIRONMENT_PATHS diff --git a/envs/env2.yaml b/envs/env2.yaml index 8ac6e2d..b3d620f 100644 --- a/envs/env2.yaml +++ b/envs/env2.yaml @@ -6,3 +6,4 @@ dependencies: - xarray=2023.10.0 - dask=2023.10.0 - distributed=2023.10.0 + - pydap=3.5.1 diff --git a/minimum_versions.py b/minimum_versions.py index 2488295..e29074a 100644 --- a/minimum_versions.py +++ b/minimum_versions.py @@ -5,6 +5,7 @@ import sys from dataclasses import dataclass, field +import jsonschema import rich_click as click import yaml from dateutil.relativedelta import relativedelta @@ -18,27 +19,62 @@ click.rich_click.SHOW_ARGUMENTS = True -channels = ["conda-forge"] -platforms = ["noarch", "linux-64"] -ignored_packages = [ - "coveralls", - "hypothesis", - "pip", - "pytest", - "pytest-cov", - "pytest-env", - "pytest-mypy-plugins", - "pytest-timeout", - "pytest-xdist", -] + +schema = { + "type": "object", + "properties": { + "channels": {"type": "array", "items": {"type": "string"}}, + "platforms": {"type": "array", "items": {"type": "string"}}, + "policy": { + "type": "object", + "properties": { + "packages": { + "type": "object", + "patternProperties": { + "^[a-z][-a-z_]*$": {"type": "integer", "minimum": 1} + }, + "additionalProperties": False, + }, + "default": {"type": "integer", "minimum": 1}, + "overrides": { + "type": "object", + "patternProperties": { + "^[a-z][-a-z_]*": {"type": "string", "format": "date"} + }, + "additionalProperties": False, + }, + "exclude": {"type": "array", "items": {"type": "string"}}, + "ignored_violations": { + "type": "array", + "items": {"type": "string", "pattern": "^[a-z][-a-z_]*$"}, + }, + }, + "required": [ + "packages", + "default", + "overrides", + "exclude", + "ignored_violations", + ], + }, + }, + "required": ["channels", "platforms", "policy"], +} @dataclass class Policy: package_months: dict default_months: int + + channels: list[str] = field(default_factory=list) + platforms: list[str] = field(default_factory=list) + overrides: dict[str, Version] = field(default_factory=dict) + ignored_violations: list[str] = field(default_factory=list) + exclude: list[str] = field(default_factory=list) + def minimum_version(self, today, package_name, releases): if (override := self.overrides.get(package_name)) is not None: return find_release(releases, version=override) @@ -117,6 +153,28 @@ def parse_environment(text): return specs, warnings +def parse_policy(file): + policy = yaml.safe_load(file) + try: + jsonschema.validate(instance=policy, schema=schema) + except jsonschema.ValidationError as e: + raise jsonschema.ValidationError( + f"Invalid policy definition: {str(e)}" + ) from None + + package_policy = policy["policy"] + + return Policy( + channels=policy["channels"], + platforms=policy["platforms"], + exclude=package_policy["exclude"], + package_months=package_policy["packages"], + default_months=package_policy["default"], + ignored_violations=package_policy["ignored_violations"], + overrides=package_policy["overrides"], + ) + + def is_preview(version): candidates = {"rc", "b", "a"} @@ -175,11 +233,15 @@ def lookup_spec_release(spec, releases): return releases[spec.name][version] -def compare_versions(environments, policy_versions): +def compare_versions(environments, policy_versions, ignored_violations): status = {} for env, specs in environments.items(): env_status = any( - spec.version > policy_versions[spec.name].version for spec in specs + ( + spec.name not in ignored_violations + and spec.version > policy_versions[spec.name].version + ) + for spec in specs ) status[env] = env_status return status @@ -194,7 +256,7 @@ def version_comparison_symbol(required, policy): return "=" -def format_bump_table(specs, policy_versions, releases, warnings): +def format_bump_table(specs, policy_versions, releases, warnings, ignored_violations): table = Table( Column("Package", width=20), Column("Required", width=8), @@ -221,7 +283,10 @@ def format_bump_table(specs, policy_versions, releases, warnings): required_date = lookup_spec_release(spec, releases).timestamp status = version_comparison_symbol(required_version, policy_version) - style = styles[status] + if status == ">" and spec.name in ignored_violations: + style = warning_style + else: + style = styles[status] table.add_row( spec.name, @@ -255,15 +320,26 @@ def format_bump_table(specs, policy_versions, releases, warnings): return grid +def parse_date(string): + if not string: + return None + + return datetime.datetime.strptime(string, "%Y-%m-%d").date() + + @click.command() @click.argument( "environment_paths", type=click.Path(exists=True, readable=True, path_type=pathlib.Path), nargs=-1, ) -def main(environment_paths): +@click.option("--today", type=parse_date, default=None) +@click.option("--policy", "policy_file", type=click.File(mode="r"), required=True) +def main(today, policy_file, environment_paths): console = Console() + policy = parse_policy(policy_file) + parsed_environments = { path.stem: parse_environment(path.read_text()) for path in environment_paths } @@ -272,7 +348,7 @@ def main(environment_paths): env: dict(warnings_) for env, (_, warnings_) in parsed_environments.items() } environments = { - env: [spec for spec in specs if spec.name not in ignored_packages] + env: [spec for spec in specs if spec.name not in policy.exclude] for env, (specs, _) in parsed_environments.items() } @@ -280,22 +356,14 @@ def main(environment_paths): dict.fromkeys(spec.name for spec in concat(environments.values())) ) - policy_months = { - "python": 30, - "numpy": 18, - } - policy_months_default = 12 - overrides = {} - - policy = Policy( - policy_months, default_months=policy_months_default, overrides=overrides - ) - gateway = Gateway() - query = gateway.query(channels, platforms, all_packages, recursive=False) + query = gateway.query( + policy.channels, policy.platforms, all_packages, recursive=False + ) records = asyncio.run(query) - today = datetime.date.today() + if today is None: + today = datetime.date.today() package_releases = pipe( records, concat, @@ -307,13 +375,19 @@ def main(environment_paths): package_releases, curry(find_policy_versions, policy, today), ) - status = compare_versions(environments, policy_versions) + status = compare_versions(environments, policy_versions, policy.ignored_violations) release_lookup = { n: {r.version: r for r in releases} for n, releases in package_releases.items() } grids = { - env: format_bump_table(specs, policy_versions, release_lookup, warnings[env]) + env: format_bump_table( + specs, + policy_versions, + release_lookup, + warnings[env], + policy.ignored_violations, + ) for env, specs in environments.items() } root_grid = Table.grid() diff --git a/policy.yaml b/policy.yaml new file mode 100644 index 0000000..59a629b --- /dev/null +++ b/policy.yaml @@ -0,0 +1,28 @@ +channels: + - conda-forge +platforms: + - noarch + - linux-64 +policy: + # all packages in months + packages: + python: 30 + numpy: 18 + default: 12 + # overrides for the policy + overrides: {} + # these packages are completely ignored + exclude: + - coveralls + - pip + - pytest + - pytest-asyncio + - pytest-cov + - pytest-env + - pytest-mypy-plugins + - pytest-timeout + - pytest-xdist + - pytest-hypothesis + # these packages don't fail the CI, but will be printed in the report + ignored_violations: + - pydap diff --git a/requirements.txt b/requirements.txt index 3a72cd4..21cbe21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ rich-click cytoolz pyyaml python-dateutil +jsonschema +rfc3339-validator diff --git a/test_script.py b/test_script.py index 32d4db3..22a62f9 100644 --- a/test_script.py +++ b/test_script.py @@ -44,7 +44,7 @@ def test_spec_parse(text, expected_spec, expected_name, expected_warnings): ), ( "scipy", - Policy({"numpy": 6}, 8, {"scipy": Version("1.1.1")}), + Policy({"numpy": 6}, 8, overrides={"scipy": Version("1.1.1")}), dt.date(2024, 9, 5), Release(Version("1.1.1"), 0, dt.datetime(2023, 12, 1)), ),