diff --git a/.github/scripts/check_updates.py b/.github/scripts/check_updates.py new file mode 100755 index 00000000..4becfc8e --- /dev/null +++ b/.github/scripts/check_updates.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +""" +Check for PolicyEngine package updates and generate PR summary. + +This script checks PyPI for newer versions of PolicyEngine packages, +updates setup.py if needed, and generates changelog summaries. +""" + +import os +import re +import sys + +import requests +import yaml + +# Packages to track (US only - UK is updated separately) +PACKAGES = ["policyengine_us"] + +# Map package names to GitHub repos +REPO_MAP = {"policyengine_us": "PolicyEngine/policyengine-us"} + + +def parse_version(version_str): + """Parse a version string into a tuple of integers.""" + return tuple(map(int, version_str.split("."))) + + +def get_current_versions(setup_content): + """Extract current package versions from setup.py content.""" + current_versions = {} + for pkg in PACKAGES: + pattern = rf'{pkg.replace("_", "[-_]")}==([0-9]+\.[0-9]+\.[0-9]+)' + match = re.search(pattern, setup_content) + if match: + current_versions[pkg] = match.group(1) + return current_versions + + +def get_latest_versions(): + """Fetch latest versions from PyPI for all tracked packages.""" + latest_versions = {} + for pkg in PACKAGES: + pypi_name = pkg.replace("_", "-") + resp = requests.get(f"https://pypi.org/pypi/{pypi_name}/json") + if resp.status_code == 200: + latest_versions[pkg] = resp.json()["info"]["version"] + return latest_versions + + +def find_updates(current_versions, latest_versions): + """Compare current and latest versions to find updates.""" + updates = {} + for pkg in PACKAGES: + if pkg in current_versions and pkg in latest_versions: + if current_versions[pkg] != latest_versions[pkg]: + updates[pkg] = { + "old": current_versions[pkg], + "new": latest_versions[pkg], + } + return updates + + +def update_setup_content(setup_content, updates): + """Update setup.py content with new versions.""" + new_content = setup_content + for pkg, versions in updates.items(): + pattern = rf'({pkg.replace("_", "[-_]")}==)[0-9]+\.[0-9]+\.[0-9]+' + new_content = re.sub(pattern, rf'\g<1>{versions["new"]}', new_content) + return new_content + + +def fetch_changelog(pkg): + """Fetch changelog from GitHub for a package.""" + repo = REPO_MAP.get(pkg) + if not repo: + return None + url = f"https://raw.githubusercontent.com/{repo}/main/changelog.yaml" + resp = requests.get(url) + if resp.status_code == 200: + return yaml.safe_load(resp.text) + return None + + +def get_changes_between_versions(changelog, old_version, new_version): + """Extract changelog entries between old and new versions.""" + if not changelog: + return [] + + old_v = parse_version(old_version) + new_v = parse_version(new_version) + + entries_with_versions = [] + current_version = None + + for entry in changelog: + if "version" in entry: + current_version = parse_version(entry["version"]) + elif current_version and "bump" in entry: + bump = entry["bump"] + major, minor, patch = current_version + if bump == "major": + current_version = (major + 1, 0, 0) + elif bump == "minor": + current_version = (major, minor + 1, 0) + elif bump == "patch": + current_version = (major, minor, patch + 1) + + if current_version: + entries_with_versions.append((current_version, entry)) + + relevant_entries = [] + for version, entry in entries_with_versions: + if old_v < version <= new_v: + relevant_entries.append(entry) + + return relevant_entries + + +def format_changes(entries): + """Format changelog entries as markdown.""" + added = [] + changed = [] + fixed = [] + removed = [] + + for entry in entries: + changes = entry.get("changes", {}) + added.extend(changes.get("added", [])) + changed.extend(changes.get("changed", [])) + fixed.extend(changes.get("fixed", [])) + removed.extend(changes.get("removed", [])) + + sections = [] + if added: + sections.append( + "### Added\n" + "\n".join(f"- {item}" for item in added) + ) + if changed: + sections.append( + "### Changed\n" + "\n".join(f"- {item}" for item in changed) + ) + if fixed: + sections.append( + "### Fixed\n" + "\n".join(f"- {item}" for item in fixed) + ) + if removed: + sections.append( + "### Removed\n" + "\n".join(f"- {item}" for item in removed) + ) + + return ( + "\n\n".join(sections) if sections else "No detailed changes available." + ) + + +def generate_summary(updates): + """Generate PR summary with version table and changelogs.""" + summary_parts = [] + + # Version table + version_table = "| Package | Old Version | New Version |\n|---------|-------------|-------------|\n" + for pkg, versions in updates.items(): + version_table += f"| {pkg} | {versions['old']} | {versions['new']} |\n" + summary_parts.append(version_table) + + # Changelog for each package + for pkg, versions in updates.items(): + changelog = fetch_changelog(pkg) + if changelog: + entries = get_changes_between_versions( + changelog, versions["old"], versions["new"] + ) + if entries: + formatted = format_changes(entries) + summary_parts.append( + f"## What Changed ({pkg} {versions['old']} → {versions['new']})\n\n{formatted}" + ) + else: + summary_parts.append( + f"## What Changed ({pkg} {versions['old']} → {versions['new']})\n\nNo changelog entries found between these versions." + ) + + return "\n\n".join(summary_parts) + + +def generate_changelog_entry(updates): + """Generate changelog entry for this repo.""" + new_version = updates["policyengine_us"]["new"] + return f"""- bump: patch + changes: + changed: + - Update PolicyEngine US to {new_version} +""" + + +def write_github_output(key, value): + """Write output to GitHub Actions output file.""" + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + f.write(f"{key}={value}\n") + + +def main(): + """Main entry point for the script.""" + # Read current versions from setup.py + with open("setup.py", "r") as f: + setup_content = f.read() + + current_versions = get_current_versions(setup_content) + print(f"Current versions: {current_versions}") + + # Get latest versions from PyPI + latest_versions = get_latest_versions() + print(f"Latest versions: {latest_versions}") + + # Check for updates + updates = find_updates(current_versions, latest_versions) + + if not updates: + print("No updates available.") + write_github_output("has_updates", "false") + return 0 + + print(f"Updates available: {updates}") + + # Update setup.py + new_setup_content = update_setup_content(setup_content, updates) + with open("setup.py", "w") as f: + f.write(new_setup_content) + + # Generate and save PR summary + full_summary = generate_summary(updates) + with open("pr_summary.md", "w") as f: + f.write(full_summary) + + # Create changelog entry + changelog_entry = generate_changelog_entry(updates) + with open("changelog_entry.yaml", "w") as f: + f.write(changelog_entry) + + # Set outputs + write_github_output("has_updates", "true") + updates_str = ", ".join( + f"{pkg} to {v['new']}" for pkg, v in updates.items() + ) + write_github_output("updates_summary", updates_str) + + print("Updates prepared successfully!") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/create_pr.sh b/.github/scripts/create_pr.sh new file mode 100755 index 00000000..3ea0fcf8 --- /dev/null +++ b/.github/scripts/create_pr.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Create or update a PR for the weekly policyengine-us update. +# +# This script reads pr_summary.md and creates/updates a PR with the +# formatted body. +# +# Usage: ./create_pr.sh +# +# Environment variables: +# GH_TOKEN - GitHub token for authentication (required) +# +set -e + +BRANCH_NAME="bot/weekly-us-update" +PR_TITLE="Weekly policyengine-us update" + +# Build PR body with summary +if [ ! -f "pr_summary.md" ]; then + echo "Error: pr_summary.md not found" + exit 1 +fi + +PR_SUMMARY=$(cat pr_summary.md) + +PR_BODY="## Summary + +Automated weekly update of policyengine-us. + +Related to #1178 + +## Version Updates + +${PR_SUMMARY} + +--- +Generated automatically by GitHub Actions" + +# Check if PR already exists +EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number --jq '.[0].number' 2>/dev/null || echo "") + +if [ -n "$EXISTING_PR" ]; then + echo "PR #$EXISTING_PR already exists, updating it" + gh pr edit "$EXISTING_PR" --body "$PR_BODY" +else + echo "Creating new PR" + gh pr create \ + --title "$PR_TITLE" \ + --body "$PR_BODY" +fi diff --git a/.github/scripts/test_check_updates.py b/.github/scripts/test_check_updates.py new file mode 100644 index 00000000..7566f65f --- /dev/null +++ b/.github/scripts/test_check_updates.py @@ -0,0 +1,180 @@ +"""Unit tests for check_updates.py""" + +import pytest + +from check_updates import ( + find_updates, + format_changes, + generate_changelog_entry, + get_changes_between_versions, + get_current_versions, + parse_version, + update_setup_content, +) + + +class TestParseVersion: + def test_simple_version(self): + assert parse_version("1.2.3") == (1, 2, 3) + + def test_large_numbers(self): + assert parse_version("10.20.30") == (10, 20, 30) + + def test_zero_version(self): + assert parse_version("0.0.1") == (0, 0, 1) + + +class TestGetCurrentVersions: + def test_extracts_underscore_version(self): + setup_content = """ +install_requires=[ + "policyengine_us==1.2.3", +] +""" + versions = get_current_versions(setup_content) + assert versions == {"policyengine_us": "1.2.3"} + + def test_extracts_hyphen_version(self): + setup_content = """ +install_requires=[ + "policyengine-us==4.5.6", +] +""" + versions = get_current_versions(setup_content) + assert versions == {"policyengine_us": "4.5.6"} + + def test_no_match_returns_empty(self): + setup_content = """ +install_requires=[ + "some_other_package==1.0.0", +] +""" + versions = get_current_versions(setup_content) + assert versions == {} + + +class TestFindUpdates: + def test_finds_update_when_versions_differ(self): + current = {"policyengine_us": "1.0.0"} + latest = {"policyengine_us": "1.1.0"} + updates = find_updates(current, latest) + assert updates == {"policyengine_us": {"old": "1.0.0", "new": "1.1.0"}} + + def test_no_update_when_versions_match(self): + current = {"policyengine_us": "1.0.0"} + latest = {"policyengine_us": "1.0.0"} + updates = find_updates(current, latest) + assert updates == {} + + def test_handles_missing_package(self): + current = {} + latest = {"policyengine_us": "1.0.0"} + updates = find_updates(current, latest) + assert updates == {} + + +class TestUpdateSetupContent: + def test_updates_version_with_underscore(self): + setup_content = "policyengine_us==1.0.0" + updates = {"policyengine_us": {"old": "1.0.0", "new": "2.0.0"}} + result = update_setup_content(setup_content, updates) + assert result == "policyengine_us==2.0.0" + + def test_updates_version_with_hyphen(self): + setup_content = "policyengine-us==1.0.0" + updates = {"policyengine_us": {"old": "1.0.0", "new": "2.0.0"}} + result = update_setup_content(setup_content, updates) + assert result == "policyengine-us==2.0.0" + + def test_preserves_other_content(self): + setup_content = """install_requires=[ + "flask==2.0.0", + "policyengine_us==1.0.0", + "requests==2.28.0", +]""" + updates = {"policyengine_us": {"old": "1.0.0", "new": "1.5.0"}} + result = update_setup_content(setup_content, updates) + assert "flask==2.0.0" in result + assert "policyengine_us==1.5.0" in result + assert "requests==2.28.0" in result + + +class TestGetChangesBetweenVersions: + def test_returns_empty_for_none_changelog(self): + result = get_changes_between_versions(None, "1.0.0", "2.0.0") + assert result == [] + + def test_filters_entries_between_versions(self): + changelog = [ + {"version": "1.0.0", "changes": {"added": ["Initial"]}}, + {"bump": "patch", "changes": {"fixed": ["Bug fix"]}}, + {"bump": "minor", "changes": {"added": ["New feature"]}}, + {"bump": "major", "changes": {"changed": ["Breaking change"]}}, + ] + # Version progression: 1.0.0 -> 1.0.1 -> 1.1.0 -> 2.0.0 + result = get_changes_between_versions(changelog, "1.0.0", "1.1.0") + # Should include 1.0.1 and 1.1.0, but not 1.0.0 or 2.0.0 + assert len(result) == 2 + assert result[0]["changes"]["fixed"] == ["Bug fix"] + assert result[1]["changes"]["added"] == ["New feature"] + + def test_returns_empty_for_same_version(self): + changelog = [ + {"version": "1.0.0", "changes": {"added": ["Initial"]}}, + ] + result = get_changes_between_versions(changelog, "1.0.0", "1.0.0") + assert result == [] + + +class TestFormatChanges: + def test_formats_all_categories(self): + entries = [ + { + "changes": { + "added": ["Feature A"], + "changed": ["Update B"], + "fixed": ["Fix C"], + "removed": ["Remove D"], + } + } + ] + result = format_changes(entries) + assert "### Added" in result + assert "- Feature A" in result + assert "### Changed" in result + assert "- Update B" in result + assert "### Fixed" in result + assert "- Fix C" in result + assert "### Removed" in result + assert "- Remove D" in result + + def test_combines_multiple_entries(self): + entries = [ + {"changes": {"added": ["Feature 1"]}}, + {"changes": {"added": ["Feature 2"]}}, + ] + result = format_changes(entries) + assert "- Feature 1" in result + assert "- Feature 2" in result + + def test_returns_default_for_empty_entries(self): + result = format_changes([]) + assert result == "No detailed changes available." + + def test_handles_missing_change_categories(self): + entries = [{"changes": {"added": ["Only added"]}}] + result = format_changes(entries) + assert "### Added" in result + assert "### Changed" not in result + assert "### Fixed" not in result + assert "### Removed" not in result + + +class TestGenerateChangelogEntry: + def test_generates_correct_format(self): + updates = {"policyengine_us": {"old": "1.0.0", "new": "1.5.0"}} + result = generate_changelog_entry(updates) + assert "- bump: patch" in result + assert "changes:" in result + assert "changed:" in result + assert "Update PolicyEngine US to 1.5.0" in result diff --git a/.github/workflows/weekly-update.yaml b/.github/workflows/weekly-update.yaml new file mode 100644 index 00000000..7fe2d079 --- /dev/null +++ b/.github/workflows/weekly-update.yaml @@ -0,0 +1,54 @@ +name: Weekly US Household API Update + +on: + schedule: + # Every Wednesday at 5 PM EST (10 PM UTC) + - cron: "0 22 * * 3" + # Allow manual trigger + workflow_dispatch: + +jobs: + update-packages: + name: Update PolicyEngine Packages + runs-on: ubuntu-latest + if: github.repository == 'PolicyEngine/policyengine-household-api' + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + ref: main + token: ${{ secrets.POLICYENGINE_GITHUB }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install requests pyyaml + + - name: Check for updates and generate summary + id: check_updates + run: python .github/scripts/check_updates.py + + - name: No updates available + if: steps.check_updates.outputs.has_updates != 'true' + run: | + echo "::notice::No package updates available. policyengine_us is already at the latest version." + + - name: Commit and push changes + if: steps.check_updates.outputs.has_updates == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b bot/weekly-us-update + git add setup.py changelog_entry.yaml + git commit -m "Update policyengine-us (${{ steps.check_updates.outputs.updates_summary }})" + git push -f origin bot/weekly-us-update + + - name: Create Pull Request + if: steps.check_updates.outputs.has_updates == 'true' + env: + GH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB }} + run: .github/scripts/create_pr.sh diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29b..cf3e08f6 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,4 @@ +- bump: minor + changes: + added: + - Add GitHub Actions workflow for weekly policyengine-us updates with changelog summary generation.