diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c18594c..bb6839bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,22 +142,17 @@ jobs: pip install . pip install ".[test]" - - name: Cache modflow6 examples - id: cache-examples - uses: actions/cache@v3 - with: - path: modflow6-examples/examples - key: modflow6-examples-${{ hashFiles('modflow6-examples/data/**') }} - - name: Install extra Python packages - if: steps.cache-examples.outputs.cache-hit != 'true' working-directory: modflow6-examples/etc run: | pip install -r requirements.pip.txt pip install -r requirements.usgs.txt + - name: Update flopy + working-directory: modflow6/autotest + run: python update_flopy.py + - name: Build modflow6 example models - if: steps.cache-examples.outputs.cache-hit != 'true' working-directory: modflow6-examples/etc run: python ci_build_files.py @@ -167,6 +162,7 @@ jobs: BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} GITHUB_TOKEN: ${{ github.token }} + GITHUB_USER: "MODFLOW-USGS" run: pytest -v -n auto --durations 0 --ignore modflow_devtools/test/test_download.py - name: Run network-dependent tests @@ -179,4 +175,5 @@ jobs: BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} GITHUB_TOKEN: ${{ github.token }} + GITHUB_USER: "MODFLOW-USGS" run: pytest -v -n auto --durations 0 modflow_devtools/test/test_download.py \ No newline at end of file diff --git a/.github/workflows/make_package.yml b/.github/workflows/make_package.yml new file mode 100644 index 00000000..d756ab4d --- /dev/null +++ b/.github/workflows/make_package.yml @@ -0,0 +1,152 @@ +name: Make package +on: + # workflow_call trigger lets this workflow be called from elsewhere + workflow_call: + inputs: + branch: + description: Branch to release from. + required: true + type: string + cliff_config: + description: Path of the git-cliff config file relative to the project root. + required: false + default: cliff.toml + type: string + cumulative_changelog: + description: Path of the cumulative changelog file relative to the project root. + required: false + default: HISTORY.md + type: string + package_name: + # currently assumes module dir is in project root, + # and module name is the same as package name with + # hyphens swapped for underscores + description: Name of the Python package to release. + required: true + type: string + python: + description: Python version to build the package with. + required: true + default: '3.8' + type: string + run_tests: + # currently assumes tests are in autotest/ + description: Run tests after building binaries. + required: false + type: boolean + default: true + trunk_branch: + description: Name of the trunk branch (e.g. 'main' or 'master'). + required: false + default: main + type: string + version: + description: Version number to use for release. + required: true + type: string + +jobs: + prep: + name: Prepare release + runs-on: ubuntu-22.04 + if: github.event_name == 'push' && github.ref_name != inputs.trunk_branch + permissions: + contents: write + pull-requests: write + defaults: + run: + shell: bash + steps: + + - name: Checkout release branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Install Python build/test dependencies + run: | + pip install --upgrade pip + pip install black build twine + pip install . + pip install ".[lint, test]" + + - name: Update version + id: version + run: | + ref="${{ github.ref_name }}" + version="${ref#"v"}" + package="${{ inputs.package_name }}" + # assume module name is the same as package + # name with hyphens swapped for underscores + module="${package//-/_}" + python scripts/update_version.py -v "$version" + black -v $module/version.py + python -c "import $module; print('Version: ', $module.__version__)" + echo "version=$version" >> $GITHUB_OUTPUT + + - name: Build package + run: python -m build + + - name: Check package + run: twine check --strict dist/* + + - name: Upload package + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Run tests + if: inputs.run_tests == true + # todo make configurable + working-directory: autotest + run: pytest -v -n=auto --durations=0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Touch changelog + run: touch HISTORY.md + + - name: Generate changelog + id: cliff + uses: orhun/git-cliff-action@v1 + with: + config: ${{ inputs.cliff_config }} + args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} + env: + OUTPUT: CHANGELOG.md + + - name: Update changelog + id: update-changelog + run: | + # move changelog + clog="CHANGELOG_${{ steps.version.outputs.version }}.md" + echo "changelog=$clog" >> $GITHUB_OUTPUT + sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" + + # show current release changelog + cat "$clog" + + # substitute full group names + sed -i 's/#### Ci/#### Continuous integration/' "$clog" + sed -i 's/#### Feat/#### New features/' "$clog" + sed -i 's/#### Fix/#### Bug fixes/' "$clog" + sed -i 's/#### Refactor/#### Refactoring/' "$clog" + sed -i 's/#### Test/#### Testing/' "$clog" + + cat "$clog" HISTORY.md > temp_history.md + sudo mv temp_history.md HISTORY.md + + # show full changelog + cat HISTORY.md + + - name: Upload changelog + uses: actions/upload-artifact@v3 + with: + name: changelog + path: ${{ steps.update-changelog.outputs.changelog }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58819c57..472eb6e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,133 +1,171 @@ name: Release on: - push: - branches: - - main - - v[0-9]+.[0-9]+.[0-9]+* - release: - types: - - published + # workflow_call trigger lets this be called from other workflows. + # note this workflow may be called multiple times in a release, + # first to trigger the distribution build, then to draft the PR + # to trunk, then publish the package, reinitialize develop, etc. + workflow_call: + inputs: + branch: + description: Branch to release from. + required: true + type: string + cliff_config: + description: Path of the git-cliff config file relative to the project root. + required: false + default: cliff.toml + type: string + cumulative_changelog: + description: Path of the cumulative changelog file relative to the project root. + required: false + default: HISTORY.md + type: string + draft_release: + description: Draft a release post with assets and changelog. + required: false + default: true + type: boolean + environment_name: + description: Name of the GitHub environment to use for the release. + required: false + default: release + type: string + package_name: + # currently assumes module dir is in project root, + # and module name is the same as package name with + # hyphens swapped for underscores + description: Name of the Python package to release. + required: true + type: string + publish_package: + description: Publish the package to PyPI. + required: false + default: true + type: boolean + python: + description: Python version to build the package with. + required: false + default: '3.8' + type: string + reset_develop_version: + description: Version to reset the develop branch to. + required: false + default: '' + type: string + run_tests: + # currently assumes tests are in autotest/ + description: Run tests after building binaries. + required: false + type: boolean + default: true + trunk_branch: + description: Name of the trunk branch (e.g. 'main' or 'master'). + required: false + default: main + type: string + version: + description: Version number to use for release. + required: true + type: string jobs: - prep: - name: Prepare release - runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' && github.ref_name != 'main' }} + make_dist: + name: Make distribution + if: github.event_name != 'workflow_dispatch' && github.ref_name != inputs.trunk_branch + uses: ./.github/workflows/make_package.yml + with: + branch: ${{ inputs.branch }} + cliff_config: ${{ inputs.cliff_config }} + cumulative_changelog: ${{ inputs.cumulative_changelog }} + package_name: ${{ inputs.package_name }} + python: ${{ inputs.python }} + run_tests: ${{ inputs.run_tests }} + trunk_branch: ${{ inputs.trunk_branch }} + version: ${{ inputs.version }} + + release_pr: + name: Draft release PR + if: github.event_name != 'workflow_dispatch' && !contains(github.ref_name, 'rc') + needs: + - make_dist + runs-on: ubuntu-22.04 permissions: contents: write pull-requests: write - defaults: - run: - shell: bash steps: - - name: Checkout release branch + - name: Checkout repo uses: actions/checkout@v3 with: - fetch-depth: 0 + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ github.ref }} - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 - cache: 'pip' - cache-dependency-path: pyproject.toml + python-version: ${{ inputs.python }} - - name: Install Python dependencies + - name: Install Python build/test dependencies run: | pip install --upgrade pip - pip install build twine - pip install . - pip install ".[lint, test]" + pip install black filelock - name: Update version id: version run: | ref="${{ github.ref_name }}" version="${ref#"v"}" + package="${{ inputs.package_name }}" + # assume module name is the same as package + # name with hyphens swapped for underscores + module="${package//-/_}" python scripts/update_version.py -v "$version" - python scripts/lint.py - python -c "import modflow_devtools; print('Version: ', modflow_devtools.__version__)" + black -v $module/version.py + python -c "import $module; print('Version: ', $module.__version__)" echo "version=$version" >> $GITHUB_OUTPUT - - name: Touch changelog - run: touch HISTORY.md - - - name: Generate changelog - id: cliff - uses: orhun/git-cliff-action@v1 - with: - config: cliff.toml - args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} - env: - OUTPUT: CHANGELOG.md - - - name: Update changelog - id: update-changelog - run: | - # move changelog - clog="CHANGELOG_${{ steps.version.outputs.version }}.md" - echo "changelog=$clog" >> $GITHUB_OUTPUT - sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" - - # show current release changelog - cat "$clog" - - # substitute full group names - sed -i 's/#### Ci/#### Continuous integration/' "$clog" - sed -i 's/#### Feat/#### New features/' "$clog" - sed -i 's/#### Fix/#### Bug fixes/' "$clog" - sed -i 's/#### Refactor/#### Refactoring/' "$clog" - sed -i 's/#### Test/#### Testing/' "$clog" - - cat "$clog" HISTORY.md > temp_history.md - sudo mv temp_history.md HISTORY.md - - # show full changelog - cat HISTORY.md - - - name: Upload changelog - uses: actions/upload-artifact@v3 - with: - name: changelog - path: ${{ steps.update-changelog.outputs.changelog }} - - - name: Push release branch + - name: Commit & push changes env: GITHUB_TOKEN: ${{ github.token }} run: | - ver="${{ steps.version.outputs.version }}" - changelog=$(cat ${{ steps.update-changelog.outputs.changelog }} | grep -v "### Version $ver") - - # remove this release's changelog so we don't commit it - # the changes have already been prepended to HISTORY.md - rm ${{ steps.update-changelog.outputs.changelog }} - rm -f CHANGELOG.md - - # commit and push changes git config core.sharedRepository true git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A - git commit -m "ci(release): set version to ${{ steps.version.outputs.version }}, update changelog" + git commit -m "ci(release): version ${{ inputs.version }}" git push origin "${{ github.ref_name }}" + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Create release PR + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ver="${{ inputs.version }}" + changelog=$(cat CHANGELOG.md) + title="Release $ver" + trunk="${{ inputs.trunk_branch }}" body=' # Release '$ver' - The release can be approved by merging this pull request into `main`. This will trigger jobs to publish the release to PyPI and reset `develop` from `main`, incrementing the patch version number. + The release can be approved by merging this pull request into `$trunk`. This will trigger jobs to publish the release to PyPI and reset `develop` from `$trunk`. ## Changelog '$changelog' ' - gh pr create -B "main" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" + gh pr create -B "$trunk" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" release: name: Draft release - # runs only when changes are merged to main - if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - runs-on: ubuntu-latest + needs: + - make_dist + # runs only when changes are merged to trunk + if: github.event_name == 'push' && github.ref_name == inputs.trunk_branch && (inputs.draft_release == '' || inputs.draft_release == true) + runs-on: ubuntu-22.04 permissions: contents: write pull-requests: write @@ -136,7 +174,9 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - ref: main + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ inputs.trunk_branch }} # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog - name: Download artifacts @@ -147,10 +187,10 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: | version=$(cat version.txt) - title="MODFLOW developer tools $version" + title="${{ inputs.package_name }} $version" notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") gh release create "$version" \ - --target main \ + --target ${{ inputs.trunk_branch }} \ --title "$title" \ --notes "$notes" \ --draft \ @@ -158,69 +198,78 @@ jobs: publish: name: Publish package + needs: + - make_dist + - release # runs only after release is published (manually promoted from draft) - if: ${{ github.event_name == 'release' }} + if: github.event_name == 'release' && (inputs.publish_package == '' || inputs.publish_package == true) runs-on: ubuntu-22.04 permissions: contents: write pull-requests: write + id-token: write # mandatory for trusted publishing + environment: + name: ${{ inputs.environment_name }} + url: https://pypi.org/p/${{ inputs.package_name }} steps: - - name: Checkout main branch + - name: Checkout trunk uses: actions/checkout@v3 with: - ref: main + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ inputs.trunk_branch }} - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ inputs.python }} + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 - name: Install Python dependencies run: | pip install --upgrade pip - pip install build twine - pip install . - - - name: Build package - run: python -m build + pip install twine - name: Check package run: twine check --strict dist/* + # looks in dist/ dir by default - name: Publish package - if: ${{ env.TWINE_USERNAME != '' }} - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: twine upload dist/* - - reset: + uses: pypa/gh-action-pypi-publish@release/v1 + + reset_pr: name: Draft reset PR - if: ${{ github.event_name == 'release' }} + needs: + - make_dist + - release + if: github.event_name == 'release' && inputs.reset_develop_version != '' runs-on: ubuntu-22.04 permissions: contents: write pull-requests: write steps: - - name: Checkout main branch + - name: Checkout trunk branch uses: actions/checkout@v3 with: - ref: main + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ inputs.trunk_branch }} - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ inputs.python }} cache: 'pip' - cache-dependency-path: pyproject.toml - name: Install Python dependencies run: | pip install --upgrade pip - pip install . - pip install ".[lint, test]" + pip install black filelock - name: Get release tag uses: oprypin/find-latest-tag@v1 @@ -233,30 +282,29 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - # create reset branch from main + # create reset branch from trunk reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" git switch -c $reset_branch - # increment patch version - major_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f1) - minor_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f2) - patch_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f3) - version="$major_version.$minor_version.$((patch_version + 1))" - python scripts/update_version.py -v "$version" - python scripts/lint.py + # update version (add + to version.txt to indicate development status) + package=${{ inputs.package_name }} + module=${package//-/_} + python scripts/update_version.py -v "${{ inputs.reset_develop_version }}" + black -v $module/version.py # commit and push reset branch + trunk=${{ inputs.trunk_branch }} git config core.sharedRepository true git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A - git commit -m "ci(release): update to development version $version" + git commit -m "ci(post-release): update develop from $trunk" git push -u origin $reset_branch # create PR into develop body=' # Reinitialize for development - Updates the `develop` branch from `main` following a successful release. Increments the patch version number. + Updates the `develop` branch from `$trunk` following a successful release. ' gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file diff --git a/.github/workflows/release_dispatch.yml b/.github/workflows/release_dispatch.yml new file mode 100644 index 00000000..da852333 --- /dev/null +++ b/.github/workflows/release_dispatch.yml @@ -0,0 +1,132 @@ +name: Release +on: + push: + branches: + # initial trigger on semver branch with leading 'v' + - v[0-9]+.[0-9]+.[0-9]+* + # second phase trigger after merging to trunk + - main + release: + types: + # third phase trigger after release is published + - published + # workflow_dispatch trigger to start release via GitHub UI or CLI, + # as an alternative to triggering when a release branch is pushed. + # see https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: + inputs: + branch: + description: Branch to release from. + required: true + type: string + draft_release: + description: Draft a release post with assets and changelog. + required: false + default: true + type: boolean + publish_package: + description: Publish the package to PyPI. + required: false + default: true + type: boolean + python: + description: Python version to use for release. + required: false + default: '3.8' + type: string + reset_develop_version: + description: Version to reset the develop branch to. + required: false + default: '' + type: string + version: + description: Version number to use for release. + required: true + type: string +jobs: + # configure options which may be set as dispatch + # inputs or dynamically assigned default values + set_options: + name: Set release options + if: github.ref_name != 'master' + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l {0} + outputs: + branch: ${{ steps.set_branch.outputs.branch }} + python: ${{ steps.set_python.outputs.python }} + version: ${{ steps.set_version.outputs.version }} + steps: + + - name: Set branch + id: set_branch + run: | + # if branch was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.branch }}") ]]; then + branch="${{ inputs.branch }}" + # prevent releases from develop or master + if [[ ("$branch" == "develop") || ("$branch" == "master") ]]; then + echo "error: releases may not be triggered from branch $branch" + exit 1 + fi + echo "using branch $branch from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, use that branch + branch="${{ github.ref_name }}" + echo "using branch $branch from ref ${{ github.ref }}" + else + # otherwise exit with an error + echo "error: this workflow should not have triggered for event ${{ github.event_name }} on branch ${{ github.ref_name }}" + exit 1 + fi + echo "branch=$branch" >> $GITHUB_OUTPUT + + - name: Set Python + id: set_python + run: | + # if python version was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.python }}") ]]; then + python="${{ inputs.python }}" + echo "using python $python from workflow_dispatch" + else + # otherwise use 3.8 + python=3.8 + fi + echo "python=$python" >> $GITHUB_OUTPUT + + - name: Set version + id: set_version + run: | + # if version number was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.version }}") ]]; then + ver="${{ inputs.version }}" + echo "using version number $ver from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, parse version number from branch name (sans leading 'v') + ref="${{ github.ref_name }}" + ver="${ref#"v"}" + echo "parsed version number $ver from branch name $ref" + else + # otherwise exit with an error + echo "error: version number not provided explicitly (via workflow_dispatch input) or implicitly (via branch name)" + exit 1 + fi + echo "version=$ver" >> $GITHUB_OUTPUT + + release: + name: Do release + uses: ./.github/workflows/release.yml + needs: + - set_options + with: + branch: ${{ needs.set_options.outputs.branch }} + draft_release: true + package_name: modflow-devtools + publish_package: false + python: ${{ needs.set_options.outputs.python }} + reset_develop_version: ${{ inputs.reset_develop_version }} + run_tests: true + trunk_branch: main + version: ${{ needs.set_options.outputs.version }} + \ No newline at end of file diff --git a/DEVELOPER.md b/DEVELOPER.md index 5a44c4f3..58798c3f 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -40,6 +40,7 @@ This repository's tests expect a few environment variables: - `BIN_PATH`: path to MODFLOW 6 and related executables - `REPOS_PATH`: the path to MODFLOW 6 example model repositories - `GITHUB_TOKEN`: a GitHub authentication token +- `GIITHUB_USER`: the developer's GitHub username These may be set manually, but the recommended approach is to configure environment variables in a `.env` file in the project root, for instance: @@ -71,6 +72,8 @@ Tests which must write to disk should use `pytest`'s built-in `temp_dir` fixture ## Releasing +todo: update from https://github.com/EC-USGS/pywatershed/blob/develop/.github/RELEASE.md since it has more accurate/detailed explication, then once pywatershed consumes `reusable_release.yml` from this repo, just link from there back here + The `modflow-devtools` release procedure is automated with GitHub Actions in [`.github/workflows/release.yml`](.github/workflows/release.yml). Making a release involves the following steps: 1. Release from `master` branch diff --git a/README.md b/README.md index a6833dbe..3e48ce87 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # MODFLOW developer tools +[![CI](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml/badge.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml) +[![Documentation Status](https://readthedocs.org/projects/modflow-devtools/badge/?version=latest)](https://modflow-devtools.readthedocs.io/en/latest/?badge=latest) +[![Formatted with black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) + [![GitHub tag](https://img.shields.io/github/tag/MODFLOW-USGS/modflow-devtools.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/tags/latest) [![PyPI Version](https://img.shields.io/pypi/v/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) -[![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) [![PyPI Status](https://img.shields.io/pypi/status/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) -[![CI](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml/badge.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml) -[![Documentation Status](https://readthedocs.org/projects/modflow-devtools/badge/?version=latest)](https://modflow-devtools.readthedocs.io/en/latest/?badge=latest) +[![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) Python tools for MODFLOW development and testing. @@ -65,6 +67,8 @@ This package contains shared tools for developing and testing MODFLOW 6 and FloP - Python packages installed - executables available on the path +A [reusable GitHub Actions workflow](https://docs.github.com/en/actions/using-workflows/reusing-workflows) is also available for EC-related Python package releases. This workflow is used by the `modflow-devtools` project itself, and can be consumed by other projects provided a few conventions are followed. See [the documentation](https://modflow-devtools.readthedocs.io/en/latest/md/release.html) for details. + ## Quickstart To import `pytest` fixtures in a project consuming `modflow-devtools`, add the following to a `conftest.py` file: diff --git a/conftest.py b/conftest.py index 74bcb562..6fd20f03 100644 --- a/conftest.py +++ b/conftest.py @@ -1 +1,18 @@ +from os import environ + +import pytest + pytest_plugins = ["modflow_devtools.fixtures"] + + +def get_github_user(): + user = environ.get("GITHUB_USER", None) + return user + + +@pytest.fixture +def github_user() -> str: + user = get_github_user() + if not user: + pytest.skip(reason="GITHUB_USER required") + return user diff --git a/docs/index.rst b/docs/index.rst index 91030def..0bfd1a69 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ The `modflow-devtools` package provides a set of tools for developing and testin md/download.md md/zip.md + md/release.md .. toctree:: diff --git a/docs/md/release.md b/docs/md/release.md new file mode 100644 index 00000000..7faaf327 --- /dev/null +++ b/docs/md/release.md @@ -0,0 +1,127 @@ +# Releasing Python packages + +This repository provides a [reusable GitHub Actions workflow](https://docs.github.com/en/actions/using-workflows/reusing-workflows) for EC-related Python packages. This workflow is used by the `modflow-devtools` project itself, and can be used by other projects provided a few requirements are met. + +## Overview + +There are two reusable release workflows in this repository: + +- `make_package.yml`: builds (and optionally tests) the Python package to be released +- `release.yml`: orchestrates optional release procedures after the package is built, including merging to trunk and back to `develop`, publishing to PyPI, creating a release post, and reinitializing the `develop` from trunk. + +A third workflow, `release_dispatch.yml`, contains triggers to start the release procedure either on: + +- manual dispatch from the GitHub UI or CLI +- push of a branch named `vx.y.z` or `vx.y.zrc`, where `x`, `y`, and `z` are semantic version numbers + +## Requirements + +A few assumptions are made about projects consuming the release workflows defined here. + +1. Trusted publishing + +The release workflow assumes the consuming repository is configured for [trusted publishing](https://docs.pypi.org/trusted-publishers/) to PyPI. As such, PyPI credentials need not be stored as repo secrets. Note that trusted publishing requires a [deployment environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment) named `pypi`. The environment may be empty. + +2. Conventional commits + +The workflow automatically generates a changelog for each release with [`git-cliff`](https://github.com/orhun/git-cliff). By default, this requires [conventional commits](https://www.conventionalcommits.org/en/about/). Custom [configuration](https://git-cliff.org/docs/configuration) may be provided to include unconventional commits in the changelog, but this leads to a less readable project history and is not recommended. By default, `git-cliff` configuration is expected in a `cliff.toml` file in the project root. A different file path/name (relative to the project root) may be provided with the `cliff_config` input. + +3. Cumulative changelog + +The workflow reserves the right to prepend each release's changelog to a version-controlled cumulative changelog file. By default, this is `HISTORY.md` in the project root. An alternative file path/name (relative to the project root) may be specified with the `cumulative_changelog` input. + +4. Organization permissions + +The organization to which a consuming repository belongs [must permit public reusable workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows#access-to-reusable-workflows). + +5. Release branch naming + +The release workflow is triggered when a release branch is pushed to the repository. The branch name *must* follow format `v{major}.{minor}.{patch}` ([semantic version](https://semver.org/) number with a leading 'v'). + +## Usage + +Reusable workflows are [called directly from jobs](https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow) rather than from steps within a job. For instance, to call the release workflow from a workflow in a repository containing a Python package named `mypackage` which should be built with Python 3.9: + +```yaml +name: Release mypackage +on: + push: + branches: + # initial trigger on semver branch with leading 'v' + - v[0-9]+.[0-9]+.[0-9]+* + # second phase trigger after merging to trunk + - main # substitute 'master' if needed + release: + types: + # third phase trigger after release is published + - published +jobs: + # configure options which may be set as dispatch + # inputs or dynamically assigned default values + set_options: + name: Set release options + if: github.ref_name != 'master' + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l {0} + outputs: + branch: ${{ steps.set_branch.outputs.branch }} + version: ${{ steps.set_version.outputs.version }} + steps: + + - name: Set branch + id: set_branch + run: | + # if branch was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.branch }}") ]]; then + branch="${{ inputs.branch }}" + # prevent releases from develop or master + if [[ ("$branch" == "develop") || ("$branch" == "master") ]]; then + echo "error: releases may not be triggered from branch $branch" + exit 1 + fi + echo "using branch $branch from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, use that branch + branch="${{ github.ref_name }}" + echo "using branch $branch from ref ${{ github.ref }}" + else + # otherwise exit with an error + echo "error: this workflow should not have triggered for event ${{ github.event_name }} on branch ${{ github.ref_name }}" + exit 1 + fi + echo "branch=$branch" >> $GITHUB_OUTPUT + + - name: Set version + id: set_version + run: | + # if version number was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.version }}") ]]; then + ver="${{ inputs.version }}" + echo "using version number $ver from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, parse version number from branch name (sans leading 'v') + ref="${{ github.ref_name }}" + ver="${ref#"v"}" + echo "parsed version number $ver from branch name $ref" + else + # otherwise exit with an error + echo "error: version number not provided explicitly (via workflow_dispatch input) or implicitly (via branch name)" + exit 1 + fi + echo "version=$ver" >> $GITHUB_OUTPUT + + release: + uses: MODFLOW-USGS/modflow-devtools/.github/workflows/release.yml@main + with: + branch: ${{ needs.set_options.outputs.branch }} + draft_release: true + package_name: mypackage + publish_package: true + python: '3.9' + reset_develop: true + run_tests: true + trunk_branch: main # substitute master if needed + version: '0.1.2' +``` \ No newline at end of file diff --git a/modflow_devtools/test/test_download.py b/modflow_devtools/test/test_download.py index 9fbfff74..119b99d0 100644 --- a/modflow_devtools/test/test_download.py +++ b/modflow_devtools/test/test_download.py @@ -1,4 +1,8 @@ +from os import environ +from typing import List + import pytest +from conftest import get_github_user from flaky import flaky from modflow_devtools.download import ( download_and_unzip, @@ -9,11 +13,21 @@ ) from modflow_devtools.markers import requires_github -_repos = [ - "MODFLOW-USGS/modflow6", - "MODFLOW-USGS/modflow6-nightly-build", - "MODFLOW-USGS/executables", -] + +def get_repos(user): + return [ + user + "/" + repo + for repo in ["modflow6", "modflow6-nightly-build", "executables"] + ] + + +@pytest.fixture +def repos(github_user) -> List[str]: + return get_repos(github_user) + + +_github_user = get_github_user() +_repos = get_repos(_github_user) @pytest.mark.parametrize("per_page", [-1, 0, 1.5, 101, 1000]) @@ -36,37 +50,37 @@ def test_get_releases(repo): assets = [a for aa in [r["assets"] for r in releases] for a in aa] assert all(repo in a["browser_download_url"] for a in assets) - # test page size option - if repo == "MODFLOW-USGS/modflow6-nightly-build": - assert len(releases) <= 31 # 30-day retention period - @flaky @requires_github -@pytest.mark.parametrize("repo", _repos) -def test_get_release(repo): +@pytest.mark.parametrize("github_repo", _repos) +def test_get_release(github_repo, github_user): tag = "latest" - release = get_release(repo, tag, verbose=True) + release = get_release(github_repo, tag, verbose=True) assets = release["assets"] expected_names = ["linux.zip", "mac.zip", "win64.zip"] + expected_ostags = [a.replace(".zip", "") for a in expected_names] actual_names = [asset["name"] for asset in assets] - if repo == "MODFLOW-USGS/modflow6": + if github_repo == f"{github_user}/modflow6": # can remove if modflow6 releases follow asset name conventions followed in executables and nightly build repos assert set([a.rpartition("_")[2] for a in actual_names]) >= set( [a for a in expected_names if not a.startswith("win")] ) else: - assert set(actual_names) >= set(expected_names) + for ostag in expected_ostags: + assert any( + ostag in a for a in actual_names + ), f"dist not found for {ostag}" @flaky @requires_github @pytest.mark.parametrize("name", [None, "rtd-files", "run-time-comparison"]) @pytest.mark.parametrize("per_page", [None, 100]) -def test_list_artifacts(tmp_path, name, per_page): +def test_list_artifacts(github_user, name, per_page): artifacts = list_artifacts( - "MODFLOW-USGS/modflow6", + f"{github_user}/modflow6", name=name, per_page=per_page, max_pages=2, @@ -80,8 +94,8 @@ def test_list_artifacts(tmp_path, name, per_page): @flaky @requires_github @pytest.mark.parametrize("delete_zip", [True, False]) -def test_download_artifact(function_tmpdir, delete_zip): - repo = "MODFLOW-USGS/modflow6" +def test_download_artifact(github_user, function_tmpdir, delete_zip): + repo = f"{github_user}/modflow6" artifacts = list_artifacts(repo, max_pages=1, verbose=True) first = next(iter(artifacts), None) @@ -104,14 +118,16 @@ def test_download_artifact(function_tmpdir, delete_zip): @flaky @requires_github @pytest.mark.parametrize("delete_zip", [True, False]) -def test_download_and_unzip(function_tmpdir, delete_zip): - zip_name = "mf6.3.0_linux.zip" +def test_download_and_unzip(github_user, function_tmpdir, delete_zip): + github_user = "MODFLOW-USGS" # comment to use releases on a fork + version = "6.3.0" + ostag = "linux" + zip_name = f"mf{version}_{ostag}.zip" dir_name = zip_name.replace(".zip", "") - url = f"https://github.com/MODFLOW-USGS/modflow6/releases/download/6.3.0/{zip_name}" + url = f"https://github.com/{github_user}/modflow6/releases/download/{version}/{zip_name}" download_and_unzip( url, function_tmpdir, delete_zip=delete_zip, verbose=True ) - assert (function_tmpdir / zip_name).is_file() != delete_zip dir_path = function_tmpdir / dir_name diff --git a/modflow_devtools/test/test_misc.py b/modflow_devtools/test/test_misc.py index 9ff517e3..4643902f 100644 --- a/modflow_devtools/test/test_misc.py +++ b/modflow_devtools/test/test_misc.py @@ -210,7 +210,7 @@ def test_get_namefile_paths_largetestmodels(): def test_get_namefile_paths_exclude_patterns(models): path, expected_count = models paths = get_namefile_paths(path, excluded=["gwf"]) - assert len(paths) == expected_count + assert len(paths) >= expected_count @pytest.mark.skipif(not any(_example_paths), reason="examples not found") @@ -225,10 +225,10 @@ def test_get_namefile_paths_select_prefix(): @pytest.mark.skipif(not any(_example_paths), reason="examples not found") def test_get_namefile_paths_select_patterns(): paths = get_namefile_paths(_examples_path, selected=["gwf"]) - assert len(paths) == 70 + assert any(paths) @pytest.mark.skipif(not any(_example_paths), reason="examples not found") def test_get_namefile_paths_select_packages(): paths = get_namefile_paths(_examples_path, packages=["wel"]) - assert len(paths) == 43 + assert any(paths) diff --git a/pyproject.toml b/pyproject.toml index 46d8cfdb..a1709230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ keywords = [ readme = "README.md" license = {text = "CC0"} classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", "Programming Language :: Python :: 3 :: Only", @@ -68,6 +68,9 @@ docs = [ "sphinx-rtd-theme", "myst-parser" ] +all = [ + "modflow-devtools[lint, test, docs]", +] [project.urls] "Documentation" = "https://modflow-devtools.readthedocs.io/en/latest/"