From f137228b7e0aecc719d2cc107613183c885a9901 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Thu, 25 May 2023 13:41:23 -0400 Subject: [PATCH 01/10] ci: draft reusable release workflow inputs/secrets --- .github/workflows/release.yml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58819c57..b824db5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,11 +2,41 @@ name: Release on: push: branches: - - main + # initial trigger on semver branch with leading 'v' - v[0-9]+.[0-9]+.[0-9]+* + # second phase trigger after merging to trunk (either 'main' or 'master') + - main + - master release: types: + # third phase trigger after release is published - published + workflow_call: + inputs: + install_modflow: + description: whether to install MODFLOW and related executables for the build/test job + required: false + default: false + type: boolean + conda_env: + description: path to a conda environment file, relative to the project root (by default, an environment is created from dependencies in pyproject.toml with setup-python) + required: false + type: string + python_version: + description: the Python version to use to build and optionally test the package + required: true + default: 3.8 + type: number + cliff_config: + description: path to the git-cliff config file, relative to the project root (empty string disables changelog generation) + required: false + default: cliff.toml + type: string + secrets: + pypi_username: + required: false + pypi_password: + required: false jobs: prep: name: Prepare release From 7b60569ba288756805c525aa99b306cc704f3c71 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Fri, 26 May 2023 15:35:18 -0400 Subject: [PATCH 02/10] ci: refactor reusable release workflow, add docs, add all dep group to pyproject.yaml --- .github/workflows/release.yml | 284 +----------------------- .github/workflows/reusable_release.yml | 290 +++++++++++++++++++++++++ DEVELOPER.md | 2 + README.md | 2 + docs/index.rst | 1 + docs/md/release.md | 51 +++++ pyproject.toml | 3 + 7 files changed, 354 insertions(+), 279 deletions(-) create mode 100644 .github/workflows/reusable_release.yml create mode 100644 docs/md/release.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b824db5b..f0c5fb54 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,289 +4,15 @@ on: branches: # initial trigger on semver branch with leading 'v' - v[0-9]+.[0-9]+.[0-9]+* - # second phase trigger after merging to trunk (either 'main' or 'master') + # second phase trigger after merging to trunk - main - - master release: types: # third phase trigger after release is published - published - workflow_call: - inputs: - install_modflow: - description: whether to install MODFLOW and related executables for the build/test job - required: false - default: false - type: boolean - conda_env: - description: path to a conda environment file, relative to the project root (by default, an environment is created from dependencies in pyproject.toml with setup-python) - required: false - type: string - python_version: - description: the Python version to use to build and optionally test the package - required: true - default: 3.8 - type: number - cliff_config: - description: path to the git-cliff config file, relative to the project root (empty string disables changelog generation) - required: false - default: cliff.toml - type: string - secrets: - pypi_username: - required: false - pypi_password: - required: false jobs: - prep: - name: Prepare release - runs-on: ubuntu-latest - if: ${{ github.event_name == 'push' && github.ref_name != 'main' }} - permissions: - contents: write - pull-requests: write - defaults: - run: - shell: bash - steps: - - - name: Checkout release branch - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - cache: 'pip' - cache-dependency-path: pyproject.toml - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install build twine - pip install . - pip install ".[lint, test]" - - - name: Update version - id: version - run: | - ref="${{ github.ref_name }}" - version="${ref#"v"}" - python scripts/update_version.py -v "$version" - python scripts/lint.py - python -c "import modflow_devtools; print('Version: ', modflow_devtools.__version__)" - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Touch changelog - run: touch HISTORY.md - - - name: Generate changelog - id: cliff - uses: orhun/git-cliff-action@v1 - with: - config: cliff.toml - args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} - env: - OUTPUT: CHANGELOG.md - - - name: Update changelog - id: update-changelog - run: | - # move changelog - clog="CHANGELOG_${{ steps.version.outputs.version }}.md" - echo "changelog=$clog" >> $GITHUB_OUTPUT - sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" - - # show current release changelog - cat "$clog" - - # substitute full group names - sed -i 's/#### Ci/#### Continuous integration/' "$clog" - sed -i 's/#### Feat/#### New features/' "$clog" - sed -i 's/#### Fix/#### Bug fixes/' "$clog" - sed -i 's/#### Refactor/#### Refactoring/' "$clog" - sed -i 's/#### Test/#### Testing/' "$clog" - - cat "$clog" HISTORY.md > temp_history.md - sudo mv temp_history.md HISTORY.md - - # show full changelog - cat HISTORY.md - - - name: Upload changelog - uses: actions/upload-artifact@v3 - with: - name: changelog - path: ${{ steps.update-changelog.outputs.changelog }} - - - name: Push release branch - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - ver="${{ steps.version.outputs.version }}" - changelog=$(cat ${{ steps.update-changelog.outputs.changelog }} | grep -v "### Version $ver") - - # remove this release's changelog so we don't commit it - # the changes have already been prepended to HISTORY.md - rm ${{ steps.update-changelog.outputs.changelog }} - rm -f CHANGELOG.md - - # commit and push changes - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(release): set version to ${{ steps.version.outputs.version }}, update changelog" - git push origin "${{ github.ref_name }}" - - title="Release $ver" - body=' - # Release '$ver' - - The release can be approved by merging this pull request into `main`. This will trigger jobs to publish the release to PyPI and reset `develop` from `main`, incrementing the patch version number. - - ## Changelog - - '$changelog' - ' - gh pr create -B "main" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" - release: - name: Draft release - # runs only when changes are merged to main - if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout repo - uses: actions/checkout@v3 - with: - ref: main - - # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog - - name: Download artifacts - uses: dawidd6/action-download-artifact@v2 - - - name: Draft release - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - version=$(cat version.txt) - title="MODFLOW developer tools $version" - notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") - gh release create "$version" \ - --target main \ - --title "$title" \ - --notes "$notes" \ - --draft \ - --latest - - publish: - name: Publish package - # runs only after release is published (manually promoted from draft) - if: ${{ github.event_name == 'release' }} - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout main branch - uses: actions/checkout@v3 - with: - ref: main - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install build twine - pip install . - - - name: Build package - run: python -m build - - - name: Check package - run: twine check --strict dist/* - - - name: Publish package - if: ${{ env.TWINE_USERNAME != '' }} - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: twine upload dist/* - - reset: - name: Draft reset PR - if: ${{ github.event_name == 'release' }} - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout main branch - uses: actions/checkout@v3 - with: - ref: main - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - cache: 'pip' - cache-dependency-path: pyproject.toml - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install . - pip install ".[lint, test]" - - - name: Get release tag - uses: oprypin/find-latest-tag@v1 - id: latest_tag - with: - repository: ${{ github.repository }} - releases-only: true - - - name: Draft pull request - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - # create reset branch from main - reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" - git switch -c $reset_branch - - # increment patch version - major_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f1) - minor_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f2) - patch_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f3) - version="$major_version.$minor_version.$((patch_version + 1))" - python scripts/update_version.py -v "$version" - python scripts/lint.py - - # commit and push reset branch - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(release): update to development version $version" - git push -u origin $reset_branch - - # create PR into develop - body=' - # Reinitialize for development - - Updates the `develop` branch from `main` following a successful release. Increments the patch version number. - ' - gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file + uses: ./.github/workflows/reusable_release.yml + with: + package_name: mypackage + python_version: 3.9 \ No newline at end of file diff --git a/.github/workflows/reusable_release.yml b/.github/workflows/reusable_release.yml new file mode 100644 index 00000000..ed35e470 --- /dev/null +++ b/.github/workflows/reusable_release.yml @@ -0,0 +1,290 @@ +name: Release +on: + push: + branches: + # initial trigger on semver branch with leading 'v' + - v[0-9]+.[0-9]+.[0-9]+* + # second phase trigger after merging to trunk (either 'main' or 'master') + - main + - master + release: + types: + # third phase trigger after release is published + - published + workflow_call: + inputs: + package_name: + description: the name of the Python package to release (must be identical to the module name) + required: true + type: string + trunk_branch: + description: the name of the trunk branch (either 'main' or 'master') + required: false + default: main + type: string + python_version: + description: the Python version to use to build and optionally test the package + required: true + default: 3.8 + type: number + cliff_config: + description: path to the git-cliff config file, relative to the project root + required: false + default: cliff.toml + type: string + cumulative_changelog: + description: path to the cumulative changelog file, relative to the project root + required: false + default: HISTORY.md + type: string +jobs: + prep: + name: Prepare release + runs-on: ubuntu-22.04 + if: github.event_name == 'push' && github.ref_name != ${{ inputs.trunk_branch }} + permissions: + contents: write + pull-requests: write + defaults: + run: + shell: bash + steps: + + - name: Checkout release branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python build/test dependencies + run: | + pip install --upgrade pip + pip install black build twine + pip install . + + - name: Update version + id: version + run: | + ref="${{ github.ref_name }}" + version="${ref#"v"}" + python scripts/update_version.py -v "$version" + black -v ${{ inputs.package_name }}/version.py + python -c "import ${{ inputs.package_name }}; print('Version: ', ${{ inputs.package_name }}.__version__)" + echo "version=$version" >> $GITHUB_OUTPUT + + - name: Touch changelog + run: touch HISTORY.md + + - name: Generate changelog + id: cliff + uses: orhun/git-cliff-action@v1 + with: + config: ${{ inputs.cliff_config }} + args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} + env: + OUTPUT: CHANGELOG.md + + - name: Update changelog + id: update-changelog + run: | + # move changelog + clog="CHANGELOG_${{ steps.version.outputs.version }}.md" + echo "changelog=$clog" >> $GITHUB_OUTPUT + sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" + + # show current release changelog + cat "$clog" + + # substitute full group names + sed -i 's/#### Ci/#### Continuous integration/' "$clog" + sed -i 's/#### Feat/#### New features/' "$clog" + sed -i 's/#### Fix/#### Bug fixes/' "$clog" + sed -i 's/#### Refactor/#### Refactoring/' "$clog" + sed -i 's/#### Test/#### Testing/' "$clog" + + cat "$clog" HISTORY.md > temp_history.md + sudo mv temp_history.md HISTORY.md + + # show full changelog + cat HISTORY.md + + - name: Upload changelog + uses: actions/upload-artifact@v3 + with: + name: changelog + path: ${{ steps.update-changelog.outputs.changelog }} + + - name: Create release PR + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ver="${{ steps.version.outputs.version }}" + changelog=$(cat ${{ steps.update-changelog.outputs.changelog }} | grep -v "### Version $ver") + + # remove this release's changelog so we don't commit it + # the changes have already been prepended to HISTORY.md + rm ${{ steps.update-changelog.outputs.changelog }} + rm -f CHANGELOG.md + + # commit and push changes + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(release): version ${{ steps.version.outputs.version }}" + git push origin "${{ github.ref_name }}" + + title="Release $ver" + body=' + # Release '$ver' + + The release can be approved by merging this pull request into `${{ inputs.trunk_branch }}`. This will trigger jobs to publish the release to PyPI and reset `develop` from `${{ inputs.trunk_branch }}`. + + ## Changelog + + '$changelog' + ' + gh pr create -B "${{ inputs.trunk_branch }}" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" + + release: + name: Draft release + # runs only when changes are merged to trunk + if: github.event_name == 'push' && github.ref_name == ${{ inputs.trunk_branch }} + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout repo + uses: actions/checkout@v3 + with: + ref: ${{ inputs.trunk_branch }} + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Draft release + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + version=$(cat version.txt) + title="${{ inputs.package_name }} $version" + notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") + gh release create "$version" \ + --target ${{ inputs.trunk_branch }} \ + --title "$title" \ + --notes "$notes" \ + --draft \ + --latest + + publish: + name: Publish package + # runs only after release is published (manually promoted from draft) + if: ${{ github.event_name == 'release' }} + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + id-token: write # mandatory for trusted publishing + environment: # requires a 'pypi' environment in repo settings + name: pypi + url: https://pypi.org/p/${{ inputs.package_name }} + steps: + + - name: Checkout trunk + uses: actions/checkout@v3 + with: + ref: ${{ inputs.trunk_branch }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install build twine + pip install . + + - name: Build package + run: python -m build + + - name: Check package + run: twine check --strict dist/* + + - name: Publish package + run: twine upload dist/* + + reset: + name: Draft reset PR + if: ${{ github.event_name == 'release' }} + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout trunk branch + uses: actions/checkout@v3 + with: + ref: ${{ inputs.trunk_branch }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + cache: 'pip' + cache-dependency-path: pyproject.toml + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install black filelock + + - name: Get release tag + uses: oprypin/find-latest-tag@v1 + id: latest_tag + with: + repository: ${{ github.repository }} + releases-only: true + + - name: Draft pull request + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + # create reset branch from trunk + reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" + git switch -c $reset_branch + + # increment patch version + major_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f1) + minor_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f2) + patch_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f3) + version="$major_version.$minor_version.$((patch_version + 1))" + + # update version (add + to version.txt to indicate development status) + python scripts/update_version.py -v "$version" + black -v ${{ inputs.package_name}}/version.py + + # commit and push reset branch + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(post-release): update develop from ${{ inputs.trunk_branch }}" + git push -u origin $reset_branch + + # create PR into develop + body=' + # Reinitialize for development + + Updates the `develop` branch from `${{ inputs.trunk_branch }}` following a successful release. + ' + gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file diff --git a/DEVELOPER.md b/DEVELOPER.md index 5a44c4f3..703767de 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -71,6 +71,8 @@ Tests which must write to disk should use `pytest`'s built-in `temp_dir` fixture ## Releasing +todo: update from https://github.com/EC-USGS/pywatershed/blob/develop/.github/RELEASE.md since it has more accurate/detailed explication, then once pywatershed consumes `reusable_release.yml` from this repo, just link from there back here + The `modflow-devtools` release procedure is automated with GitHub Actions in [`.github/workflows/release.yml`](.github/workflows/release.yml). Making a release involves the following steps: 1. Release from `master` branch diff --git a/README.md b/README.md index a6833dbe..1a390c52 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ This package contains shared tools for developing and testing MODFLOW 6 and FloP - Python packages installed - executables available on the path +A [reusable GitHub Actions workflow](https://docs.github.com/en/actions/using-workflows/reusing-workflows) is also available for EC-related Python package releases. This workflow is used by the `modflow-devtools` project itself, and can be consumed by other projects provided a few conventions are followed. See [the documentation](https://modflow-devtools.readthedocs.io/en/latest/md/release.html) for details. + ## Quickstart To import `pytest` fixtures in a project consuming `modflow-devtools`, add the following to a `conftest.py` file: diff --git a/docs/index.rst b/docs/index.rst index 91030def..0bfd1a69 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ The `modflow-devtools` package provides a set of tools for developing and testin md/download.md md/zip.md + md/release.md .. toctree:: diff --git a/docs/md/release.md b/docs/md/release.md new file mode 100644 index 00000000..f2fcc8b0 --- /dev/null +++ b/docs/md/release.md @@ -0,0 +1,51 @@ +# Releasing Python packages + +This repository provides a [reusable GitHub Actions workflow](https://docs.github.com/en/actions/using-workflows/reusing-workflows) for EC-related Python packages. This workflow is used by the `modflow-devtools` project itself, and can be used by other projects provided a few requirements are met. + +## Requirements + +1. Trusted publishing + +The release workflow assumes the consuming repository is configured for [trusted publishing](https://docs.pypi.org/trusted-publishers/) to PyPI. As such, PyPI credentials need not be stored as repo secrets. Note that trusted publishing requires a [deployment environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment) named `pypi`. The environment may be empty. + +2. Conventional commits + +The workflow automatically generates a changelog for each release with [`git-cliff`](https://github.com/orhun/git-cliff). By default, this requires [conventional commits](https://www.conventionalcommits.org/en/about/). Custom [configuration](https://git-cliff.org/docs/configuration) may be provided to include unconventional commits in the changelog, but this leads to a less readable project history and is not recommended. By default, `git-cliff` configuration is expected in a `cliff.toml` file in the project root. A different file path/name (relative to the project root) may be provided with the `cliff_config` input. + +3. Cumulative changelog + +The workflow reserves the right to prepend each release's changelog to a version-controlled cumulative changelog file. By default, this is `HISTORY.md` in the project root. An alternative file path/name (relative to the project root) may be specified with the `cumulative_changelog` input. + +4. Organization permissions + +The organization to which a consuming repository belongs [must permit public reusable workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows#access-to-reusable-workflows). + +5. Release branch naming + +The release workflow is triggered when a release branch is pushed to the repository. The branch name *must* follow format `v{major}.{minor}.{patch}` ([semantic version](https://semver.org/) number with a leading 'v'). + +## Usage + +Reusable workflows are [called directly from jobs](https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow) rather than from steps within a job. For instance, to call the release workflow from a workflow in a repository containing a Python package named `mypackage` which should be built with Python 3.9, add a workflow like the following: + +```yaml +name: Release +on: + push: + branches: + # initial trigger on semver branch with leading 'v' + - v[0-9]+.[0-9]+.[0-9]+* + # second phase trigger after merging to trunk + - main # substitute 'master' if needed + release: + types: + # third phase trigger after release is published + - published +jobs: + release: + uses: MODFLOW-USGS/modflow-devtools/.github/workflows/reusable_release.yml@main + with: + package_name: mypackage + python_version: 3.9 + trunk_branch: master +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 46d8cfdb..1efc9390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ docs = [ "sphinx-rtd-theme", "myst-parser" ] +all = [ + "modflow-devtools[lint, test, docs]", +] [project.urls] "Documentation" = "https://modflow-devtools.readthedocs.io/en/latest/" From fcc19c1d34f3a9eac908689ceab22a2e81ea4ca8 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Sat, 24 Jun 2023 18:54:45 -0400 Subject: [PATCH 03/10] ci: refactor release workflows --- .github/workflows/release.yml | 305 +++++++++++++++++++++++-- .github/workflows/release_dispatch.yml | 18 ++ .github/workflows/reusable_release.yml | 290 ----------------------- 3 files changed, 309 insertions(+), 304 deletions(-) create mode 100644 .github/workflows/release_dispatch.yml delete mode 100644 .github/workflows/reusable_release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0c5fb54..1f554748 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,18 +1,295 @@ name: Release on: - push: - branches: - # initial trigger on semver branch with leading 'v' - - v[0-9]+.[0-9]+.[0-9]+* - # second phase trigger after merging to trunk - - main - release: - types: - # third phase trigger after release is published - - published + # workflow_call event lets this workflow be called from elsewhere + workflow_call: + inputs: + cliff_config: + description: path to the git-cliff config file, relative to the project root + required: false + default: cliff.toml + type: string + cumulative_changelog: + description: path to the cumulative changelog file, relative to the project root + required: false + default: HISTORY.md + type: string + draft_release: + description: whether to draft a release post with assets and changelo + required: false + default: true + type: boolean + package_name: + description: the name of the Python package to release (must be identical to the module name) + required: true + type: string + publish_package: + description: whether to publish the package to PyPI + required: false + default: true + type: boolean + python_version: + description: the Python version to use to build and optionally test the package + required: true + default: 3.8 + type: number + reset_develop: + description: whether to reset the develop branch from the trunk branch after publishing the release + required: false + default: true + type: boolean + trunk_branch: + description: the name of the trunk branch (either 'main' or 'master') + required: false + default: main + type: string jobs: + prep: + name: Prepare release + runs-on: ubuntu-22.04 + if: github.event_name == 'push' && github.ref_name != inputs.trunk_branch + permissions: + contents: write + pull-requests: write + defaults: + run: + shell: bash + steps: + + - name: Checkout release branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python build/test dependencies + run: | + pip install --upgrade pip + pip install black build twine + pip install . + + - name: Update version + id: version + run: | + ref="${{ github.ref_name }}" + version="${ref#"v"}" + python scripts/update_version.py -v "$version" + black -v ${{ inputs.package_name }}/version.py + python -c "import ${{ inputs.package_name }}; print('Version: ', ${{ inputs.package_name }}.__version__)" + echo "version=$version" >> $GITHUB_OUTPUT + + - name: Touch changelog + run: touch HISTORY.md + + - name: Generate changelog + id: cliff + uses: orhun/git-cliff-action@v1 + with: + config: ${{ inputs.cliff_config }} + args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} + env: + OUTPUT: CHANGELOG.md + + - name: Update changelog + id: update-changelog + run: | + # move changelog + clog="CHANGELOG_${{ steps.version.outputs.version }}.md" + echo "changelog=$clog" >> $GITHUB_OUTPUT + sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" + + # show current release changelog + cat "$clog" + + # substitute full group names + sed -i 's/#### Ci/#### Continuous integration/' "$clog" + sed -i 's/#### Feat/#### New features/' "$clog" + sed -i 's/#### Fix/#### Bug fixes/' "$clog" + sed -i 's/#### Refactor/#### Refactoring/' "$clog" + sed -i 's/#### Test/#### Testing/' "$clog" + + cat "$clog" HISTORY.md > temp_history.md + sudo mv temp_history.md HISTORY.md + + # show full changelog + cat HISTORY.md + + - name: Upload changelog + uses: actions/upload-artifact@v3 + with: + name: changelog + path: ${{ steps.update-changelog.outputs.changelog }} + + - name: Create release PR + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ver="${{ steps.version.outputs.version }}" + changelog=$(cat ${{ steps.update-changelog.outputs.changelog }} | grep -v "### Version $ver") + + # remove this release's changelog so we don't commit it + # the changes have already been prepended to HISTORY.md + rm ${{ steps.update-changelog.outputs.changelog }} + rm -f CHANGELOG.md + + # commit and push changes + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(release): version ${{ steps.version.outputs.version }}" + git push origin "${{ github.ref_name }}" + + title="Release $ver" + body=' + # Release '$ver' + + The release can be approved by merging this pull request into `${{ inputs.trunk_branch }}`. This will trigger jobs to publish the release to PyPI and reset `develop` from `${{ inputs.trunk_branch }}`. + + ## Changelog + + '$changelog' + ' + gh pr create -B "${{ inputs.trunk_branch }}" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" + release: - uses: ./.github/workflows/reusable_release.yml - with: - package_name: mypackage - python_version: 3.9 \ No newline at end of file + name: Draft release + # runs only when changes are merged to trunk + if: github.event_name == 'push' && github.ref_name == inputs.trunk_branch && inputs.draft_release == true + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout repo + uses: actions/checkout@v3 + with: + ref: ${{ inputs.trunk_branch }} + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Draft release + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + version=$(cat version.txt) + title="${{ inputs.package_name }} $version" + notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") + gh release create "$version" \ + --target ${{ inputs.trunk_branch }} \ + --title "$title" \ + --notes "$notes" \ + --draft \ + --latest + + publish: + name: Publish package + # runs only after release is published (manually promoted from draft) + if: github.event_name == 'release' && inputs.publish_package == true + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + id-token: write # mandatory for trusted publishing + environment: # requires a 'pypi' environment in repo settings + name: pypi + url: https://pypi.org/p/${{ inputs.package_name }} + steps: + + - name: Checkout trunk + uses: actions/checkout@v3 + with: + ref: ${{ inputs.trunk_branch }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install build twine + pip install . + + - name: Build package + run: python -m build + + - name: Check package + run: twine check --strict dist/* + + - name: Publish package + run: twine upload dist/* + + reset: + name: Draft reset PR + if: github.event_name == 'release' && inputs.reset_develop == true + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout trunk branch + uses: actions/checkout@v3 + with: + ref: ${{ inputs.trunk_branch }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + cache: 'pip' + cache-dependency-path: pyproject.toml + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install black filelock + + - name: Get release tag + uses: oprypin/find-latest-tag@v1 + id: latest_tag + with: + repository: ${{ github.repository }} + releases-only: true + + - name: Draft pull request + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + # create reset branch from trunk + reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" + git switch -c $reset_branch + + # increment patch version + major_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f1) + minor_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f2) + patch_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f3) + version="$major_version.$minor_version.$((patch_version + 1))" + + # update version (add + to version.txt to indicate development status) + python scripts/update_version.py -v "$version" + black -v ${{ inputs.package_name}}/version.py + + # commit and push reset branch + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(post-release): update develop from ${{ inputs.trunk_branch }}" + git push -u origin $reset_branch + + # create PR into develop + body=' + # Reinitialize for development + + Updates the `develop` branch from `${{ inputs.trunk_branch }}` following a successful release. + ' + gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file diff --git a/.github/workflows/release_dispatch.yml b/.github/workflows/release_dispatch.yml new file mode 100644 index 00000000..0aad07bd --- /dev/null +++ b/.github/workflows/release_dispatch.yml @@ -0,0 +1,18 @@ +name: Release +on: + push: + branches: + # initial trigger on semver branch with leading 'v' + - v[0-9]+.[0-9]+.[0-9]+* + # second phase trigger after merging to trunk + - main + release: + types: + # third phase trigger after release is published + - published +jobs: + release: + uses: ./.github/workflows/release.yml + with: + package_name: mypackage + python_version: 3.9 \ No newline at end of file diff --git a/.github/workflows/reusable_release.yml b/.github/workflows/reusable_release.yml deleted file mode 100644 index ed35e470..00000000 --- a/.github/workflows/reusable_release.yml +++ /dev/null @@ -1,290 +0,0 @@ -name: Release -on: - push: - branches: - # initial trigger on semver branch with leading 'v' - - v[0-9]+.[0-9]+.[0-9]+* - # second phase trigger after merging to trunk (either 'main' or 'master') - - main - - master - release: - types: - # third phase trigger after release is published - - published - workflow_call: - inputs: - package_name: - description: the name of the Python package to release (must be identical to the module name) - required: true - type: string - trunk_branch: - description: the name of the trunk branch (either 'main' or 'master') - required: false - default: main - type: string - python_version: - description: the Python version to use to build and optionally test the package - required: true - default: 3.8 - type: number - cliff_config: - description: path to the git-cliff config file, relative to the project root - required: false - default: cliff.toml - type: string - cumulative_changelog: - description: path to the cumulative changelog file, relative to the project root - required: false - default: HISTORY.md - type: string -jobs: - prep: - name: Prepare release - runs-on: ubuntu-22.04 - if: github.event_name == 'push' && github.ref_name != ${{ inputs.trunk_branch }} - permissions: - contents: write - pull-requests: write - defaults: - run: - shell: bash - steps: - - - name: Checkout release branch - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - - - name: Install Python build/test dependencies - run: | - pip install --upgrade pip - pip install black build twine - pip install . - - - name: Update version - id: version - run: | - ref="${{ github.ref_name }}" - version="${ref#"v"}" - python scripts/update_version.py -v "$version" - black -v ${{ inputs.package_name }}/version.py - python -c "import ${{ inputs.package_name }}; print('Version: ', ${{ inputs.package_name }}.__version__)" - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Touch changelog - run: touch HISTORY.md - - - name: Generate changelog - id: cliff - uses: orhun/git-cliff-action@v1 - with: - config: ${{ inputs.cliff_config }} - args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} - env: - OUTPUT: CHANGELOG.md - - - name: Update changelog - id: update-changelog - run: | - # move changelog - clog="CHANGELOG_${{ steps.version.outputs.version }}.md" - echo "changelog=$clog" >> $GITHUB_OUTPUT - sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" - - # show current release changelog - cat "$clog" - - # substitute full group names - sed -i 's/#### Ci/#### Continuous integration/' "$clog" - sed -i 's/#### Feat/#### New features/' "$clog" - sed -i 's/#### Fix/#### Bug fixes/' "$clog" - sed -i 's/#### Refactor/#### Refactoring/' "$clog" - sed -i 's/#### Test/#### Testing/' "$clog" - - cat "$clog" HISTORY.md > temp_history.md - sudo mv temp_history.md HISTORY.md - - # show full changelog - cat HISTORY.md - - - name: Upload changelog - uses: actions/upload-artifact@v3 - with: - name: changelog - path: ${{ steps.update-changelog.outputs.changelog }} - - - name: Create release PR - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - ver="${{ steps.version.outputs.version }}" - changelog=$(cat ${{ steps.update-changelog.outputs.changelog }} | grep -v "### Version $ver") - - # remove this release's changelog so we don't commit it - # the changes have already been prepended to HISTORY.md - rm ${{ steps.update-changelog.outputs.changelog }} - rm -f CHANGELOG.md - - # commit and push changes - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(release): version ${{ steps.version.outputs.version }}" - git push origin "${{ github.ref_name }}" - - title="Release $ver" - body=' - # Release '$ver' - - The release can be approved by merging this pull request into `${{ inputs.trunk_branch }}`. This will trigger jobs to publish the release to PyPI and reset `develop` from `${{ inputs.trunk_branch }}`. - - ## Changelog - - '$changelog' - ' - gh pr create -B "${{ inputs.trunk_branch }}" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" - - release: - name: Draft release - # runs only when changes are merged to trunk - if: github.event_name == 'push' && github.ref_name == ${{ inputs.trunk_branch }} - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout repo - uses: actions/checkout@v3 - with: - ref: ${{ inputs.trunk_branch }} - - # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog - - name: Download artifacts - uses: dawidd6/action-download-artifact@v2 - - - name: Draft release - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - version=$(cat version.txt) - title="${{ inputs.package_name }} $version" - notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") - gh release create "$version" \ - --target ${{ inputs.trunk_branch }} \ - --title "$title" \ - --notes "$notes" \ - --draft \ - --latest - - publish: - name: Publish package - # runs only after release is published (manually promoted from draft) - if: ${{ github.event_name == 'release' }} - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - id-token: write # mandatory for trusted publishing - environment: # requires a 'pypi' environment in repo settings - name: pypi - url: https://pypi.org/p/${{ inputs.package_name }} - steps: - - - name: Checkout trunk - uses: actions/checkout@v3 - with: - ref: ${{ inputs.trunk_branch }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install build twine - pip install . - - - name: Build package - run: python -m build - - - name: Check package - run: twine check --strict dist/* - - - name: Publish package - run: twine upload dist/* - - reset: - name: Draft reset PR - if: ${{ github.event_name == 'release' }} - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout trunk branch - uses: actions/checkout@v3 - with: - ref: ${{ inputs.trunk_branch }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - cache: 'pip' - cache-dependency-path: pyproject.toml - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install black filelock - - - name: Get release tag - uses: oprypin/find-latest-tag@v1 - id: latest_tag - with: - repository: ${{ github.repository }} - releases-only: true - - - name: Draft pull request - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - # create reset branch from trunk - reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" - git switch -c $reset_branch - - # increment patch version - major_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f1) - minor_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f2) - patch_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f3) - version="$major_version.$minor_version.$((patch_version + 1))" - - # update version (add + to version.txt to indicate development status) - python scripts/update_version.py -v "$version" - black -v ${{ inputs.package_name}}/version.py - - # commit and push reset branch - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(post-release): update develop from ${{ inputs.trunk_branch }}" - git push -u origin $reset_branch - - # create PR into develop - body=' - # Reinitialize for development - - Updates the `develop` branch from `${{ inputs.trunk_branch }}` following a successful release. - ' - gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file From 067d74dd2506db387a19d5852c7b1b1105a03a99 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Sat, 24 Jun 2023 19:07:56 -0400 Subject: [PATCH 04/10] more refactoring using modflow6 as a template --- .github/workflows/release.yml | 22 +++---- .github/workflows/release_dispatch.yml | 91 +++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f554748..6a4ee27d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,41 +4,41 @@ on: workflow_call: inputs: cliff_config: - description: path to the git-cliff config file, relative to the project root + description: Path to the git-cliff config ile, relative to the project root. required: false default: cliff.toml type: string cumulative_changelog: - description: path to the cumulative changelog file, relative to the project root + description: Path to the cumulative changelog file relative to the project root. required: false default: HISTORY.md type: string draft_release: - description: whether to draft a release post with assets and changelo + description: Draft a release post with assets and changelog. required: false - default: true + default: false type: boolean package_name: - description: the name of the Python package to release (must be identical to the module name) + description: Name of the Python package to release (must be identical to the module name). required: true type: string publish_package: - description: whether to publish the package to PyPI + description: Publish the package to PyPI. required: false - default: true + default: false type: boolean python_version: - description: the Python version to use to build and optionally test the package + description: Python version to use to build and test the package. required: true default: 3.8 type: number reset_develop: - description: whether to reset the develop branch from the trunk branch after publishing the release + description: Reset the develop branch from the trunk. required: false - default: true + default: false type: boolean trunk_branch: - description: the name of the trunk branch (either 'main' or 'master') + description: Name of the trunk branch (either 'main' or 'master'). required: false default: main type: string diff --git a/.github/workflows/release_dispatch.yml b/.github/workflows/release_dispatch.yml index 0aad07bd..22fda5ee 100644 --- a/.github/workflows/release_dispatch.yml +++ b/.github/workflows/release_dispatch.yml @@ -10,9 +10,98 @@ on: types: # third phase trigger after release is published - published + # workflow_dispatch trigger to start release via GitHub UI or CLI, + # as an alternative to triggering when a release branch is pushed. + # see https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: + inputs: + branch: + description: 'Branch to release from.' + required: true + type: string + draft_release: + description: Draft a release post with assets and changelog. + required: false + default: true + type: boolean + publish_package: + description: Publish the package to PyPI. + required: false + default: true + type: boolean + reset_develop: + description: Reset the develop branch from the trunk. + required: false + default: true + type: boolean + version: + description: 'Version number to use for release.' + required: true + type: string jobs: + set_options: + name: Set release options + if: github.ref_name != 'master' + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l {0} + outputs: + branch: ${{ steps.set_branch.outputs.branch }} + version: ${{ steps.set_version.outputs.version }} + steps: + + - name: Set branch + id: set_branch + run: | + # if branch was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.branch }}") ]]; then + branch="${{ inputs.branch }}" + # prevent releases from develop or master + if [[ ("$branch" == "develop") || ("$branch" == "master") ]]; then + echo "error: releases may not be triggered from branch $branch" + exit 1 + fi + echo "using branch $branch from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, use that branch + branch="${{ github.ref_name }}" + echo "using branch $branch from ref ${{ github.ref }}" + else + # otherwise exit with an error + echo "error: this workflow should not have triggered for event ${{ github.event_name }} on branch ${{ github.ref_name }}" + exit 1 + fi + echo "branch=$branch" >> $GITHUB_OUTPUT + + - name: Set version + id: set_version + run: | + # if version number was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.version }}") ]]; then + ver="${{ inputs.version }}" + echo "using version number $ver from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, parse version number from branch name (sans leading 'v') + ref="${{ github.ref_name }}" + ver="${ref#"v"}" + echo "parsed version number $ver from branch name $ref" + else + # otherwise exit with an error + echo "error: version number not provided explicitly (via workflow_dispatch input) or implicitly (via branch name)" + exit 1 + fi + echo "version=$ver" >> $GITHUB_OUTPUT + release: + name: Do release uses: ./.github/workflows/release.yml + needs: + - set_options with: package_name: mypackage - python_version: 3.9 \ No newline at end of file + python_version: 3.9 + draft_release: ${{ inputs.draft_release }} + publish_package: ${{ inputs.publish_package }} + reset_develop: ${{ inputs.reset_develop }} + version: ${{ needs.set_options.outputs.version }} \ No newline at end of file From b0b005fe6fff955ef8b3240dd6d3f5ce88d3d770 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Mon, 26 Jun 2023 23:43:56 -0400 Subject: [PATCH 05/10] fix/cleanup tests, update dev docs, update README badge layout --- DEVELOPER.md | 1 + README.md | 8 ++-- conftest.py | 17 ++++++++ modflow_devtools/test/test_download.py | 60 ++++++++++++++++---------- modflow_devtools/test/test_misc.py | 6 +-- pyproject.toml | 2 +- 6 files changed, 65 insertions(+), 29 deletions(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index 703767de..58798c3f 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -40,6 +40,7 @@ This repository's tests expect a few environment variables: - `BIN_PATH`: path to MODFLOW 6 and related executables - `REPOS_PATH`: the path to MODFLOW 6 example model repositories - `GITHUB_TOKEN`: a GitHub authentication token +- `GIITHUB_USER`: the developer's GitHub username These may be set manually, but the recommended approach is to configure environment variables in a `.env` file in the project root, for instance: diff --git a/README.md b/README.md index 1a390c52..3e48ce87 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # MODFLOW developer tools +[![CI](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml/badge.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml) +[![Documentation Status](https://readthedocs.org/projects/modflow-devtools/badge/?version=latest)](https://modflow-devtools.readthedocs.io/en/latest/?badge=latest) +[![Formatted with black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) + [![GitHub tag](https://img.shields.io/github/tag/MODFLOW-USGS/modflow-devtools.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/tags/latest) [![PyPI Version](https://img.shields.io/pypi/v/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) -[![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) [![PyPI Status](https://img.shields.io/pypi/status/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) -[![CI](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml/badge.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/actions/workflows/ci.yml) -[![Documentation Status](https://readthedocs.org/projects/modflow-devtools/badge/?version=latest)](https://modflow-devtools.readthedocs.io/en/latest/?badge=latest) +[![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools) Python tools for MODFLOW development and testing. diff --git a/conftest.py b/conftest.py index 74bcb562..6fd20f03 100644 --- a/conftest.py +++ b/conftest.py @@ -1 +1,18 @@ +from os import environ + +import pytest + pytest_plugins = ["modflow_devtools.fixtures"] + + +def get_github_user(): + user = environ.get("GITHUB_USER", None) + return user + + +@pytest.fixture +def github_user() -> str: + user = get_github_user() + if not user: + pytest.skip(reason="GITHUB_USER required") + return user diff --git a/modflow_devtools/test/test_download.py b/modflow_devtools/test/test_download.py index 9fbfff74..119b99d0 100644 --- a/modflow_devtools/test/test_download.py +++ b/modflow_devtools/test/test_download.py @@ -1,4 +1,8 @@ +from os import environ +from typing import List + import pytest +from conftest import get_github_user from flaky import flaky from modflow_devtools.download import ( download_and_unzip, @@ -9,11 +13,21 @@ ) from modflow_devtools.markers import requires_github -_repos = [ - "MODFLOW-USGS/modflow6", - "MODFLOW-USGS/modflow6-nightly-build", - "MODFLOW-USGS/executables", -] + +def get_repos(user): + return [ + user + "/" + repo + for repo in ["modflow6", "modflow6-nightly-build", "executables"] + ] + + +@pytest.fixture +def repos(github_user) -> List[str]: + return get_repos(github_user) + + +_github_user = get_github_user() +_repos = get_repos(_github_user) @pytest.mark.parametrize("per_page", [-1, 0, 1.5, 101, 1000]) @@ -36,37 +50,37 @@ def test_get_releases(repo): assets = [a for aa in [r["assets"] for r in releases] for a in aa] assert all(repo in a["browser_download_url"] for a in assets) - # test page size option - if repo == "MODFLOW-USGS/modflow6-nightly-build": - assert len(releases) <= 31 # 30-day retention period - @flaky @requires_github -@pytest.mark.parametrize("repo", _repos) -def test_get_release(repo): +@pytest.mark.parametrize("github_repo", _repos) +def test_get_release(github_repo, github_user): tag = "latest" - release = get_release(repo, tag, verbose=True) + release = get_release(github_repo, tag, verbose=True) assets = release["assets"] expected_names = ["linux.zip", "mac.zip", "win64.zip"] + expected_ostags = [a.replace(".zip", "") for a in expected_names] actual_names = [asset["name"] for asset in assets] - if repo == "MODFLOW-USGS/modflow6": + if github_repo == f"{github_user}/modflow6": # can remove if modflow6 releases follow asset name conventions followed in executables and nightly build repos assert set([a.rpartition("_")[2] for a in actual_names]) >= set( [a for a in expected_names if not a.startswith("win")] ) else: - assert set(actual_names) >= set(expected_names) + for ostag in expected_ostags: + assert any( + ostag in a for a in actual_names + ), f"dist not found for {ostag}" @flaky @requires_github @pytest.mark.parametrize("name", [None, "rtd-files", "run-time-comparison"]) @pytest.mark.parametrize("per_page", [None, 100]) -def test_list_artifacts(tmp_path, name, per_page): +def test_list_artifacts(github_user, name, per_page): artifacts = list_artifacts( - "MODFLOW-USGS/modflow6", + f"{github_user}/modflow6", name=name, per_page=per_page, max_pages=2, @@ -80,8 +94,8 @@ def test_list_artifacts(tmp_path, name, per_page): @flaky @requires_github @pytest.mark.parametrize("delete_zip", [True, False]) -def test_download_artifact(function_tmpdir, delete_zip): - repo = "MODFLOW-USGS/modflow6" +def test_download_artifact(github_user, function_tmpdir, delete_zip): + repo = f"{github_user}/modflow6" artifacts = list_artifacts(repo, max_pages=1, verbose=True) first = next(iter(artifacts), None) @@ -104,14 +118,16 @@ def test_download_artifact(function_tmpdir, delete_zip): @flaky @requires_github @pytest.mark.parametrize("delete_zip", [True, False]) -def test_download_and_unzip(function_tmpdir, delete_zip): - zip_name = "mf6.3.0_linux.zip" +def test_download_and_unzip(github_user, function_tmpdir, delete_zip): + github_user = "MODFLOW-USGS" # comment to use releases on a fork + version = "6.3.0" + ostag = "linux" + zip_name = f"mf{version}_{ostag}.zip" dir_name = zip_name.replace(".zip", "") - url = f"https://github.com/MODFLOW-USGS/modflow6/releases/download/6.3.0/{zip_name}" + url = f"https://github.com/{github_user}/modflow6/releases/download/{version}/{zip_name}" download_and_unzip( url, function_tmpdir, delete_zip=delete_zip, verbose=True ) - assert (function_tmpdir / zip_name).is_file() != delete_zip dir_path = function_tmpdir / dir_name diff --git a/modflow_devtools/test/test_misc.py b/modflow_devtools/test/test_misc.py index 9ff517e3..4643902f 100644 --- a/modflow_devtools/test/test_misc.py +++ b/modflow_devtools/test/test_misc.py @@ -210,7 +210,7 @@ def test_get_namefile_paths_largetestmodels(): def test_get_namefile_paths_exclude_patterns(models): path, expected_count = models paths = get_namefile_paths(path, excluded=["gwf"]) - assert len(paths) == expected_count + assert len(paths) >= expected_count @pytest.mark.skipif(not any(_example_paths), reason="examples not found") @@ -225,10 +225,10 @@ def test_get_namefile_paths_select_prefix(): @pytest.mark.skipif(not any(_example_paths), reason="examples not found") def test_get_namefile_paths_select_patterns(): paths = get_namefile_paths(_examples_path, selected=["gwf"]) - assert len(paths) == 70 + assert any(paths) @pytest.mark.skipif(not any(_example_paths), reason="examples not found") def test_get_namefile_paths_select_packages(): paths = get_namefile_paths(_examples_path, packages=["wel"]) - assert len(paths) == 43 + assert any(paths) diff --git a/pyproject.toml b/pyproject.toml index 1efc9390..a1709230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ keywords = [ readme = "README.md" license = {text = "CC0"} classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication", "Programming Language :: Python :: 3 :: Only", From f6f10a3d0cbd7a7429ef53d0b90972ed4c6c7a27 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Tue, 27 Jun 2023 09:24:47 -0400 Subject: [PATCH 06/10] update flopy in CI --- .github/workflows/ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c18594c..bb6839bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,22 +142,17 @@ jobs: pip install . pip install ".[test]" - - name: Cache modflow6 examples - id: cache-examples - uses: actions/cache@v3 - with: - path: modflow6-examples/examples - key: modflow6-examples-${{ hashFiles('modflow6-examples/data/**') }} - - name: Install extra Python packages - if: steps.cache-examples.outputs.cache-hit != 'true' working-directory: modflow6-examples/etc run: | pip install -r requirements.pip.txt pip install -r requirements.usgs.txt + - name: Update flopy + working-directory: modflow6/autotest + run: python update_flopy.py + - name: Build modflow6 example models - if: steps.cache-examples.outputs.cache-hit != 'true' working-directory: modflow6-examples/etc run: python ci_build_files.py @@ -167,6 +162,7 @@ jobs: BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} GITHUB_TOKEN: ${{ github.token }} + GITHUB_USER: "MODFLOW-USGS" run: pytest -v -n auto --durations 0 --ignore modflow_devtools/test/test_download.py - name: Run network-dependent tests @@ -179,4 +175,5 @@ jobs: BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} GITHUB_TOKEN: ${{ github.token }} + GITHUB_USER: "MODFLOW-USGS" run: pytest -v -n auto --durations 0 modflow_devtools/test/test_download.py \ No newline at end of file From 1c820f1ab93b4af61762501dfe26ccb4aa5b11e5 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Tue, 27 Jun 2023 10:06:30 -0400 Subject: [PATCH 07/10] cleanup release workflows --- .github/workflows/release.yml | 240 +++++----------------- .github/workflows/release_dispatch.yml | 274 ++++++++++++++++++++++++- 2 files changed, 312 insertions(+), 202 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6a4ee27d..dd4ec164 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,47 +1,48 @@ name: Release on: - # workflow_call event lets this workflow be called from elsewhere + # workflow_call trigger lets this workflow be called from elsewhere workflow_call: inputs: + branch: + description: Branch to release from. + required: true + type: string cliff_config: - description: Path to the git-cliff config ile, relative to the project root. + description: Path of the git-cliff config file relative to the project root. required: false default: cliff.toml type: string cumulative_changelog: - description: Path to the cumulative changelog file relative to the project root. + description: Path of the cumulative changelog file relative to the project root. required: false default: HISTORY.md type: string - draft_release: - description: Draft a release post with assets and changelog. - required: false - default: false - type: boolean package_name: + # currently assumes module dir is in project root description: Name of the Python package to release (must be identical to the module name). required: true type: string - publish_package: - description: Publish the package to PyPI. - required: false - default: false - type: boolean python_version: - description: Python version to use to build and test the package. + description: Python version to build the package with. required: true - default: 3.8 - type: number - reset_develop: - description: Reset the develop branch from the trunk. + default: '3.8' + type: string + run_tests: + # currently assumes tests are in autotest/ + description: Run tests after building binaries. required: false - default: false type: boolean + default: true trunk_branch: - description: Name of the trunk branch (either 'main' or 'master'). + description: Name of the trunk branch (e.g. 'main' or 'master'). required: false default: main type: string + version: + description: Version number to use for release. + required: true + type: string + jobs: prep: name: Prepare release @@ -76,11 +77,35 @@ jobs: run: | ref="${{ github.ref_name }}" version="${ref#"v"}" + package="${{ inputs.package_name }}" + # assume module name is the same as package + # name with hyphens swapped for underscores + module="${package//-/_}" python scripts/update_version.py -v "$version" - black -v ${{ inputs.package_name }}/version.py - python -c "import ${{ inputs.package_name }}; print('Version: ', ${{ inputs.package_name }}.__version__)" + black -v $module/version.py + python -c "import $module; print('Version: ', $module.__version__)" echo "version=$version" >> $GITHUB_OUTPUT + - name: Build package + run: python -m build + + - name: Check package + run: twine check --strict dist/* + + - name: Upload package + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Run tests + if: inputs.run_tests == true + # todo make configurable + working-directory: autotest + run: pytest -v -n=auto --durations=0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Touch changelog run: touch HISTORY.md @@ -122,174 +147,3 @@ jobs: with: name: changelog path: ${{ steps.update-changelog.outputs.changelog }} - - - name: Create release PR - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - ver="${{ steps.version.outputs.version }}" - changelog=$(cat ${{ steps.update-changelog.outputs.changelog }} | grep -v "### Version $ver") - - # remove this release's changelog so we don't commit it - # the changes have already been prepended to HISTORY.md - rm ${{ steps.update-changelog.outputs.changelog }} - rm -f CHANGELOG.md - - # commit and push changes - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(release): version ${{ steps.version.outputs.version }}" - git push origin "${{ github.ref_name }}" - - title="Release $ver" - body=' - # Release '$ver' - - The release can be approved by merging this pull request into `${{ inputs.trunk_branch }}`. This will trigger jobs to publish the release to PyPI and reset `develop` from `${{ inputs.trunk_branch }}`. - - ## Changelog - - '$changelog' - ' - gh pr create -B "${{ inputs.trunk_branch }}" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" - - release: - name: Draft release - # runs only when changes are merged to trunk - if: github.event_name == 'push' && github.ref_name == inputs.trunk_branch && inputs.draft_release == true - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout repo - uses: actions/checkout@v3 - with: - ref: ${{ inputs.trunk_branch }} - - # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog - - name: Download artifacts - uses: dawidd6/action-download-artifact@v2 - - - name: Draft release - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - version=$(cat version.txt) - title="${{ inputs.package_name }} $version" - notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") - gh release create "$version" \ - --target ${{ inputs.trunk_branch }} \ - --title "$title" \ - --notes "$notes" \ - --draft \ - --latest - - publish: - name: Publish package - # runs only after release is published (manually promoted from draft) - if: github.event_name == 'release' && inputs.publish_package == true - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - id-token: write # mandatory for trusted publishing - environment: # requires a 'pypi' environment in repo settings - name: pypi - url: https://pypi.org/p/${{ inputs.package_name }} - steps: - - - name: Checkout trunk - uses: actions/checkout@v3 - with: - ref: ${{ inputs.trunk_branch }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install build twine - pip install . - - - name: Build package - run: python -m build - - - name: Check package - run: twine check --strict dist/* - - - name: Publish package - run: twine upload dist/* - - reset: - name: Draft reset PR - if: github.event_name == 'release' && inputs.reset_develop == true - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout trunk branch - uses: actions/checkout@v3 - with: - ref: ${{ inputs.trunk_branch }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - cache: 'pip' - cache-dependency-path: pyproject.toml - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install black filelock - - - name: Get release tag - uses: oprypin/find-latest-tag@v1 - id: latest_tag - with: - repository: ${{ github.repository }} - releases-only: true - - - name: Draft pull request - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - # create reset branch from trunk - reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" - git switch -c $reset_branch - - # increment patch version - major_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f1) - minor_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f2) - patch_version=$(echo "${{ steps.latest_tag.outputs.tag }}" | cut -d. -f3) - version="$major_version.$minor_version.$((patch_version + 1))" - - # update version (add + to version.txt to indicate development status) - python scripts/update_version.py -v "$version" - black -v ${{ inputs.package_name}}/version.py - - # commit and push reset branch - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(post-release): update develop from ${{ inputs.trunk_branch }}" - git push -u origin $reset_branch - - # create PR into develop - body=' - # Reinitialize for development - - Updates the `develop` branch from `${{ inputs.trunk_branch }}` following a successful release. - ' - gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file diff --git a/.github/workflows/release_dispatch.yml b/.github/workflows/release_dispatch.yml index 22fda5ee..bd29a3d1 100644 --- a/.github/workflows/release_dispatch.yml +++ b/.github/workflows/release_dispatch.yml @@ -16,7 +16,7 @@ on: workflow_dispatch: inputs: branch: - description: 'Branch to release from.' + description: Branch to release from. required: true type: string draft_release: @@ -29,16 +29,23 @@ on: required: false default: true type: boolean + python: + description: Python version to use for release. + required: false + default: '3.8' + type: string reset_develop: description: Reset the develop branch from the trunk. required: false default: true type: boolean version: - description: 'Version number to use for release.' + description: Version number to use for release. required: true type: string jobs: + # configure options which may be set as dispatch + # inputs or dynamically assigned default values set_options: name: Set release options if: github.ref_name != 'master' @@ -48,6 +55,9 @@ jobs: shell: bash -l {0} outputs: branch: ${{ steps.set_branch.outputs.branch }} + package: ${{ steps.set_package.outputs.package }} + python: ${{ steps.set_python.outputs.python }} + trunk: ${{ steps.set_trunk.outputs.trunk }} version: ${{ steps.set_version.outputs.version }} steps: @@ -73,6 +83,27 @@ jobs: exit 1 fi echo "branch=$branch" >> $GITHUB_OUTPUT + + - name: Set package name + id: set_package + run: echo "package=modflow-devtools" >> $GITHUB_OUTPUT + + - name: Set Python + id: set_python + run: | + # if python version was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.python }}") ]]; then + python="${{ inputs.python }}" + echo "using python $python from workflow_dispatch" + else + # otherwise use 3.8 + python=3.8 + fi + echo "python=$python" >> $GITHUB_OUTPUT + + - name: Set trunk branch + id: set_trunk + run: echo "trunk=main" >> $GITHUB_OUTPUT - name: Set version id: set_version @@ -93,15 +124,240 @@ jobs: fi echo "version=$ver" >> $GITHUB_OUTPUT - release: + make_dist: name: Do release uses: ./.github/workflows/release.yml needs: - set_options with: - package_name: mypackage - python_version: 3.9 - draft_release: ${{ inputs.draft_release }} - publish_package: ${{ inputs.publish_package }} - reset_develop: ${{ inputs.reset_develop }} - version: ${{ needs.set_options.outputs.version }} \ No newline at end of file + branch: ${{ needs.set_options.outputs.branch }} + package_name: ${{ needs.set_options.outputs.package }} + python_version: ${{ needs.set_options.outputs.python }} + trunk_branch: ${{ needs.set_options.outputs.trunk }} + version: ${{ needs.set_options.outputs.version }} + + pr: + name: Draft release PR + needs: + - set_options + - make_dist + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout repo + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ github.ref }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Install Python build/test dependencies + run: | + pip install --upgrade pip + pip install black filelock + + - name: Update version + id: version + run: | + ref="${{ github.ref_name }}" + version="${ref#"v"}" + package="${{ needs.set_options.outputs.package }}" + # assume module name is the same as package + # name with hyphens swapped for underscores + module="${package//-/_}" + python scripts/update_version.py -v "$version" + black -v $module/version.py + python -c "import $module; print('Version: ', $module.__version__)" + echo "version=$version" >> $GITHUB_OUTPUT + + - name: Commit & push changes + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ver="${{ needs.set_options.outputs.version }}" + + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(release): version $ver" + git push origin "${{ github.ref_name }}" + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Create release PR + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ver="${{ needs.set_options.outputs.version }}" + changelog=$(cat CHANGELOG.md) + + title="Release $ver" + trunk="${{ needs.set_options.outputs.trunk }}" + body=' + # Release '$ver' + + The release can be approved by merging this pull request into `$trunk`. This will trigger jobs to publish the release to PyPI and reset `develop` from `$trunk`. + + ## Changelog + + '$changelog' + ' + gh pr create -B "$trunk" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" + + release: + name: Draft release + needs: + - set_options + - make_dist + # runs only when changes are merged to trunk + if: github.event_name == 'push' && github.ref_name == needs.set_options.outputs.trunk && inputs.draft_release == true + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout repo + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ needs.set_options.outputs.trunk }} + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Draft release + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + version=$(cat version.txt) + title="${{ needs.set_options.outputs.package }} $version" + notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") + gh release create "$version" \ + --target ${{ needs.set_options.outputs.trunk }} \ + --title "$title" \ + --notes "$notes" \ + --draft \ + --latest + + publish: + name: Publish package + needs: + - set_options + - make_dist + # runs only after release is published (manually promoted from draft) + if: github.event_name == 'release' && inputs.publish_package == true + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + id-token: write # mandatory for trusted publishing + environment: # requires a 'pypi' environment in repo settings + name: pypi + url: https://pypi.org/p/${{ needs.set_options.outputs.package }} + steps: + + - name: Checkout trunk + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ needs.set_options.outputs.trunk }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ needs.set_options.outputs.python }} + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install twine + + - name: Check package + run: twine check --strict dist/* + + - name: Publish package + run: twine upload dist/* + + reset: + name: Draft reset PR + needs: + - set_options + - make_dist + if: github.event_name == 'release' && inputs.reset_develop == true + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout trunk branch + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ needs.set_options.outputs.trunk }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ needs.set_options.outputs.python }} + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install black filelock + + - name: Get release tag + uses: oprypin/find-latest-tag@v1 + id: latest_tag + with: + repository: ${{ github.repository }} + releases-only: true + + - name: Draft pull request + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + # create reset branch from trunk + reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" + git switch -c $reset_branch + + # update version (add + to version.txt to indicate development status) + package=${{ needs.set_options.outputs.package }} + module=${package//-/_} + version=$(python update_version.py -g) + python scripts/update_version.py -v "$version+" + black -v $module/version.py + + # commit and push reset branch + trunk=${{ needs.set_options.outputs.trunk }} + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(post-release): update develop from $trunk" + git push -u origin $reset_branch + + # create PR into develop + body=' + # Reinitialize for development + + Updates the `develop` branch from `$trunk` following a successful release. + ' + gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file From d4d104aea6f155db6349972895a009e2ba6f92b4 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Tue, 27 Jun 2023 13:30:09 -0400 Subject: [PATCH 08/10] refactor reusable release automation (extra layer of indirection), update docs --- .github/workflows/make_package.yml | 151 +++++++++++++ .github/workflows/release.yml | 286 +++++++++++++++++++------ .github/workflows/release_dispatch.yml | 249 +-------------------- docs/md/release.md | 86 +++++++- 4 files changed, 460 insertions(+), 312 deletions(-) create mode 100644 .github/workflows/make_package.yml diff --git a/.github/workflows/make_package.yml b/.github/workflows/make_package.yml new file mode 100644 index 00000000..ea608a2a --- /dev/null +++ b/.github/workflows/make_package.yml @@ -0,0 +1,151 @@ +name: Make package +on: + # workflow_call trigger lets this workflow be called from elsewhere + workflow_call: + inputs: + branch: + description: Branch to release from. + required: true + type: string + cliff_config: + description: Path of the git-cliff config file relative to the project root. + required: false + default: cliff.toml + type: string + cumulative_changelog: + description: Path of the cumulative changelog file relative to the project root. + required: false + default: HISTORY.md + type: string + package_name: + # currently assumes module dir is in project root, + # and module name is the same as package name with + # hyphens swapped for underscores + description: Name of the Python package to release. + required: true + type: string + python: + description: Python version to build the package with. + required: true + default: '3.8' + type: string + run_tests: + # currently assumes tests are in autotest/ + description: Run tests after building binaries. + required: false + type: boolean + default: true + trunk_branch: + description: Name of the trunk branch (e.g. 'main' or 'master'). + required: false + default: main + type: string + version: + description: Version number to use for release. + required: true + type: string + +jobs: + prep: + name: Prepare release + runs-on: ubuntu-22.04 + if: github.event_name == 'push' && github.ref_name != inputs.trunk_branch + permissions: + contents: write + pull-requests: write + defaults: + run: + shell: bash + steps: + + - name: Checkout release branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Install Python build/test dependencies + run: | + pip install --upgrade pip + pip install black build twine + pip install . + + - name: Update version + id: version + run: | + ref="${{ github.ref_name }}" + version="${ref#"v"}" + package="${{ inputs.package_name }}" + # assume module name is the same as package + # name with hyphens swapped for underscores + module="${package//-/_}" + python scripts/update_version.py -v "$version" + black -v $module/version.py + python -c "import $module; print('Version: ', $module.__version__)" + echo "version=$version" >> $GITHUB_OUTPUT + + - name: Build package + run: python -m build + + - name: Check package + run: twine check --strict dist/* + + - name: Upload package + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Run tests + if: inputs.run_tests == true + # todo make configurable + working-directory: autotest + run: pytest -v -n=auto --durations=0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Touch changelog + run: touch HISTORY.md + + - name: Generate changelog + id: cliff + uses: orhun/git-cliff-action@v1 + with: + config: ${{ inputs.cliff_config }} + args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} + env: + OUTPUT: CHANGELOG.md + + - name: Update changelog + id: update-changelog + run: | + # move changelog + clog="CHANGELOG_${{ steps.version.outputs.version }}.md" + echo "changelog=$clog" >> $GITHUB_OUTPUT + sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" + + # show current release changelog + cat "$clog" + + # substitute full group names + sed -i 's/#### Ci/#### Continuous integration/' "$clog" + sed -i 's/#### Feat/#### New features/' "$clog" + sed -i 's/#### Fix/#### Bug fixes/' "$clog" + sed -i 's/#### Refactor/#### Refactoring/' "$clog" + sed -i 's/#### Test/#### Testing/' "$clog" + + cat "$clog" HISTORY.md > temp_history.md + sudo mv temp_history.md HISTORY.md + + # show full changelog + cat HISTORY.md + + - name: Upload changelog + uses: actions/upload-artifact@v3 + with: + name: changelog + path: ${{ steps.update-changelog.outputs.changelog }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd4ec164..b5fef234 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,9 @@ name: Release on: - # workflow_call trigger lets this workflow be called from elsewhere + # workflow_call trigger lets this be called from other workflows. + # note this workflow may be called multiple times in a release, + # first to trigger the distribution build, then to draft the PR + # to trunk, then publish the package, reinitialize develop, etc. workflow_call: inputs: branch: @@ -17,16 +20,33 @@ on: required: false default: HISTORY.md type: string + draft_release: + description: Draft a release post with assets and changelog. + required: false + default: true + type: boolean package_name: - # currently assumes module dir is in project root - description: Name of the Python package to release (must be identical to the module name). + # currently assumes module dir is in project root, + # and module name is the same as package name with + # hyphens swapped for underscores + description: Name of the Python package to release. required: true type: string - python_version: + publish_package: + description: Publish the package to PyPI. + required: false + default: true + type: boolean + python: description: Python version to build the package with. - required: true + required: false default: '3.8' type: string + reset_develop: + description: Reset the develop branch from the trunk. + required: false + default: true + type: boolean run_tests: # currently assumes tests are in autotest/ description: Run tests after building binaries. @@ -42,35 +62,48 @@ on: description: Version number to use for release. required: true type: string - jobs: - prep: - name: Prepare release + make_dist: + name: Make distribution + if: github.event_name != 'workflow_dispatch' && github.ref_name != inputs.trunk_branch + uses: ./.github/workflows/make_package.yml + with: + branch: ${{ inputs.branch }} + cliff_config: ${{ inputs.cliff_config }} + cumulative_changelog: ${{ inputs.cumulative_changelog }} + package_name: ${{ inputs.package_name }} + python: ${{ inputs.python }} + run_tests: ${{ inputs.run_tests }} + trunk_branch: ${{ inputs.trunk_branch }} + version: ${{ inputs.version }} + + release_pr: + name: Draft release PR + if: github.event_name != 'workflow_dispatch' && !contains(github.ref_name, 'rc') + needs: + - make_dist runs-on: ubuntu-22.04 - if: github.event_name == 'push' && github.ref_name != inputs.trunk_branch permissions: contents: write pull-requests: write - defaults: - run: - shell: bash steps: - - name: Checkout release branch + - name: Checkout repo uses: actions/checkout@v3 with: - fetch-depth: 0 + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ github.ref }} - name: Setup Python uses: actions/setup-python@v4 with: - python-version: ${{ inputs.python_version }} + python-version: ${{ inputs.python }} - name: Install Python build/test dependencies run: | pip install --upgrade pip - pip install black build twine - pip install . + pip install black filelock - name: Update version id: version @@ -86,64 +119,183 @@ jobs: python -c "import $module; print('Version: ', $module.__version__)" echo "version=$version" >> $GITHUB_OUTPUT - - name: Build package - run: python -m build + - name: Commit & push changes + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(release): version ${{ inputs.version }}" + git push origin "${{ github.ref_name }}" + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Create release PR + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ver="${{ inputs.version }}" + changelog=$(cat CHANGELOG.md) + + title="Release $ver" + trunk="${{ inputs.trunk_branch }}" + body=' + # Release '$ver' + + The release can be approved by merging this pull request into `$trunk`. This will trigger jobs to publish the release to PyPI and reset `develop` from `$trunk`. + + ## Changelog + + '$changelog' + ' + gh pr create -B "$trunk" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" + + release: + name: Draft release + needs: + - make_dist + # runs only when changes are merged to trunk + if: github.event_name == 'push' && github.ref_name == inputs.trunk_branch && (inputs.draft_release == '' || inputs.draft_release == true) + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: + + - name: Checkout repo + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ inputs.trunk_branch }} + + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + + - name: Draft release + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + version=$(cat version.txt) + title="${{ inputs.package_name }} $version" + notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") + gh release create "$version" \ + --target ${{ inputs.trunk_branch }} \ + --title "$title" \ + --notes "$notes" \ + --draft \ + --latest + + publish: + name: Publish package + needs: + - make_dist + - release + # runs only after release is published (manually promoted from draft) + if: github.event_name == 'release' && (inputs.publish_package == '' || inputs.publish_package == true) + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + id-token: write # mandatory for trusted publishing + environment: # requires a 'pypi' environment in repo settings + name: pypi + url: https://pypi.org/p/${{ inputs.package_name }} + steps: + + - name: Checkout trunk + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ inputs.trunk_branch }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install twine - name: Check package run: twine check --strict dist/* - - name: Upload package - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist - - - name: Run tests - if: inputs.run_tests == true - # todo make configurable - working-directory: autotest - run: pytest -v -n=auto --durations=0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish package + run: twine upload dist/* + + reset_pr: + name: Draft reset PR + needs: + - make_dist + - release + if: github.event_name == 'release' && (inputs.reset_develop == '' || inputs.reset_develop == true) + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + steps: - - name: Touch changelog - run: touch HISTORY.md + - name: Checkout trunk branch + uses: actions/checkout@v3 + with: + # use repository_owner to allow testing on a fork + repository: ${{ github.repository_owner }}/modflow-devtools + ref: ${{ inputs.trunk_branch }} - - name: Generate changelog - id: cliff - uses: orhun/git-cliff-action@v1 + - name: Setup Python + uses: actions/setup-python@v4 with: - config: ${{ inputs.cliff_config }} - args: --verbose --unreleased --tag ${{ steps.version.outputs.version }} - env: - OUTPUT: CHANGELOG.md + python-version: ${{ inputs.python }} + cache: 'pip' - - name: Update changelog - id: update-changelog + - name: Install Python dependencies run: | - # move changelog - clog="CHANGELOG_${{ steps.version.outputs.version }}.md" - echo "changelog=$clog" >> $GITHUB_OUTPUT - sudo cp "${{ steps.cliff.outputs.changelog }}" "$clog" - - # show current release changelog - cat "$clog" - - # substitute full group names - sed -i 's/#### Ci/#### Continuous integration/' "$clog" - sed -i 's/#### Feat/#### New features/' "$clog" - sed -i 's/#### Fix/#### Bug fixes/' "$clog" - sed -i 's/#### Refactor/#### Refactoring/' "$clog" - sed -i 's/#### Test/#### Testing/' "$clog" - - cat "$clog" HISTORY.md > temp_history.md - sudo mv temp_history.md HISTORY.md - - # show full changelog - cat HISTORY.md + pip install --upgrade pip + pip install black filelock - - name: Upload changelog - uses: actions/upload-artifact@v3 + - name: Get release tag + uses: oprypin/find-latest-tag@v1 + id: latest_tag with: - name: changelog - path: ${{ steps.update-changelog.outputs.changelog }} + repository: ${{ github.repository }} + releases-only: true + + - name: Draft pull request + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + # create reset branch from trunk + reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" + git switch -c $reset_branch + + # update version (add + to version.txt to indicate development status) + package=${{ inputs.package_name }} + module=${package//-/_} + version=$(python update_version.py -g) + python scripts/update_version.py -v "$version+" + black -v $module/version.py + + # commit and push reset branch + trunk=${{ inputs.trunk_branch }} + git config core.sharedRepository true + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "ci(post-release): update develop from $trunk" + git push -u origin $reset_branch + + # create PR into develop + body=' + # Reinitialize for development + + Updates the `develop` branch from `$trunk` following a successful release. + ' + gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file diff --git a/.github/workflows/release_dispatch.yml b/.github/workflows/release_dispatch.yml index bd29a3d1..b0264f0a 100644 --- a/.github/workflows/release_dispatch.yml +++ b/.github/workflows/release_dispatch.yml @@ -55,9 +55,7 @@ jobs: shell: bash -l {0} outputs: branch: ${{ steps.set_branch.outputs.branch }} - package: ${{ steps.set_package.outputs.package }} python: ${{ steps.set_python.outputs.python }} - trunk: ${{ steps.set_trunk.outputs.trunk }} version: ${{ steps.set_version.outputs.version }} steps: @@ -83,10 +81,6 @@ jobs: exit 1 fi echo "branch=$branch" >> $GITHUB_OUTPUT - - - name: Set package name - id: set_package - run: echo "package=modflow-devtools" >> $GITHUB_OUTPUT - name: Set Python id: set_python @@ -101,10 +95,6 @@ jobs: fi echo "python=$python" >> $GITHUB_OUTPUT - - name: Set trunk branch - id: set_trunk - run: echo "trunk=main" >> $GITHUB_OUTPUT - - name: Set version id: set_version run: | @@ -124,240 +114,19 @@ jobs: fi echo "version=$ver" >> $GITHUB_OUTPUT - make_dist: + release: name: Do release uses: ./.github/workflows/release.yml needs: - set_options with: branch: ${{ needs.set_options.outputs.branch }} - package_name: ${{ needs.set_options.outputs.package }} - python_version: ${{ needs.set_options.outputs.python }} - trunk_branch: ${{ needs.set_options.outputs.trunk }} + draft_release: true + package_name: modflow-devtools + publish_package: false + python: ${{ needs.set_options.outputs.python }} + reset_develop: true + run_tests: true + trunk_branch: main version: ${{ needs.set_options.outputs.version }} - - pr: - name: Draft release PR - needs: - - set_options - - make_dist - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout repo - uses: actions/checkout@v3 - with: - # use repository_owner to allow testing on a fork - repository: ${{ github.repository_owner }}/modflow-devtools - ref: ${{ github.ref }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python }} - - - name: Install Python build/test dependencies - run: | - pip install --upgrade pip - pip install black filelock - - - name: Update version - id: version - run: | - ref="${{ github.ref_name }}" - version="${ref#"v"}" - package="${{ needs.set_options.outputs.package }}" - # assume module name is the same as package - # name with hyphens swapped for underscores - module="${package//-/_}" - python scripts/update_version.py -v "$version" - black -v $module/version.py - python -c "import $module; print('Version: ', $module.__version__)" - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Commit & push changes - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - ver="${{ needs.set_options.outputs.version }}" - - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(release): version $ver" - git push origin "${{ github.ref_name }}" - - # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog - - name: Download artifacts - uses: dawidd6/action-download-artifact@v2 - - - name: Create release PR - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - ver="${{ needs.set_options.outputs.version }}" - changelog=$(cat CHANGELOG.md) - - title="Release $ver" - trunk="${{ needs.set_options.outputs.trunk }}" - body=' - # Release '$ver' - - The release can be approved by merging this pull request into `$trunk`. This will trigger jobs to publish the release to PyPI and reset `develop` from `$trunk`. - - ## Changelog - - '$changelog' - ' - gh pr create -B "$trunk" -H "${{ github.ref_name }}" --title "$title" --draft --body "$body" - - release: - name: Draft release - needs: - - set_options - - make_dist - # runs only when changes are merged to trunk - if: github.event_name == 'push' && github.ref_name == needs.set_options.outputs.trunk && inputs.draft_release == true - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout repo - uses: actions/checkout@v3 - with: - # use repository_owner to allow testing on a fork - repository: ${{ github.repository_owner }}/modflow-devtools - ref: ${{ needs.set_options.outputs.trunk }} - - # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog - - name: Download artifacts - uses: dawidd6/action-download-artifact@v2 - - - name: Draft release - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - version=$(cat version.txt) - title="${{ needs.set_options.outputs.package }} $version" - notes=$(cat "changelog/CHANGELOG_$version.md" | grep -v "### Version $version") - gh release create "$version" \ - --target ${{ needs.set_options.outputs.trunk }} \ - --title "$title" \ - --notes "$notes" \ - --draft \ - --latest - - publish: - name: Publish package - needs: - - set_options - - make_dist - # runs only after release is published (manually promoted from draft) - if: github.event_name == 'release' && inputs.publish_package == true - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - id-token: write # mandatory for trusted publishing - environment: # requires a 'pypi' environment in repo settings - name: pypi - url: https://pypi.org/p/${{ needs.set_options.outputs.package }} - steps: - - - name: Checkout trunk - uses: actions/checkout@v3 - with: - # use repository_owner to allow testing on a fork - repository: ${{ github.repository_owner }}/modflow-devtools - ref: ${{ needs.set_options.outputs.trunk }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ needs.set_options.outputs.python }} - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install twine - - - name: Check package - run: twine check --strict dist/* - - - name: Publish package - run: twine upload dist/* - - reset: - name: Draft reset PR - needs: - - set_options - - make_dist - if: github.event_name == 'release' && inputs.reset_develop == true - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write - steps: - - - name: Checkout trunk branch - uses: actions/checkout@v3 - with: - # use repository_owner to allow testing on a fork - repository: ${{ github.repository_owner }}/modflow-devtools - ref: ${{ needs.set_options.outputs.trunk }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ needs.set_options.outputs.python }} - cache: 'pip' - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install black filelock - - - name: Get release tag - uses: oprypin/find-latest-tag@v1 - id: latest_tag - with: - repository: ${{ github.repository }} - releases-only: true - - - name: Draft pull request - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - # create reset branch from trunk - reset_branch="post-release-${{ steps.latest_tag.outputs.tag }}-reset" - git switch -c $reset_branch - - # update version (add + to version.txt to indicate development status) - package=${{ needs.set_options.outputs.package }} - module=${package//-/_} - version=$(python update_version.py -g) - python scripts/update_version.py -v "$version+" - black -v $module/version.py - - # commit and push reset branch - trunk=${{ needs.set_options.outputs.trunk }} - git config core.sharedRepository true - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "ci(post-release): update develop from $trunk" - git push -u origin $reset_branch - - # create PR into develop - body=' - # Reinitialize for development - - Updates the `develop` branch from `$trunk` following a successful release. - ' - gh pr create -B "develop" -H "$reset_branch" --title "Reinitialize develop branch" --draft --body "$body" \ No newline at end of file + \ No newline at end of file diff --git a/docs/md/release.md b/docs/md/release.md index f2fcc8b0..7faaf327 100644 --- a/docs/md/release.md +++ b/docs/md/release.md @@ -2,8 +2,22 @@ This repository provides a [reusable GitHub Actions workflow](https://docs.github.com/en/actions/using-workflows/reusing-workflows) for EC-related Python packages. This workflow is used by the `modflow-devtools` project itself, and can be used by other projects provided a few requirements are met. +## Overview + +There are two reusable release workflows in this repository: + +- `make_package.yml`: builds (and optionally tests) the Python package to be released +- `release.yml`: orchestrates optional release procedures after the package is built, including merging to trunk and back to `develop`, publishing to PyPI, creating a release post, and reinitializing the `develop` from trunk. + +A third workflow, `release_dispatch.yml`, contains triggers to start the release procedure either on: + +- manual dispatch from the GitHub UI or CLI +- push of a branch named `vx.y.z` or `vx.y.zrc`, where `x`, `y`, and `z` are semantic version numbers + ## Requirements +A few assumptions are made about projects consuming the release workflows defined here. + 1. Trusted publishing The release workflow assumes the consuming repository is configured for [trusted publishing](https://docs.pypi.org/trusted-publishers/) to PyPI. As such, PyPI credentials need not be stored as repo secrets. Note that trusted publishing requires a [deployment environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment) named `pypi`. The environment may be empty. @@ -26,10 +40,10 @@ The release workflow is triggered when a release branch is pushed to the reposit ## Usage -Reusable workflows are [called directly from jobs](https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow) rather than from steps within a job. For instance, to call the release workflow from a workflow in a repository containing a Python package named `mypackage` which should be built with Python 3.9, add a workflow like the following: +Reusable workflows are [called directly from jobs](https://docs.github.com/en/actions/using-workflows/reusing-workflows#calling-a-reusable-workflow) rather than from steps within a job. For instance, to call the release workflow from a workflow in a repository containing a Python package named `mypackage` which should be built with Python 3.9: ```yaml -name: Release +name: Release mypackage on: push: branches: @@ -42,10 +56,72 @@ on: # third phase trigger after release is published - published jobs: + # configure options which may be set as dispatch + # inputs or dynamically assigned default values + set_options: + name: Set release options + if: github.ref_name != 'master' + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash -l {0} + outputs: + branch: ${{ steps.set_branch.outputs.branch }} + version: ${{ steps.set_version.outputs.version }} + steps: + + - name: Set branch + id: set_branch + run: | + # if branch was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.branch }}") ]]; then + branch="${{ inputs.branch }}" + # prevent releases from develop or master + if [[ ("$branch" == "develop") || ("$branch" == "master") ]]; then + echo "error: releases may not be triggered from branch $branch" + exit 1 + fi + echo "using branch $branch from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, use that branch + branch="${{ github.ref_name }}" + echo "using branch $branch from ref ${{ github.ref }}" + else + # otherwise exit with an error + echo "error: this workflow should not have triggered for event ${{ github.event_name }} on branch ${{ github.ref_name }}" + exit 1 + fi + echo "branch=$branch" >> $GITHUB_OUTPUT + + - name: Set version + id: set_version + run: | + # if version number was provided explicitly via workflow_dispatch, use it + if [[ ("${{ github.event_name }}" == "workflow_dispatch") && (-n "${{ inputs.version }}") ]]; then + ver="${{ inputs.version }}" + echo "using version number $ver from workflow_dispatch" + elif [[ ("${{ github.event_name }}" == "push") && ("${{ github.ref_name }}" != "master") ]]; then + # if release was triggered by pushing a release branch, parse version number from branch name (sans leading 'v') + ref="${{ github.ref_name }}" + ver="${ref#"v"}" + echo "parsed version number $ver from branch name $ref" + else + # otherwise exit with an error + echo "error: version number not provided explicitly (via workflow_dispatch input) or implicitly (via branch name)" + exit 1 + fi + echo "version=$ver" >> $GITHUB_OUTPUT + release: - uses: MODFLOW-USGS/modflow-devtools/.github/workflows/reusable_release.yml@main + uses: MODFLOW-USGS/modflow-devtools/.github/workflows/release.yml@main with: + branch: ${{ needs.set_options.outputs.branch }} + draft_release: true package_name: mypackage - python_version: 3.9 - trunk_branch: master + publish_package: true + python: '3.9' + reset_develop: true + run_tests: true + trunk_branch: main # substitute master if needed + version: '0.1.2' ``` \ No newline at end of file From 6907ba89045ac8e8377f6af67708f781139fc0bc Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Thu, 29 Jun 2023 12:45:31 -0400 Subject: [PATCH 09/10] allow specifying version to reset develop branch to --- .github/workflows/release.yml | 29 +++++++++++++++++--------- .github/workflows/release_dispatch.yml | 10 ++++----- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5fef234..472eb6e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,11 @@ on: required: false default: true type: boolean + environment_name: + description: Name of the GitHub environment to use for the release. + required: false + default: release + type: string package_name: # currently assumes module dir is in project root, # and module name is the same as package name with @@ -42,11 +47,11 @@ on: required: false default: '3.8' type: string - reset_develop: - description: Reset the develop branch from the trunk. + reset_develop_version: + description: Version to reset the develop branch to. required: false - default: true - type: boolean + default: '' + type: string run_tests: # currently assumes tests are in autotest/ description: Run tests after building binaries. @@ -203,8 +208,8 @@ jobs: contents: write pull-requests: write id-token: write # mandatory for trusted publishing - environment: # requires a 'pypi' environment in repo settings - name: pypi + environment: + name: ${{ inputs.environment_name }} url: https://pypi.org/p/${{ inputs.package_name }} steps: @@ -220,6 +225,10 @@ jobs: with: python-version: ${{ inputs.python }} + # actions/download-artifact won't look at previous workflow runs but we need to in order to get changelog + - name: Download artifacts + uses: dawidd6/action-download-artifact@v2 + - name: Install Python dependencies run: | pip install --upgrade pip @@ -228,15 +237,16 @@ jobs: - name: Check package run: twine check --strict dist/* + # looks in dist/ dir by default - name: Publish package - run: twine upload dist/* + uses: pypa/gh-action-pypi-publish@release/v1 reset_pr: name: Draft reset PR needs: - make_dist - release - if: github.event_name == 'release' && (inputs.reset_develop == '' || inputs.reset_develop == true) + if: github.event_name == 'release' && inputs.reset_develop_version != '' runs-on: ubuntu-22.04 permissions: contents: write @@ -279,8 +289,7 @@ jobs: # update version (add + to version.txt to indicate development status) package=${{ inputs.package_name }} module=${package//-/_} - version=$(python update_version.py -g) - python scripts/update_version.py -v "$version+" + python scripts/update_version.py -v "${{ inputs.reset_develop_version }}" black -v $module/version.py # commit and push reset branch diff --git a/.github/workflows/release_dispatch.yml b/.github/workflows/release_dispatch.yml index b0264f0a..da852333 100644 --- a/.github/workflows/release_dispatch.yml +++ b/.github/workflows/release_dispatch.yml @@ -34,11 +34,11 @@ on: required: false default: '3.8' type: string - reset_develop: - description: Reset the develop branch from the trunk. + reset_develop_version: + description: Version to reset the develop branch to. required: false - default: true - type: boolean + default: '' + type: string version: description: Version number to use for release. required: true @@ -125,7 +125,7 @@ jobs: package_name: modflow-devtools publish_package: false python: ${{ needs.set_options.outputs.python }} - reset_develop: true + reset_develop_version: ${{ inputs.reset_develop_version }} run_tests: true trunk_branch: main version: ${{ needs.set_options.outputs.version }} From 6e18bad2b0a0067cec5461b3d1fae37c734d9a37 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Fri, 30 Jun 2023 18:57:54 -0400 Subject: [PATCH 10/10] fix python install in prepare release job in make_package.yml --- .github/workflows/make_package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/make_package.yml b/.github/workflows/make_package.yml index ea608a2a..d756ab4d 100644 --- a/.github/workflows/make_package.yml +++ b/.github/workflows/make_package.yml @@ -73,6 +73,7 @@ jobs: pip install --upgrade pip pip install black build twine pip install . + pip install ".[lint, test]" - name: Update version id: version