diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ab34fba3..24a66e1e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,6 +5,10 @@ on: pull_request: types: [opened, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: lint: runs-on: ubuntu-latest diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml index 1ff4c1c9..1eb5453b 100644 --- a/.github/workflows/readme.yml +++ b/.github/workflows/readme.yml @@ -5,6 +5,10 @@ on: pull_request: types: [opened, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: readme: runs-on: ${{ matrix.os }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..6fcb92cb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,81 @@ +name: Release +on: + release: + types: [published] + branches: + - main + - stable + + workflow_dispatch: + inputs: + candidate: + description: 'Release candidate.' + required: true + type: boolean + default: true + test_pypi: + description: 'Test PyPI.' + type: boolean + default: false +jobs: + release: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.candidate && 'main' || 'stable' }} + + - name: Set up latest Python + uses: actions/setup-python@v5 + with: + python-version-file: 'pyproject.toml' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install .[dev] + + - name: Create wheel + run: | + make dist + + - name: Publish a Python distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: ${{ inputs.test_pypi && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} + + - name: Bump version to next candidate + if: ${{ inputs.candidate && !inputs.test_pypi }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + bump-my-version bump candidate --no-tag --no-commit + + - name: Create pull request + if: ${{ inputs.candidate && !inputs.test_pypi }} + id: cpr + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ secrets.GH_ACCESS_TOKEN }} + commit-message: bumpversion-candidate + committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> + signoff: false + delete-branch: true + title: Automated Bump Version Candidate + body: "This is an auto-generated PR that bumps the version to the next candidate." + branch: bumpversion-candidate-update + branch-suffix: short-commit-hash + add-paths: | + ctgan/__init__.py + pyproject.toml + draft: false + base: main + + - name: Enable Pull Request Automerge + if: ${{ steps.cpr.outputs.pull-request-operation == 'created' }} + run: gh pr merge "${{ steps.cpr.outputs.pull-request-number }}" --squash --admin + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.gitignore b/.gitignore index 2482c08f..0c950932 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.github/.tmp/ +tests/readme_test/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -104,5 +107,8 @@ ENV/ # mypy .mypy_cache/ +# other +.DS_Store + # Vim -.*.swp +.*.swp \ No newline at end of file diff --git a/Makefile b/Makefile index dfd4360e..d5cd8e80 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,9 @@ install-test: clean-build clean-pyc ## install the package and test dependencies install-develop: clean-build clean-pyc ## install the package in editable mode and dependencies for development pip install -e .[dev] +.PHONY: install-readme +install-readme: clean-build clean-pyc ## install the package in editable mode and readme dependencies for developement + pip install -e .[readme] # LINT TARGETS @@ -87,7 +90,6 @@ lint: fix-lint: invoke fix-lint - # TEST TARGETS .PHONY: test-unit @@ -112,10 +114,6 @@ test: test-unit test-integration test-readme ## test everything that needs test .PHONY: test-devel test-devel: lint ## test everything that needs development dependencies -.PHONY: test-all -test-all: ## run tests on every Python version with tox - tox -r - .PHONY: coverage coverage: ## check code coverage quickly with the default Python @@ -147,26 +145,31 @@ publish-test: dist publish-confirm ## package and upload a release on TestPyPI publish: dist publish-confirm ## package and upload a release twine upload dist/* -.PHONY: bumpversion-release -bumpversion-release: ## Merge main to stable and bumpversion release +.PHONY: git-merge-main-stable +git-merge-main-stable: ## Merge main into stable git checkout stable || git checkout -b stable git merge --no-ff main -m"make release-tag: Merge branch 'main' into stable" - bump-my-version bump release + +.PHONY: git-merge-stable-main +git-merge-stable-main: ## Merge stable into main + git checkout main + git merge stable + +.PHONY: git-push +git-push: ## Simply push the repository to github + git push + +.PHONY: git-push-tags-stable +git-push-tags-stable: ## Push tags and stable to github git push --tags origin stable -.PHONY: bumpversion-release-test -bumpversion-release-test: ## Merge main to stable and bumpversion release - git checkout stable || git checkout -b stable - git merge --no-ff main -m"make release-tag: Merge branch 'main' into stable" +.PHONY: bumpversion-release +bumpversion-release: ## Bump the version to the next release bump-my-version bump release --no-tag - @echo git push --tags origin stable .PHONY: bumpversion-patch -bumpversion-patch: ## Merge stable to main and bumpversion patch - git checkout main - git merge stable +bumpversion-patch: ## Bump the version to the next patch bump-my-version bump --no-tag patch - git push .PHONY: bumpversion-candidate bumpversion-candidate: ## Bump the version to the next candidate @@ -182,11 +185,13 @@ bumpversion-major: ## Bump the version the next major skipping the release .PHONY: bumpversion-revert bumpversion-revert: ## Undo a previous bumpversion-release + git tag --delete $(shell git tag --points-at HEAD) git checkout main git branch -D stable CLEAN_DIR := $(shell git status --short | grep -v ??) CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) +CURRENT_VERSION := $(shell grep "^current_version" pyproject.toml | grep -o "dev[0-9]*") CHANGELOG_LINES := $(shell git diff HEAD..origin/stable HISTORY.md 2>&1 | wc -l) .PHONY: check-clean @@ -201,41 +206,36 @@ ifneq ($(CURRENT_BRANCH),main) $(error Please make the release from main branch\n) endif +.PHONY: check-candidate +check-candidate: ## Check if a release candidate has been made +ifeq ($(CURRENT_VERSION),dev0) + $(error Please make a release candidate and test it before atempting a release) +endif + .PHONY: check-history check-history: ## Check if HISTORY.md has been modified ifeq ($(CHANGELOG_LINES),0) $(error Please insert the release notes in HISTORY.md before releasing) endif -.PHONY: git-push -git-push: ## Simply push the repository to github - git push +.PHONY: check-deps +check-deps: # Dependency targets + $(eval allow_list='numpy=|pandas=|tqdm=|torch=|rdt=') + pip freeze | grep -v "CTGAN.git" | grep -E $(allow_list) > $(OUTPUT_FILEPATH) .PHONY: check-release -check-release: check-clean check-main check-history ## Check if the release can be made +check-release: check-clean check-candidate check-main check-history ## Check if the release can be made @echo "A new release can be made" .PHONY: release -release: check-release bumpversion-release publish bumpversion-patch +release: check-release git-merge-main-stable bumpversion-release git-push-tags-stable \ + git-merge-stable-main bumpversion-patch git-push .PHONY: release-test -release-test: check-release bumpversion-release-test publish-test bumpversion-revert +release-test: check-release git-merge-main-stable bumpversion-release bumpversion-revert .PHONY: release-candidate release-candidate: check-main publish bumpversion-candidate git-push .PHONY: release-candidate-test -release-candidate-test: check-clean check-main publish-test - -.PHONY: release-minor -release-minor: check-release bumpversion-minor release - -.PHONY: release-major -release-major: check-release bumpversion-major release - -# Dependency targets - -.PHONY: check-deps -check-deps: - $(eval allow_list='numpy=|pandas=|tqdm=|torch=|rdt=') - pip freeze | grep -v "CTGAN.git" | grep -E $(allow_list) > $(OUTPUT_FILEPATH) +release-candidate-test: check-clean check-main publish-test \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..d54be0b3 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,174 @@ +# Release workflow + +The process of releasing a new version involves several steps: + +1. [Install CTGAN from source](#install-ctgan-from-source) + +2. [Linting and tests](#linting-and-tests) + +3. [Make a release candidate](#make-a-release-candidate) + +4. [Integration with SDV](#integration-with-sdv) + +5. [Milestone](#milestone) + +6. [Update HISTORY](#update-history) + +7. [Check the release](#check-the-release) + +8. [Update stable branch and bump version](#update-stable-branch-and-bump-version) + +9. [Create the Release on GitHub](#create-the-release-on-github) + +10. [Close milestone and create new milestone](#close-milestone-and-create-new-milestone) + +## Install CTGAN from source + +Clone the project and install the development requirements before starting the release process. Alternatively, with your virtualenv activated: + +```bash +git clone https://github.com/sdv-dev/CTGAN.git +cd CTGAN +git checkout main +make install-develop +``` + +## Linting and tests + +Execute the tests and linting. The tests must end with no errors: + +```bash +make test && make lint +``` + +And you will see something like this: + +``` +Coverage XML written to file ./integration_cov.xml + +======================= 24 passed, 7 warnings in 51.23s ======================== +.... +invoke lint +No broken requirements found. +All checks passed! +28 files already formatted +``` + +The execution has finished with no errors, 0 test skipped and 166 warnings. + +## Make a release candidate + +1. On the CTGAN GitHub page, navigate to the [Actions][actions] tab. +2. Select the `Release` action. +3. Run it on the main branch. Make sure `Release candidate` is checked and `Test PyPI` is not. +4. Check on [PyPI][ctgan-pypi] to assure the release candidate was successfully uploaded. + - You should see X.Y.ZdevN PRE-RELEASE + +[actions]: https://github.com/sdv-dev/CTGAN/actions +[ctgan-pypi]: https://pypi.org/project/CTGAN/#history + +## Integration with SDV + +### Create a branch on SDV to test the candidate + +Before doing the actual release, we need to test that the candidate works with SDV. To do this, we can create a branch on SDV that points to the release candidate we just created using the following steps: + +1. Create a new branch on the SDV repository. + +```bash +git checkout -b test-ctgan-X.Y.Z +``` + +2. Update the pyproject.toml to set the minimum version of CTGAN to be the same as the version of the release. For example, + +```toml +'ctgan>=X.Y.Z.dev0' +``` + +3. Push this branch. This should trigger all the tests to run. + +```bash +git push --set-upstream origin test-ctgan-X.Y.Z +``` + +4. Check the [Actions][sdv-actions] tab on SDV to make sure all the tests pass. + +[sdv-actions]: https://github.com/sdv-dev/SDV/actions + +## Milestone + +It's important to check that the GitHub and milestone issues are up to date with the release. + +You neet to check that: + +- The milestone for the current release exists. +- All the issues closed since the latest release are associated to the milestone. If they are not, associate them. +- All the issues associated to the milestone are closed. If there are open issues but the milestone needs to + be released anyway, move them to the next milestone. +- All the issues in the milestone are assigned to at least one person. +- All the pull requests closed since the latest release are associated to an issue. If necessary, create issues + and assign them to the milestone. Also assign the person who opened the issue to them. + +## Update HISTORY +Run the [Release Prep](https://github.com/sdv-dev/CTGAN/actions/workflows/prepare_release.yml) workflow. This workflow will create a pull request with updates to HISTORY.md + +Make sure HISTORY.md is updated with the issues of the milestone: + +``` +# History + +## X.Y.Z (YYYY-MM-DD) + +### New Features + +* - [Issue #](https://github.com/sdv-dev/CTGAN/issues/) by @resolver + +### General Improvements + +* - [Issue #](https://github.com/sdv-dev/CTGAN/issues/) by @resolver + +### Bug Fixed + +* - [Issue #](https://github.com/sdv-dev/CTGAN/issues/) by @resolver +``` + +The issue list per milestone can be found [here][milestones]. + +[milestones]: https://github.com/sdv-dev/CTGAN/milestones + +Put the pull request up for review and get 2 approvals to merge into `main`. + +## Check the release +Once HISTORY.md has been updated on `main`, check if the release can be made: + +```bash +make check-release +``` + +## Update stable branch and bump version +The `stable` branch needs to be updated with the changes from `main` and the version needs to be bumped. +Depending on the type of release, run one of the following: + +* `make release`: This will release a patch, which is the most common type of release. Use this when the changes are bugfixes or enhancements that do not modify the existing user API. Changes that modify the user API to add new features but that do not modify the usage of the previous features can also be released as a patch. +* `make release-minor`: This will release the next minor version. Use this if the changes modify the existing user API in any way, even if it is backwards compatible. Minor backwards incompatible changes can also be released as minor versions while the library is still in beta state. After the major version 1 has been released, minor version can only be used to add backwards compatible API changes. +* `make release-major`: This will release the next major version. Use this to if the changes modify the user API in a backwards incompatible way after the major version 1 has been released. + +Running one of these will **push commits directly** to `main`. +At the end, you should see the 3 commits on `main` (from oldest to newest): +- `make release-tag: Merge branch 'main' into stable` +- `Bump version: X.Y.Z.devN → X.Y.Z` +- `Bump version: X.Y.Z -> X.Y.A.dev0` + +## Create the Release on GitHub + +After the update to HISTORY.md is merged into `main` and the version is bumped, it is time to [create the release GitHub](https://github.com/sdv-dev/CTGAN/releases/new). +- Create a new tag with the version number with a v prefix (e.g. v0.3.1) +- The target should be the `stable` branch +- Release title is the same as the tag (e.g. v0.3.1) +- This is not a pre-release (`Set as a pre-release` should be unchecked) + +Click `Publish release`, which will kickoff the release workflow and automatically upload the package to [public PyPI](https://pypi.org/project/ctgan/). + +## Close milestone and create new milestone + +Finaly, **close the milestone** and, if it does not exist, **create the next milestone**. diff --git a/pyproject.toml b/pyproject.toml index 2cece31a..577626a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,6 @@ dev = [ # Advanced testing 'coverage>=4.5.1,<6', - 'tox>=2.9.1,<4', 'invoke', ] diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b344cf45..00000000 --- a/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -envlist = py39-lint, py3{8,9,10,11,12,13}-{unit,integration,readme} - -[testenv] -skipsdist = false -skip_install = false -deps = - invoke - readme: rundoc -extras = - lint: dev - unit: test - integration: test -commands = - lint: invoke lint - unit: invoke unit - integration: invoke integration - readme: invoke readme - invoke rmdir --path {envdir}