From a2f5c3c4d10ff18920b8e8dcc08e02e165dc8322 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 23 Oct 2024 10:45:15 +0200 Subject: [PATCH 01/11] Use new release and pre_release workflows --- .github/workflows/_changelog_entry_check.yaml | 30 ---- .../workflows/_version_conflict_check.yaml | 30 ---- .github/workflows/pre_release.yaml | 74 ++++++++ .github/workflows/release.yaml | 108 ++++++++++++ .github/workflows/run_release.yaml | 166 ------------------ scripts/check_changelog_entry.py | 24 --- scripts/check_version_conflict.py | 14 -- scripts/print_current_package_version.py | 7 - scripts/update_version_for_prerelease.py | 48 ----- 9 files changed, 182 insertions(+), 319 deletions(-) delete mode 100644 .github/workflows/_changelog_entry_check.yaml delete mode 100644 .github/workflows/_version_conflict_check.yaml create mode 100644 .github/workflows/pre_release.yaml create mode 100644 .github/workflows/release.yaml delete mode 100644 .github/workflows/run_release.yaml delete mode 100755 scripts/check_changelog_entry.py delete mode 100755 scripts/check_version_conflict.py delete mode 100755 scripts/print_current_package_version.py delete mode 100755 scripts/update_version_for_prerelease.py diff --git a/.github/workflows/_changelog_entry_check.yaml b/.github/workflows/_changelog_entry_check.yaml deleted file mode 100644 index 78ebefc5..00000000 --- a/.github/workflows/_changelog_entry_check.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Changelog entry check - -on: - workflow_call: - -env: - PYTHON_VERSION: 3.12 - -jobs: - check_changelog_entry: - name: Changelog entry check - runs-on: ubuntu-latest - if: (!startsWith(github.event.pull_request.title, 'docs:')) - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - name: Run changelog entry check - run: make check-changelog-entry diff --git a/.github/workflows/_version_conflict_check.yaml b/.github/workflows/_version_conflict_check.yaml deleted file mode 100644 index e02341b4..00000000 --- a/.github/workflows/_version_conflict_check.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Version conflict check - -on: - workflow_call: - -env: - PYTHON_VERSION: 3.12 - -jobs: - check_version_conflict: - name: Version conflict check - runs-on: ubuntu-latest - if: (!startsWith(github.event.pull_request.title, 'docs:')) - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - name: Run version conflict check - run: make check-version-conflict diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml new file mode 100644 index 00000000..93b0f491 --- /dev/null +++ b/.github/workflows/pre_release.yaml @@ -0,0 +1,74 @@ +name: Create a pre-release + +on: + # Trigger a beta version release (pre-release) on push to the master branch. + push: + branches: + - master + tags-ignore: + - "**" # Ignore all tags to prevent duplicate builds when tags are pushed. + +jobs: + release_metadata: + if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci')" + name: Prepare release metadata + runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.release_metadata.outputs.version_number }} + tag_name: ${{ steps.release_metadata.outputs.tag_name }} + changelog: ${{ steps.release_metadata.outputs.changelog }} + steps: + - uses: apify/workflows/git-cliff-release@main + id: release_metadata + name: Prepare release metadata + with: + release_type: prerelease + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit + + update_changelog: + name: Update changelog + needs: [release_metadata, lint_check, type_check, unit_tests, integration_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + publish_to_pypi: + name: Publish to PyPI + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/apify-client + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: apify-client + is_prerelease: "yes" + version_number: ${{ needs.release_metadata.outputs.version_number }} + ref: ${{ needs.update_changelog.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..c785c4ab --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,108 @@ +name: Create a release + +on: + # Trigger a stable version release via GitHub's UI, with the ability to specify the type of release. + workflow_dispatch: + inputs: + release_type: + description: Release type + required: true + type: choice + default: auto + options: + - auto + - custom + - patch + - minor + - major + custom_version: + description: The custom version to bump to (only for "custom" type) + required: false + type: string + default: "" + +jobs: + release_metadata: + name: Prepare release metadata + runs-on: ubuntu-latest + outputs:etadata, lint_check, type_check, unit_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + version_number: ${{ steps.release_metadata.outputs.version_number }} + tag_name: ${{ steps.release_metadata.outputs.tag_name }} + changelog: ${{ steps.release_metadata.outputs.changelog }} + release_notes: ${{ steps.release_metadata.outputs.release_notes }} + steps: + - uses: apify/workflows/git-cliff-release@main + name: Prepare release metadata + id: release_metadata + with: + release_type: ${{ inputs.release_type }} + custom_version: ${{ inputs.custom_version }} + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit + + update_changelog: + name: Update changelog + needs: [release_metadata, lint_check, type_check, unit_tests, integration_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + create_github_release: + name: Create github release + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Create release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.release_metadata.outputs.tag_name }} + name: ${{ needs.release_metadata.outputs.version_number }} + target_commitish: ${{ needs.update_changelog.outputs.changelog_commitish }} + body: ${{ needs.release_metadata.outputs.release_notes }} + + publish_to_pypi: + name: Publish to PyPI + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/apify-client + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: apify-client + is_prerelease: "" + version_number: ${{ needs.release_metadata.outputs.version_number }} + ref: ${{ needs.update_changelog.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml deleted file mode 100644 index 95eb8d11..00000000 --- a/.github/workflows/run_release.yaml +++ /dev/null @@ -1,166 +0,0 @@ -name: Run release - -on: - # Push to master will publish a beta version. - push: - branches: - - master - tags-ignore: - - "**" - - # A release via GitHub releases will publish a stable version. - release: - types: [published] - - # Workflow dispatch will publish whatever you choose. - workflow_dispatch: - inputs: - release_type: - description: Release type - required: true - type: choice - default: alpha - options: - - alpha - - beta - - final - -env: - PYTHON_VERSION: 3.12 - -jobs: - should_release: - name: Check whether to release - if: (!startsWith(github.event.head_commit.message, 'docs:') || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - steps: - - name: Dummy step - run: "true" - - lint_check: - name: Lint check - needs: [should_release] - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main - - type_check: - name: Type check - needs: [should_release] - uses: apify/workflows/.github/workflows/python_type_check.yaml@main - - unit_tests: - name: Unit tests - needs: [should_release] - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - - async_docstrings: - name: Async dostrings check - uses: ./.github/workflows/_async_docstrings_check.yaml - - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - changelog_entry_check: - name: Changelog entry check - needs: [should_release] - uses: ./.github/workflows/_changelog_entry_check.yaml - - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - version_conflict_check: - name: Version conflict check - needs: [should_release] - uses: ./.github/workflows/_version_conflict_check.yaml - - integration_tests: - name: Integration tests - needs: [should_release] - uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - secrets: inherit - - publish_to_pypi: - name: Publish to PyPI - needs: - [ - should_release, - lint_check, - type_check, - unit_tests, - async_docstrings, - changelog_entry_check, - version_conflict_check, - integration_tests, - ] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - environment: - name: pypi - url: https://pypi.org/project/apify-client/ - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Python dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - # Determine if this is a prerelease or latest release - name: Determine release type - id: get-release-type - run: | - if [ ${{ github.event_name }} = release ]; then - release_type="final" - elif [ ${{ github.event_name }} = push ]; then - release_type="beta" - elif [ ${{ github.event_name }} = workflow_dispatch ]; then - release_type=${{ github.event.inputs.release_type }} - fi - - if [ ${release_type} = final ]; then - docker_image_tag="latest" - elif [ ${release_type} = beta ]; then - docker_image_tag="beta" - else - docker_image_tag="" - fi - - echo "release_type=${release_type}" >> $GITHUB_OUTPUT - echo "docker_image_tag=${docker_image_tag}" >> $GITHUB_OUTPUT - - - # Check whether the released version is listed in CHANGELOG.md - name: Check whether the released version is listed in the changelog - if: steps.get-release-type.outputs.release_type != 'alpha' - run: make check-changelog-entry - - - # Check version consistency and increment pre-release version number for prereleases (must be the last step before build) - name: Bump pre-release version - if: steps.get-release-type.outputs.release_type != 'final' - run: python ./scripts/update_version_for_prerelease.py ${{ steps.get-release-type.outputs.release_type }} - - # Builds the package. - - name: Build package - run: make build - - # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - - # Tag the current commit with the version tag if this is not made from the release event (releases are tagged with the release process) - name: Tag Version - if: github.event_name != 'release' - run: | - git_tag=v`python ./scripts/print_current_package_version.py` - git tag $git_tag - git push origin $git_tag - - - # Upload the build artifacts to the release - name: Upload the build artifacts to release - if: github.event_name == 'release' - run: gh release upload ${{ github.ref_name }} dist/* - env: - GH_TOKEN: ${{ github.token }} diff --git a/scripts/check_changelog_entry.py b/scripts/check_changelog_entry.py deleted file mode 100755 index 8abb3945..00000000 --- a/scripts/check_changelog_entry.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -import re - -from utils import REPO_ROOT, get_current_package_version - -CHANGELOG_PATH = REPO_ROOT / 'CHANGELOG.md' - -# Checks whether the current package version has an entry in the CHANGELOG.md file -if __name__ == '__main__': - current_package_version = get_current_package_version() - - if not CHANGELOG_PATH.is_file(): - raise RuntimeError('Unable to find CHANGELOG.md file') - - with open(CHANGELOG_PATH, encoding='utf-8') as changelog_file: - for line in changelog_file: - # Ensure that the heading in the changelog entry for the specified version includes a version number - # enclosed in square brackets. This version number is formatted as a link to the corresponding - # version tag on GitHub. - if re.match(rf'## \[{current_package_version}\].*$', line): - break - else: - raise RuntimeError(f'There is no entry in the changelog for the current package version ({current_package_version})') diff --git a/scripts/check_version_conflict.py b/scripts/check_version_conflict.py deleted file mode 100755 index 57b6a4ea..00000000 --- a/scripts/check_version_conflict.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -from utils import get_current_package_version, get_published_package_versions - -# Checks whether the current package version number was not already used in a published release. -if __name__ == '__main__': - current_version = get_current_package_version() - - # Load the version numbers of the currently published versions from PyPI - published_versions = get_published_package_versions() - - # We don't want to try to publish a version with the same version number as an already released stable version - if current_version in published_versions: - raise RuntimeError(f'The current version {current_version} was already released!') diff --git a/scripts/print_current_package_version.py b/scripts/print_current_package_version.py deleted file mode 100755 index 3c78a5b6..00000000 --- a/scripts/print_current_package_version.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 - -from utils import get_current_package_version - -# Print the current package version from the pyproject.toml file to stdout -if __name__ == '__main__': - print(get_current_package_version(), end='') diff --git a/scripts/update_version_for_prerelease.py b/scripts/update_version_for_prerelease.py deleted file mode 100755 index 236fbf3e..00000000 --- a/scripts/update_version_for_prerelease.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys - -from utils import get_current_package_version, get_published_package_versions, set_current_package_version - -# Checks whether the current package version number was not already used in a published release, -# and if not, modifies the package version number in pyproject.toml -# from a stable release version (X.Y.Z) to a prerelease version (X.Y.ZbN or X.Y.Z.aN or X.Y.Z.rcN) -if __name__ == '__main__': - if len(sys.argv) != 2: - raise RuntimeError('You must pass the prerelease type as an argument to this script!') - - prerelease_type = sys.argv[1] - if prerelease_type not in ['alpha', 'beta', 'rc']: - raise RuntimeError(f'The prerelease type must be one of "alpha", "beta" or "rc", got "{prerelease_type}"!') - - if prerelease_type == 'alpha': - prerelease_prefix = 'a' - elif prerelease_type == 'beta': - prerelease_prefix = 'b' - elif prerelease_type == 'rc': - prerelease_prefix = 'rc' - - current_version = get_current_package_version() - - # We can only transform a stable release version (X.Y.Z) to a prerelease version (X.Y.ZxxxN) - if not re.match(r'^\d+\.\d+\.\d+$', current_version): - raise RuntimeError(f'The current version {current_version} does not match the proper semver format for stable releases (X.Y.Z)') - - # Load the version numbers of the currently published versions from PyPI - published_versions = get_published_package_versions() - - # We don't want to publish a prerelease version with the same version number as an already released stable version - if current_version in published_versions: - raise RuntimeError(f'The current version {current_version} was already released!') - - # Find the highest prerelease version number that was already published - latest_prerelease = 0 - for version in published_versions: - if version.startswith(f'{current_version}{prerelease_prefix}'): - prerelease_version = int(version.split(prerelease_prefix)[1]) - latest_prerelease = max(prerelease_version, latest_prerelease) - - # Write the latest prerelease version number to pyproject.toml - new_prerelease_version_number = f'{current_version}{prerelease_prefix}{latest_prerelease + 1}' - set_current_package_version(new_prerelease_version_number) From e4077f6a111d171e53b9ca5d0aa94d8ef16ff1db Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 29 Oct 2024 14:05:17 +0100 Subject: [PATCH 02/11] Prepend to changelog instead of overwriting --- .github/workflows/pre_release.yaml | 1 + .github/workflows/release.yaml | 1 + CHANGELOG.md | 27 +++++++++++++++++---------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml index 93b0f491..325f0198 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/pre_release.yaml @@ -23,6 +23,7 @@ jobs: name: Prepare release metadata with: release_type: prerelease + existing_changelog_path: CHANGELOG.md lint_check: name: Lint check diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c785c4ab..4da8b532 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -42,6 +42,7 @@ jobs: with: release_type: ${{ inputs.release_type }} custom_version: ${{ inputs.custom_version }} + existing_changelog_path: CHANGELOG.md lint_check: name: Lint check diff --git a/CHANGELOG.md b/CHANGELOG.md index ca825fde..f3a8fead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,26 +1,33 @@ # Changelog -## [1.8.2](../../releases/tag/v1.8.2) - Unreleased +All notable changes to this project will be documented in this file. -... + +## 1.9.0 - **not yet released** -## [1.8.1](../../releases/tag/v1.8.1) - 2024-09-17 +### 🚀 Features -### Fixed +- Add user.update_limits ([#279](https://github.com/apify/apify-client-python/pull/279)) ([7aed9c9](https://github.com/apify/apify-client-python/commit/7aed9c928958831168ac8d293538d6fd3adbc5e5)) by [@MFori](https://github.com/MFori), closes [#329](https://github.com/apify/apify-client-python/issues/329) -- batch add requests can handle more than 25 requests -## [1.8.0](../../releases/tag/v1.8.0) - 2024-08-30 + +## [1.8.1](https://github.com/apify/apify-client-python/releases/tags/v1.8.1) (2024-09-17) -### Added +### 🐛 Bug Fixes + +- Batch add requests can handle more than 25 requests ([#268](https://github.com/apify/apify-client-python/pull/268)) ([9110ee0](https://github.com/apify/apify-client-python/commit/9110ee08954762aed00ac09cd042e802c1d041f7)) by [@vdusek](https://github.com/vdusek), closes [#264](https://github.com/apify/apify-client-python/issues/264) -- add `headers_template` kwarg to webhook create and update -- allow passing list of fields to `unwind` parameter in dataset item listing endpoints -### Others +## [1.8.0](https://github.com/apify/apify-client-python/releases/tags/v1.8.0) (2024-08-30) - drop support for Python 3.8 +### 🚀 Features + +- Adds headers_template to webhooks and webhooks_collection ([#239](https://github.com/apify/apify-client-python/pull/239)) ([6dbd781](https://github.com/apify/apify-client-python/commit/6dbd781d24d9deb6a7669193ce4d5a4190fe5026)) by [@jakerobers](https://github.com/jakerobers) +- Add actor standby ([#248](https://github.com/apify/apify-client-python/pull/248)) ([dd4bf90](https://github.com/apify/apify-client-python/commit/dd4bf9072a4caa189af5f90e513e37df325dc929)) by [@jirimoravcik](https://github.com/jirimoravcik) +- Allow passing list of fields to unwind parameter ([#256](https://github.com/apify/apify-client-python/pull/256)) ([036b455](https://github.com/apify/apify-client-python/commit/036b455c51243e0ef81cb74a44fe670abc085ce7)) by [@fnesveda](https://github.com/fnesveda) + ## [1.7.1](../../releases/tag/v1.7.1) - 2024-07-11 ### Fixed From c8134c5b75b17ff348ffb56a0b82d3ef9830a61e Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 29 Oct 2024 14:10:53 +0100 Subject: [PATCH 03/11] Remove unnecessary checks --- .github/workflows/pre_release.yaml | 4 ++++ .github/workflows/release.yaml | 4 ++++ Makefile | 11 ++--------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml index 325f0198..268f0cf8 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/pre_release.yaml @@ -37,6 +37,10 @@ jobs: name: Unit tests uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + async_docstrings: + name: Async dostrings check + uses: ./.github/workflows/_async_docstrings_check.yaml + integration_tests: name: Integration tests uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4da8b532..25ad4a0c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -56,6 +56,10 @@ jobs: name: Unit tests uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + async_docstrings: + name: Async dostrings check + uses: ./.github/workflows/_async_docstrings_check.yaml + integration_tests: name: Integration tests uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main diff --git a/Makefile b/Makefile index 002b2112..4ffbae5c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ .PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ integration-tests format check-code check-async-docstrings fix-async-docstrings \ - check-version-availability check-changelog-entry check-version-conflict build-api-reference \ - run-docs + build-api-reference run-docs DIRS_WITH_CODE = src tests scripts @@ -46,15 +45,9 @@ format: check-async-docstrings: poetry run python scripts/check_async_docstrings.py -check-changelog-entry: - poetry run python scripts/check_changelog_entry.py - -check-version-conflict: - poetry run python scripts/check_version_conflict.py - # The check-code target runs a series of checks equivalent to those performed by pre-commit hooks # and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests check-async-docstrings check-changelog-entry check-version-conflict +check-code: lint type-check unit-tests check-async-docstrings fix-async-docstrings: poetry run python scripts/fix_async_docstrings.py From b9a100f28eb15cd4de05b555b64141970427ca77 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 29 Oct 2024 14:11:48 +0100 Subject: [PATCH 04/11] Update version in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7f2eb97c..7d395f33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "apify_client" -version = "1.8.2" +version = "1.9.0" description = "Apify API client for Python" authors = ["Apify Technologies s.r.o. "] license = "Apache-2.0" From d2e68eeda36a4bc6f0ff5cf2f62ee46bf5762919 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 29 Oct 2024 17:54:49 +0100 Subject: [PATCH 05/11] Clean up precommit config --- .pre-commit-config.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f2c3500..b574f024 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,15 +18,3 @@ repos: entry: make check-async-docstrings language: system pass_filenames: false - - - id: check-changelog-entry - name: Check changelog entry - entry: make check-changelog-entry - language: system - pass_filenames: false - - - id: check-version-conflict - name: Check version conflict - entry: make check-version-conflict - language: system - pass_filenames: false From bb3d467635b6694dfe0017e7ac04b02efe5e6059 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 29 Oct 2024 17:55:27 +0100 Subject: [PATCH 06/11] Do not run pre_release in forks --- .github/workflows/pre_release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml index 268f0cf8..b328879d 100644 --- a/.github/workflows/pre_release.yaml +++ b/.github/workflows/pre_release.yaml @@ -10,7 +10,7 @@ on: jobs: release_metadata: - if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci')" + if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci') && startsWith(github.repository, 'apify/')" name: Prepare release metadata runs-on: ubuntu-latest outputs: From 19fb5c1df88741a4f85049d268ce0b91a3970461 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 30 Oct 2024 10:54:01 +0100 Subject: [PATCH 07/11] Update CONTRIBUTING.md --- CONTRIBUTING.md | 192 +++++++++++++++++++++++++++--------------------- 1 file changed, 109 insertions(+), 83 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35cdea04..b5344a10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,52 +1,76 @@ # Development +Here you'll find a contributing guide to get started with development. + ## Environment -For local development, it is required to have Python 3.8 (or a later version) installed. +For local development, it is required to have Python 3.9 (or a later version) installed. + +We use [Poetry](https://python-poetry.org/) for project management. Install it and set up your IDE accordingly. -It is recommended to set up a virtual environment while developing this package to isolate your development environment, -however, due to the many varied ways Python can be installed and virtual environments can be set up, -this is left up to the developers to do themselves. +## Dependencies -One recommended way is with the built-in `venv` module: +To install this package and its development dependencies, run: -```bash -python3 -m venv .venv -source .venv/bin/activate +```sh +make install-dev ``` -To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version, -and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder. +## Code checking -## Dependencies +To execute all code checking tools together, run: -To install this package and its development dependencies, run `make install-dev`. +```sh +make check-code +``` -## Code checking +### Linting -To run all our code checking tools together, just run `make check-code`. +We utilize [ruff](https://docs.astral.sh/ruff/) for linting, which analyzes code for potential issues and enforces consistent style. Refer to `pyproject.toml` for configuration details. -### Linting +To run linting: -We use [ruff](https://docs.astral.sh/ruff/) for linting to to analyze the code for potential issues and enforce -uniformed code style. See the `pyproject.toml` for its configuration. To run the linting, just run `make lint`. +```sh +make lint +``` ### Formatting -We use [ruff](https://docs.astral.sh/ruff/) for automated code formatting. It formats the code to follow uniformed -code style and addresses auto-fixable linting issues. See the `pyproject.toml` for its configuration. To run -the formatting, just run `make format`. +Our automated code formatting also leverages [ruff](https://docs.astral.sh/ruff/), ensuring uniform style and addressing fixable linting issues. Configuration specifics are outlined in `pyproject.toml`. + +To run formatting: + +```sh +make format +``` ### Type checking -We use [mypy](https://mypy.readthedocs.io/en/stable/) for type checking. See the `mypy.ini` for its configuration. -To run the type checking, just run `make type-check`. +Type checking is handled by [mypy](https://mypy.readthedocs.io/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`. + +To run type checking: + +```sh +make type-check +``` ### Unit tests -We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. See the `pyproject.toml` for -both its configuration and the list of installed plugins. To run unit tests execute `make unit-tests`. To run unit -tests with HTML coverage report execute `make unit-tests-cov`. +We employ pytest as our testing framework, equipped with various plugins. Check pyproject.toml for configuration details and installed plugins. + +We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. Check `pyproject.toml` for configuration details and installed plugins. + +To run unit tests: + +```sh +make unit-tests +``` + +To run unit tests with HTML coverage report: + +```sh +make unit-tests-cov +``` ## Integration tests @@ -59,82 +83,84 @@ the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to t ## Documentation -We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting the code. -We document every user-facing class or method, and enforce that using the flake8-docstrings library. +We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). -The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy post-processing, -and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published to GitHub pages. +Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. -## Release process +To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps: -Publishing new versions to [PyPI](https://pypi.org/project/apify-client) happens automatically through GitHub Actions. +Navigate to the `website/` directory: -On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml` -and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI. +```sh +cd website/ +``` -A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`. -The built package assets are automatically uploaded to the GitHub release. +Enable Corepack, which installs Yarn automatically: -If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails, -so don't forget to update the version number before releasing a new version. -The release process also fails when the released version is not described in `CHANGELOG.md`, -so don't forget to describe the changes in the new version there. +```sh +corepack enable +``` -### Beta release checklist +Build the API reference: -Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before you do that check the following: +```sh +./build_api_reference.sh +``` -- Make sure that in the [pyproject.toml](./pyproject.toml) a project version is set to the latest non-published version. -- Describe your changes to the [CHANGELOG.md](./CHANGELOG.md) in the section with the latest non-published version. +Install the necessary dependencies: -### Production release checklist +```sh +yarn +``` -Production release happens after the GitHub release is created. Before you do that check the following: +Start the project in development mode with Hot Module Replacement (HMR): -- Make sure [here](https://pypi.org/project/apify-client/#history) that the beta release with the latest commit is successfully deployed. -- Make sure that all changes that happened from the last production release are described in the [CHANGELOG.md](./CHANGELOG.md) (it's okay to skip DX related changes, repo setup etc). -- When drafting a new GitHub release: - - Create a new tag in the format of `v1.2.3` targeting the master branch. - - Fill in the release title in the format of `1.2.3`. - - Copy the changes from the [CHANGELOG.md](./CHANGELOG.md) and paste them into the release description. Make sure that all changes are properly categorized using headlines (`Added`, `Fixed` or `Internal changes`). - - Check the "Set as the latest release" option. +```sh +yarn start +``` -Currently, there is no explicit approval process, so when you are done with the checklist, proceed with the release. +Or using `make`: -Once released, manually bump the version in `pyproject.toml` ([PR example](https://github.com/apify/apify-client-python/pull/180)). +```sh +make run-doc +``` + +## Release process -## Maintanance +Publishing new versions to [PyPI](https://pypi.org/project/crawlee) is automated through GitHub Actions. -### Removing Support for an outdated Python version +- **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI. +- **Stable releases**: A stable version release may be created by triggering the `run_release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overriden using the `custom` release type. -- Todo: Fill in once Python 3.8 is deprecated. +### Publishing to PyPI manually -### Adding support for a new Python version +1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `run_release` workflow instead. +2. **Make sure you know what you're doing.** -1) Firstly, ensure that the package ( - [apify-sdk-python](https://github.com/apify/apify-sdk-python), - [apify-client-python](https://github.com/apify/apify-client-python), - [apify-shared-python](https://github.com/apify/apify-shared-python) -) is compatible with the new Python version. Both in our code base and -the dependencies we use. Then, release a new version of the package. - - For inspiration, see the PR - [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121), - where support for Python 3.12 was added to the Apify Python SDK. +3. Update the version number: -2) Next, build and publish the new versions of Python base Docker images. - - For inspiration, see the PR - [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112), - where support for Python 3.12 was added. - - Apify base Docker images are built using GitHub Actions, accessible at - [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions). +- Modify the `version` field under `tool.poetry` in `pyproject.toml`. -3) Integrate the new Python version into the CI/CD workflows -of existing Python projects ( - [apify-sdk-python](https://github.com/apify/apify-sdk-python), - [apify-client-python](https://github.com/apify/apify-client-python), - [apify-shared-python](https://github.com/apify/apify-shared-python), - [actor-templates](https://github.com/apify/actor-templates) -). - - For inspiration, see the PR - [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124), - where support for Python 3.12 was added to the CI/CD of the Apify Python SDK. +```toml +[tool.poetry] +name = "apify_client" +version = "x.z.y" +``` + +4. Generate the distribution archives for the package: + +```shell +poetry build +``` + +5. Set up the PyPI API token for authentication: + +```shell +poetry config pypi-token.pypi YOUR_API_TOKEN +``` + +6. Upload the package to PyPI: + +```shell +poetry publish +``` From 9cb88b393a211b7dddc2b5f22e67402123c09318 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 30 Oct 2024 10:59:13 +0100 Subject: [PATCH 08/11] Forgotten crawlee --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b5344a10..6c3860dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,7 +127,7 @@ make run-doc ## Release process -Publishing new versions to [PyPI](https://pypi.org/project/crawlee) is automated through GitHub Actions. +Publishing new versions to [PyPI](https://pypi.org/project/apify_client) is automated through GitHub Actions. - **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI. - **Stable releases**: A stable version release may be created by triggering the `run_release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overriden using the `custom` release type. From a79f805b99ec2c83cf0c58f63d550fe5f46eaf32 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 30 Oct 2024 11:00:07 +0100 Subject: [PATCH 09/11] Update workflow name --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c3860dd..44e9daa4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,11 +130,11 @@ make run-doc Publishing new versions to [PyPI](https://pypi.org/project/apify_client) is automated through GitHub Actions. - **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI. -- **Stable releases**: A stable version release may be created by triggering the `run_release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overriden using the `custom` release type. +- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overriden using the `custom` release type. ### Publishing to PyPI manually -1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `run_release` workflow instead. +1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead. 2. **Make sure you know what you're doing.** 3. Update the version number: From 68350bfdd0c057dd3040094c0f71590caa3c96a1 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 30 Oct 2024 11:03:38 +0100 Subject: [PATCH 10/11] Remove checks --- .github/workflows/run_code_checks.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index 684ef60f..b051842f 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -23,16 +23,6 @@ jobs: name: Async dostrings check uses: ./.github/workflows/_async_docstrings_check.yaml - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - changelog_entry_check: - name: Changelog entry check - uses: ./.github/workflows/_changelog_entry_check.yaml - - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - version_conflict_check: - name: Version conflict check - uses: ./.github/workflows/_version_conflict_check.yaml - docs_check: name: Docs check uses: apify/workflows/.github/workflows/python_docs_check.yaml@main From 648f8766dd0a09cfbc0427b6cfe8111b2021d525 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 30 Oct 2024 11:35:26 +0100 Subject: [PATCH 11/11] Update Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4ffbae5c..317e1214 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ - integration-tests format check-code check-async-docstrings fix-async-docstrings \ - build-api-reference run-docs + integration-tests format check-async-docstrings check-code fix-async-docstrings \ + build-api-reference build-docs run-docs DIRS_WITH_CODE = src tests scripts