diff --git a/.github/workflows/_changelog_entry_check.yaml b/.github/workflows/_changelog_entry_check.yaml deleted file mode 100644 index 78ebefc5..00000000 --- a/.github/workflows/_changelog_entry_check.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Changelog entry check - -on: - workflow_call: - -env: - PYTHON_VERSION: 3.12 - -jobs: - check_changelog_entry: - name: Changelog entry check - runs-on: ubuntu-latest - if: (!startsWith(github.event.pull_request.title, 'docs:')) - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - name: Run changelog entry check - run: make check-changelog-entry diff --git a/.github/workflows/_version_conflict_check.yaml b/.github/workflows/_version_conflict_check.yaml deleted file mode 100644 index e02341b4..00000000 --- a/.github/workflows/_version_conflict_check.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Version conflict check - -on: - workflow_call: - -env: - PYTHON_VERSION: 3.12 - -jobs: - check_version_conflict: - name: Version conflict check - runs-on: ubuntu-latest - if: (!startsWith(github.event.pull_request.title, 'docs:')) - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - name: Run version conflict check - run: make check-version-conflict diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml new file mode 100644 index 00000000..7724fcf8 --- /dev/null +++ b/.github/workflows/pre_release.yaml @@ -0,0 +1,90 @@ +name: Create a pre-release + +on: + # Trigger a beta version release (pre-release) on push to the master branch. + push: + branches: + - master + tags-ignore: + - "**" # Ignore all tags to prevent duplicate builds when tags are pushed. + +jobs: + release_metadata: + if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci') && startsWith(github.repository, 'apify/')" + name: Prepare release metadata + runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.release_metadata.outputs.version_number }} + tag_name: ${{ steps.release_metadata.outputs.tag_name }} + changelog: ${{ steps.release_metadata.outputs.changelog }} + existing_changelog_path: CHANGELOG.md + steps: + - uses: apify/workflows/git-cliff-release@main + id: release_metadata + name: Prepare release metadata + with: + release_type: prerelease + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit + + update_changelog: + name: Update changelog + needs: [release_metadata, lint_check, type_check, unit_tests, integration_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + publish_to_pypi: + name: Publish to PyPI + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/apify + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: apify + is_prerelease: "yes" + version_number: ${{ needs.release_metadata.outputs.version_number }} + ref: ${{ needs.update_changelog.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + trigger_docker_build: + name: Trigger Docker image build + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + steps: + - # Trigger building the Python Docker images in apify/apify-actor-docker repo + name: Trigger Docker image build + run: | + gh api -X POST "/repos/apify/apify-actor-docker/dispatches" \ + -F event_type=build-python-images \ + -F client_payload[release_tag]=beta \ + -F client_payload[apify_version]=${{ needs.release_metadata.outputs.version_number }} + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..61b76e6f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,119 @@ +name: Create a release + +on: + # Trigger a stable version release via GitHub's UI, with the ability to specify the type of release. + workflow_dispatch: + inputs: + release_type: + description: Release type + required: true + type: choice + default: auto + options: + - auto + - custom + - patch + - minor + - major + custom_version: + description: The custom version to bump to (only for "custom" type) + required: false + type: string + default: "" + +jobs: + release_metadata: + name: Prepare release metadata + runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.release_metadata.outputs.version_number }} + tag_name: ${{ steps.release_metadata.outputs.tag_name }} + changelog: ${{ steps.release_metadata.outputs.changelog }} + release_notes: ${{ steps.release_metadata.outputs.release_notes }} + steps: + - uses: apify/workflows/git-cliff-release@main + name: Prepare release metadata + id: release_metadata + with: + release_type: ${{ inputs.release_type }} + custom_version: ${{ inputs.custom_version }} + existing_changelog_path: CHANGELOG.md + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit + + update_changelog: + name: Update changelog + needs: [release_metadata, lint_check, type_check, unit_tests, integration_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + create_github_release: + name: Create github release + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Create release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.release_metadata.outputs.tag_name }} + name: ${{ needs.release_metadata.outputs.version_number }} + target_commitish: ${{ needs.update_changelog.outputs.changelog_commitish }} + body: ${{ needs.release_metadata.outputs.release_notes }} + + publish_to_pypi: + name: Publish to PyPI + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/apify + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: apify + is_prerelease: "" + version_number: ${{ needs.release_metadata.outputs.version_number }} + ref: ${{ needs.update_changelog.outputs.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + trigger_docker_build: + name: Trigger Docker image build + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + steps: + - # Trigger building the Python Docker images in apify/apify-actor-docker repo + name: Trigger Docker image build + run: | + gh api -X POST "/repos/apify/apify-actor-docker/dispatches" \ + -F event_type=build-python-images \ + -F client_payload[release_tag]=latest \ + -F client_payload[apify_version]=${{ needs.release_metadata.outputs.version_number }} + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index d6658b63..dd9b4d23 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -19,16 +19,6 @@ jobs: name: Unit tests uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - changelog_entry_check: - name: Changelog entry check - uses: ./.github/workflows/_changelog_entry_check.yaml - - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - version_conflict_check: - name: Version conflict check - uses: ./.github/workflows/_version_conflict_check.yaml - docs_check: name: Docs check uses: apify/workflows/.github/workflows/python_docs_check.yaml@main diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml deleted file mode 100644 index 82164516..00000000 --- a/.github/workflows/run_release.yaml +++ /dev/null @@ -1,172 +0,0 @@ -name: Run release - -on: - # Push to master will publish a beta version. - push: - branches: - - master - tags-ignore: - - "**" - - # A release via GitHub releases will publish a stable version. - release: - types: [published] - - # Workflow dispatch will publish whatever you choose. - workflow_dispatch: - inputs: - release_type: - description: Release type - required: true - type: choice - default: alpha - options: - - alpha - - beta - - final - -env: - PYTHON_VERSION: 3.12 - -jobs: - should_release: - name: Check whether to release - if: (!startsWith(github.event.head_commit.message, 'docs:') || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - steps: - - name: Dummy step - run: "true" - - lint_check: - name: Lint check - needs: [should_release] - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main - - type_check: - name: Type check - needs: [should_release] - uses: apify/workflows/.github/workflows/python_type_check.yaml@main - - unit_tests: - name: Unit tests - needs: [should_release] - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - changelog_entry_check: - name: Changelog entry check - needs: [should_release] - uses: ./.github/workflows/_changelog_entry_check.yaml - - # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved - version_conflict_check: - name: Version conflict check - needs: [should_release] - uses: ./.github/workflows/_version_conflict_check.yaml - - integration_tests: - name: Integration tests - needs: [should_release] - uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - secrets: inherit - - publish_to_pypi: - name: Publish to PyPI - needs: - [ - should_release, - lint_check, - type_check, - unit_tests, - changelog_entry_check, - version_conflict_check, - integration_tests, - ] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - environment: - name: pypi - url: https://pypi.org/project/apify/ - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Python dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - # Determine if this is a prerelease or latest release - name: Determine release type - id: get-release-type - run: | - if [ ${{ github.event_name }} = release ]; then - release_type="final" - elif [ ${{ github.event_name }} = push ]; then - release_type="beta" - elif [ ${{ github.event_name }} = workflow_dispatch ]; then - release_type=${{ github.event.inputs.release_type }} - fi - - if [ ${release_type} = final ]; then - docker_image_tag="latest" - elif [ ${release_type} = beta ]; then - docker_image_tag="beta" - else - docker_image_tag="" - fi - - echo "release_type=${release_type}" >> $GITHUB_OUTPUT - echo "docker_image_tag=${docker_image_tag}" >> $GITHUB_OUTPUT - - - # Check whether the released version is listed in CHANGELOG.md - name: Check whether the released version is listed in the changelog - if: steps.get-release-type.outputs.release_type != 'alpha' - run: make check-changelog-entry - - - # Check version consistency and increment pre-release version number for prereleases (must be the last step before build) - name: Bump pre-release version - if: steps.get-release-type.outputs.release_type != 'final' - run: python ./scripts/update_version_for_prerelease.py ${{ steps.get-release-type.outputs.release_type }} - - # Builds the package. - - name: Build package - run: make build - - # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - - # Tag the current commit with the version tag if this is not made from the release event (releases are tagged with the release process) - name: Tag Version - if: github.event_name != 'release' - run: | - git_tag=v`python ./scripts/print_current_package_version.py` - git tag $git_tag - git push origin $git_tag - - - # Upload the build artifacts to the release - name: Upload the build artifacts to release - if: github.event_name == 'release' - run: gh release upload ${{ github.ref_name }} dist/* - env: - GH_TOKEN: ${{ github.token }} - - - # Trigger building the Python Docker images in apify/apify-actor-docker repo - name: Trigger Docker image build - run: | - PACKAGE_VERSION=`python ./scripts/print_current_package_version.py` - gh api -X POST "/repos/apify/apify-actor-docker/dispatches" \ - -F event_type=build-python-images \ - -F client_payload[release_tag]=${{ steps.get-release-type.outputs.docker_image_tag }} \ - -F client_payload[apify_version]=$PACKAGE_VERSION - env: - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index de911ae3..8a0a7564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,57 +1,58 @@ # Changelog -## [2.0.2](../../releases/tag/v2.0.2) - Unreleased +All notable changes to this project will be documented in this file. -- ... - -## [2.0.1](../../releases/tag/v2.0.1) - 2024-10-25 +## [2.0.1](https://github.com/apify/apify-sdk-python/releases/tags/v2.0.1) (2024-10-25) ### 🚀 Features -- Add standby URL and change default standby port -- Add crawlee version to system info output +- Add standby URL, change default standby port ([#287](https://github.com/apify/apify-sdk-python/pull/287)) ([8cd2f2c](https://github.com/apify/apify-sdk-python/commit/8cd2f2cb9d1191dbc93bf1b8a2d70189881c64ad)) by [@jirimoravcik](https://github.com/jirimoravcik) +- Add crawlee version to system info print ([#304](https://github.com/apify/apify-sdk-python/pull/304)) ([c28f38f](https://github.com/apify/apify-sdk-python/commit/c28f38f4e205515e1b5d1ce97a2072be3a09d338)) by [@vdusek](https://github.com/vdusek) ### 🐛 Bug Fixes -- Allow empty timeout_at environment variable -- Use HttpHeaders type in Scrapy integration -- Adjust tests for Scrapy user data +- Adjust tests of scrapy user data ([#284](https://github.com/apify/apify-sdk-python/pull/284)) ([26ffb15](https://github.com/apify/apify-sdk-python/commit/26ffb15797effcfad1a25c840dd3d17663e26ea3)) by [@janbuchar](https://github.com/janbuchar) +- Use HttpHeaders type in Scrapy integration ([#289](https://github.com/apify/apify-sdk-python/pull/289)) ([3e33e91](https://github.com/apify/apify-sdk-python/commit/3e33e9147bfd60554b9da41b032c0451f91ba27b)) by [@vdusek](https://github.com/vdusek) +- Allow empty timeout_at env variable ([#303](https://github.com/apify/apify-sdk-python/pull/303)) ([b67ec98](https://github.com/apify/apify-sdk-python/commit/b67ec989dfcc21756cc976c52edc25735a3f0501)) by [@janbuchar](https://github.com/janbuchar), closes [#596](https://github.com/apify/apify-sdk-python/issues/596) + -## [2.0.0](../../releases/tag/v2.0.0) - 2024-09-10 +## [2.0.0](https://github.com/apify/apify-sdk-python/releases/tags/v2.0.0) (2024-09-10) - Check the [Upgrading to v2.0](https://docs.apify.com/sdk/python/docs/upgrading/upgrading-to-v2) guide. ### 🚀 Features -- Using Crawlee for Python under the hood ([#210](https://github.com/apify/apify-sdk-python/pull/210)) by [@janbuchar](https://github.com/janbuchar) -- Better Actor API typing ([#256](https://github.com/apify/crawlee-python/pull/256)) by [@janbuchar](https://github.com/janbuchar) -- Expose Request from Crawlee ([#266](https://github.com/apify/crawlee-python/pull/266)) by [@vdusek](https://github.com/vdusek) -- Improve examples in README ([#269](https://github.com/apify/crawlee-python/pull/269)) by [@vdusek](https://github.com/vdusek) -- Automatically configure logging ([#271](https://github.com/apify/crawlee-python/pull/271)) by [@janbuchar](https://github.com/janbuchar) +- Better Actor API typing ([#256](https://github.com/apify/apify-sdk-python/pull/256)) ([abb87e7](https://github.com/apify/apify-sdk-python/commit/abb87e7f3c272f88a9a76292d8394fe93b98428a)) by [@janbuchar](https://github.com/janbuchar), closes [#243](https://github.com/apify/apify-sdk-python/issues/243) +- Expose Request from Crawlee ([#266](https://github.com/apify/apify-sdk-python/pull/266)) ([1f01278](https://github.com/apify/apify-sdk-python/commit/1f01278c77f261500bc74efd700c0583ac45fd82)) by [@vdusek](https://github.com/vdusek) +- Automatically configure logging ([#271](https://github.com/apify/apify-sdk-python/pull/271)) ([1906bb2](https://github.com/apify/apify-sdk-python/commit/1906bb216b8a3f1c2ad740c551ee019c2ba0696f)) by [@janbuchar](https://github.com/janbuchar) ### 🐛 Bug Fixes -- Make `apify.log` public ([#249](https://github.com/apify/crawlee-python/pull/249)) by [@janbuchar](https://github.com/janbuchar) -- Dataset list response handling ([#257](https://github.com/apify/crawlee-python/pull/257)) by [@janbuchar](https://github.com/janbuchar) -- Ignore deprecated platform events ([#258](https://github.com/apify/crawlee-python/pull/258)) by [@janbuchar](https://github.com/janbuchar) -- Possible infinity loop in Apify-Scrapy proxy middleware ([#259](https://github.com/apify/crawlee-python/pull/259)) by [@vdusek](https://github.com/vdusek) -- Hotfix for `batch_add_requests` batch size limit ([#261](https://github.com/apify/crawlee-python/pull/261)) by [@janbuchar](https://github.com/janbuchar) +- Make apify.log public again ([#249](https://github.com/apify/apify-sdk-python/pull/249)) ([22677f5](https://github.com/apify/apify-sdk-python/commit/22677f57b2aff6c9bddbee305e5a62e39bbf5915)) by [@janbuchar](https://github.com/janbuchar) +- Dataset list response handling ([#257](https://github.com/apify/apify-sdk-python/pull/257)) ([0ea57d7](https://github.com/apify/apify-sdk-python/commit/0ea57d7c4788bff31f215c447c1881e56d6508bb)) by [@janbuchar](https://github.com/janbuchar) +- Ignore deprecated platform events ([#258](https://github.com/apify/apify-sdk-python/pull/258)) ([ed5ab3b](https://github.com/apify/apify-sdk-python/commit/ed5ab3b80c851a817aa87806c39cd8ef3e86fde5)) by [@janbuchar](https://github.com/janbuchar) +- Possible infinity loop in Apify-Scrapy proxy middleware ([#259](https://github.com/apify/apify-sdk-python/pull/259)) ([8647a94](https://github.com/apify/apify-sdk-python/commit/8647a94289423528f2940d9f7174f81682fbb407)) by [@vdusek](https://github.com/vdusek) +- Hotfix for batch_add_requests batch size limit ([#261](https://github.com/apify/apify-sdk-python/pull/261)) ([61d7a39](https://github.com/apify/apify-sdk-python/commit/61d7a392d182a752c91193170dca351f4cb0fbf3)) by [@janbuchar](https://github.com/janbuchar) -### Added +### Refactor -- Upgrade Apify Client to 1.7.1 +- Preparation for v2 release ([#210](https://github.com/apify/apify-sdk-python/pull/210)) ([2f9dcc5](https://github.com/apify/apify-sdk-python/commit/2f9dcc559414f31e3f4fc87e72417a36494b9c84)) by [@janbuchar](https://github.com/janbuchar), closes [#135](https://github.com/apify/apify-sdk-python/issues/135), [#137](https://github.com/apify/apify-sdk-python/issues/137), [#138](https://github.com/apify/apify-sdk-python/issues/138), [#147](https://github.com/apify/apify-sdk-python/issues/147), [#149](https://github.com/apify/apify-sdk-python/issues/149), [#237](https://github.com/apify/apify-sdk-python/issues/237) -## [1.7.2](../../releases/tag/v1.7.2) - 2024-07-08 -### Added +## [1.7.2](https://github.com/apify/apify-sdk-python/releases/tags/v1.7.2) (2024-07-08) -- Add Actor Standby port +### 🚀 Features -## [1.7.1](../../releases/tag/v1.7.1) - 2024-05-23 +- Add actor standby port ([#220](https://github.com/apify/apify-sdk-python/pull/220)) ([6d0d87d](https://github.com/apify/apify-sdk-python/commit/6d0d87dcaedaf42d8eeb7d23c56f6b102434cbcb)) by [@jirimoravcik](https://github.com/jirimoravcik) + + +## [1.7.1](https://github.com/apify/apify-sdk-python/releases/tags/v1.7.1) (2024-05-23) + +### 🐛 Bug Fixes + +- Set a timeout for Actor cleanup ([#206](https://github.com/apify/apify-sdk-python/pull/206)) ([cfed57d](https://github.com/apify/apify-sdk-python/commit/cfed57d6cff4fd15fe4b25578573190d53b9942c)) by [@janbuchar](https://github.com/janbuchar), closes [#200](https://github.com/apify/apify-sdk-python/issues/200) -### Fixed -- Set a timeout for Actor cleanup ## [1.7.0](../../releases/tag/v1.7.0) - 2024-03-12 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb92f3e8..1828e13d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,52 +1,76 @@ # Development +Here you'll find a contributing guide to get started with development. + ## Environment -For local development, it is required to have Python 3.8 (or a later version) installed. +For local development, it is required to have Python 3.9 (or a later version) installed. + +We use [Poetry](https://python-poetry.org/) for project management. Install it and set up your IDE accordingly. -It is recommended to set up a virtual environment while developing this package to isolate your development environment, -however, due to the many varied ways Python can be installed and virtual environments can be set up, -this is left up to the developers to do themselves. +## Dependencies -One recommended way is with the built-in `venv` module: +To install this package and its development dependencies, run: -```bash -python3 -m venv .venv -source .venv/bin/activate +```sh +make install-dev ``` -To improve on the experience, you can use [pyenv](https://github.com/pyenv/pyenv) to have an environment with a pinned Python version, -and [direnv](https://github.com/direnv/direnv) to automatically activate/deactivate the environment when you enter/exit the project folder. +## Code checking -## Dependencies +To execute all code checking tools together, run: -To install this package and its development dependencies, run `make install-dev`. +```sh +make check-code +``` -## Code checking +### Linting -To run all our code checking tools together, just run `make check-code`. +We utilize [ruff](https://docs.astral.sh/ruff/) for linting, which analyzes code for potential issues and enforces consistent style. Refer to `pyproject.toml` for configuration details. -### Linting +To run linting: -We use [ruff](https://docs.astral.sh/ruff/) for linting to to analyze the code for potential issues and enforce -uniformed code style. See the `pyproject.toml` for its configuration. To run the linting, just run `make lint`. +```sh +make lint +``` ### Formatting -We use [ruff](https://docs.astral.sh/ruff/) for automated code formatting. It formats the code to follow uniformed -code style and addresses auto-fixable linting issues. See the `pyproject.toml` for its configuration. To run -the formatting, just run `make format`. +Our automated code formatting also leverages [ruff](https://docs.astral.sh/ruff/), ensuring uniform style and addressing fixable linting issues. Configuration specifics are outlined in `pyproject.toml`. + +To run formatting: + +```sh +make format +``` ### Type checking -We use [mypy](https://mypy.readthedocs.io/en/stable/) for type checking. See the `mypy.ini` for its configuration. -To run the type checking, just run `make type-check`. +Type checking is handled by [mypy](https://mypy.readthedocs.io/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`. + +To run type checking: + +```sh +make type-check +``` ### Unit tests -We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. See the `pyproject.toml` for -both its configuration and the list of installed plugins. To run unit tests execute `make unit-tests`. To run unit -tests with HTML coverage report execute `make unit-tests-cov`. +We employ pytest as our testing framework, equipped with various plugins. Check pyproject.toml for configuration details and installed plugins. + +We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. Check `pyproject.toml` for configuration details and installed plugins. + +To run unit tests: + +```sh +make unit-tests +``` + +To run unit tests with HTML coverage report: + +```sh +make unit-tests-cov +``` ## Integration tests @@ -59,82 +83,84 @@ the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to t ## Documentation -We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting the code. -We document every user-facing class or method, and enforce that using the flake8-docstrings library. +We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). -The documentation is then rendered from the docstrings in the code, using `pydoc-markdown` and some heavy post-processing, -and from Markdown documents in the `docs` folder in the `docs` branch, and then rendered using Docusaurus and published to GitHub pages. +Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. -## Release process +To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps: -Publishing new versions to [PyPI](https://pypi.org/project/apify) happens automatically through GitHub Actions. +Navigate to the `website/` directory: -On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml` -and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI. +```sh +cd website/ +``` -A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`. -The built package assets are automatically uploaded to the GitHub release. +Enable Corepack, which installs Yarn automatically: -If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails, -so don't forget to update the version number before releasing a new version. -The release process also fails when the released version is not described in `CHANGELOG.md`, -so don't forget to describe the changes in the new version there. +```sh +corepack enable +``` -### Beta release checklist +Build the API reference: -Beta release happens automatically after you merge a pull request or add a direct commit to the master branch. Before you do that check the following: +```sh +./build_api_reference.sh +``` -- Make sure that in the [pyproject.toml](https://github.com/apify/apify-sdk-python/blob/master/pyproject.toml) a project version is set to the latest non-published version. -- Describe your changes to the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) in the section with the latest non-published version. +Install the necessary dependencies: -### Production release checklist +```sh +yarn +``` -Production release happens after the GitHub release is created. Before you do that check the following: +Start the project in development mode with Hot Module Replacement (HMR): -- Make sure [here](https://pypi.org/project/apify/#history) that the beta release with the latest commit is successfully deployed. -- Make sure that all the changes that happened from the last production release are described in the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) (it's okay to skip DX related changes, repo setup etc). -- When drafting a new GitHub release: - - Create a new tag in the format of `v1.2.3` targeting the master branch. - - Fill in the release title in the format of `1.2.3`. - - Copy the changes from the [CHANGELOG.md](https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md) and paste them into the release description. Make sure that all changes are properly categorized using headlines (`Added`, `Fixed` or `Internal changes`). - - Check the "Set as the latest release" option. +```sh +yarn start +``` -Currently, there is no explicit approval process, so when you are done with the checklist, proceed with the release. +Or using `make`: -Once released, manually bump the version in `pyproject.toml` ([PR example](https://github.com/apify/apify-sdk-python/pull/187)). +```sh +make run-doc +``` + +## Release process -## Maintanance +Publishing new versions to [PyPI](https://pypi.org/project/apify) is automated through GitHub Actions. -### Removing Support for an outdated Python version +- **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI. +- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overriden using the `custom` release type. -- Todo: Fill in once Python 3.8 is deprecated. +### Publishing to PyPI manually -### Adding support for a new Python version +1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead. +2. **Make sure you know what you're doing.** -1) Firstly, ensure that the package ( - [apify-sdk-python](https://github.com/apify/apify-sdk-python), - [apify-client-python](https://github.com/apify/apify-client-python), - [apify-shared-python](https://github.com/apify/apify-shared-python) -) is compatible with the new Python version. Both in our code base and -the dependencies we use. Then, release a new version of the package. - - For inspiration, see the PR - [apify/apify-sdk-python#121](https://github.com/apify/apify-sdk-python/pull/121), - where support for Python 3.12 was added to the Apify Python SDK. +3. Update the version number: -2) Next, build and publish the new versions of Python base Docker images. - - For inspiration, see the PR - [apify/apify-actor-docker#112](https://github.com/apify/apify-actor-docker/pull/112), - where support for Python 3.12 was added. - - Apify base Docker images are built using GitHub Actions, accessible at - [apify/apify-actor-docker/actions](https://github.com/apify/apify-actor-docker/actions). +- Modify the `version` field under `tool.poetry` in `pyproject.toml`. -3) Integrate the new Python version into the CI/CD workflows -of existing Python projects ( - [apify-sdk-python](https://github.com/apify/apify-sdk-python), - [apify-client-python](https://github.com/apify/apify-client-python), - [apify-shared-python](https://github.com/apify/apify-shared-python), - [actor-templates](https://github.com/apify/actor-templates) -). - - For inspiration, see the PR - [apify/apify-sdk-python#124](https://github.com/apify/apify-sdk-python/pull/124), - where support for Python 3.12 was added to the CI/CD of the Apify Python SDK. +```toml +[tool.poetry] +name = "apify" +version = "x.z.y" +``` + +4. Generate the distribution archives for the package: + +```shell +poetry build +``` + +5. Set up the PyPI API token for authentication: + +```shell +poetry config pypi-token.pypi YOUR_API_TOKEN +``` + +6. Upload the package to PyPI: + +```shell +poetry publish +``` diff --git a/Makefile b/Makefile index 33dd0063..96cb6298 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ .PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ - integration-tests format check-code check-version-availability check-changelog-entry \ - check-version-conflict build-api-reference run-docs + integration-tests format check-code build-api-reference build-docs run-docs -DIRS_WITH_CODE = src tests scripts +DIRS_WITH_CODE = src tests # This is default for local testing, but GitHub workflows override it to a higher value in CI INTEGRATION_TESTS_CONCURRENCY = 1 @@ -42,15 +41,9 @@ format: poetry run ruff check --fix $(DIRS_WITH_CODE) poetry run ruff format $(DIRS_WITH_CODE) -check-changelog-entry: - poetry run python scripts/check_changelog_entry.py - -check-version-conflict: - poetry run python scripts/check_version_conflict.py - # The check-code target runs a series of checks equivalent to those performed by pre-commit hooks # and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests check-changelog-entry check-version-conflict +check-code: lint type-check unit-tests build-api-reference: cd website && poetry run ./build_api_reference.sh diff --git a/scripts/check_changelog_entry.py b/scripts/check_changelog_entry.py deleted file mode 100755 index 1bc6cba4..00000000 --- a/scripts/check_changelog_entry.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import re - -from utils import REPO_ROOT, get_current_package_version - -CHANGELOG_PATH = REPO_ROOT / 'CHANGELOG.md' - -# Checks whether the current package version has an entry in the CHANGELOG.md file -if __name__ == '__main__': - current_package_version = get_current_package_version() - - if not CHANGELOG_PATH.is_file(): - raise RuntimeError('Unable to find CHANGELOG.md file') - - with open(CHANGELOG_PATH, encoding='utf-8') as changelog_file: - for line in changelog_file: - # Ensure that the heading in the changelog entry for the specified version includes a version number - # enclosed in square brackets. This version number is formatted as a link to the corresponding - # version tag on GitHub. - if re.match(rf'## \[{current_package_version}\].*$', line): - break - else: - raise RuntimeError( - f'There is no entry in the changelog for the current package version ({current_package_version})' - ) diff --git a/scripts/check_version_conflict.py b/scripts/check_version_conflict.py deleted file mode 100755 index 372d3215..00000000 --- a/scripts/check_version_conflict.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -from utils import get_current_package_version, get_published_package_versions - -# Checks whether the current package version number was not already used in a published release. -if __name__ == '__main__': - current_version = get_current_package_version() - - # Load the version numbers of the currently published versions from PyPI - published_versions = get_published_package_versions() - - # We don't want to try to publish a version with the same version number as an already released stable version - if current_version in published_versions: - raise RuntimeError(f'The current version {current_version} was already released!') diff --git a/scripts/print_current_package_version.py b/scripts/print_current_package_version.py deleted file mode 100755 index 9cff474b..00000000 --- a/scripts/print_current_package_version.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -from utils import get_current_package_version - -# Print the current package version from the pyproject.toml file to stdout -if __name__ == '__main__': - print(get_current_package_version(), end='') diff --git a/scripts/update_version_for_prerelease.py b/scripts/update_version_for_prerelease.py deleted file mode 100755 index f09dbd3d..00000000 --- a/scripts/update_version_for_prerelease.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import re -import sys - -from utils import get_current_package_version, get_published_package_versions, set_current_package_version - -# Checks whether the current package version number was not already used in a published release, -# and if not, modifies the package version number in pyproject.toml -# from a stable release version (X.Y.Z) to a prerelease version (X.Y.ZbN or X.Y.Z.aN or X.Y.Z.rcN) -if __name__ == '__main__': - if len(sys.argv) != 2: - raise RuntimeError('You must pass the prerelease type as an argument to this script!') - - prerelease_type = sys.argv[1] - if prerelease_type not in ['alpha', 'beta', 'rc']: - raise RuntimeError(f'The prerelease type must be one of "alpha", "beta" or "rc", got "{prerelease_type}"!') - - if prerelease_type == 'alpha': - prerelease_prefix = 'a' - elif prerelease_type == 'beta': - prerelease_prefix = 'b' - elif prerelease_type == 'rc': - prerelease_prefix = 'rc' - - current_version = get_current_package_version() - - # We can only transform a stable release version (X.Y.Z) to a prerelease version (X.Y.ZxxxN) - if not re.match(r'^\d+\.\d+\.\d+$', current_version): - raise RuntimeError( - f'The current version {current_version} does not match the proper semver format for stable releases (X.Y.Z)' - ) - - # Load the version numbers of the currently published versions from PyPI - published_versions = get_published_package_versions() - - # We don't want to publish a prerelease version with the same version number as an already released stable version - if current_version in published_versions: - raise RuntimeError(f'The current version {current_version} was already released!') - - # Find the highest prerelease version number that was already published - latest_prerelease = 0 - for version in published_versions: - if version.startswith(f'{current_version}{prerelease_prefix}'): - prerelease_version = int(version.split(prerelease_prefix)[1]) - latest_prerelease = max(prerelease_version, latest_prerelease) - - # Write the latest prerelease version number to pyproject.toml - new_prerelease_version_number = f'{current_version}{prerelease_prefix}{latest_prerelease + 1}' - set_current_package_version(new_prerelease_version_number) diff --git a/scripts/utils.py b/scripts/utils.py deleted file mode 100644 index ec575112..00000000 --- a/scripts/utils.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -import json -import pathlib -from urllib.error import HTTPError -from urllib.request import urlopen - -PACKAGE_NAME = 'apify' -REPO_ROOT = pathlib.Path(__file__).parent.resolve() / '..' -PYPROJECT_TOML_FILE_PATH = REPO_ROOT / 'pyproject.toml' - - -# Load the current version number from pyproject.toml -# It is on a line in the format `version = "1.2.3"` -def get_current_package_version() -> str: - with open(PYPROJECT_TOML_FILE_PATH, encoding='utf-8') as pyproject_toml_file: - for line in pyproject_toml_file: - if line.startswith('version = '): - delim = '"' if '"' in line else "'" - return line.split(delim)[1] - else: # noqa: PLW0120 - raise RuntimeError('Unable to find version string.') - - -# Write the given version number from pyproject.toml -# It replaces the version number on the line with the format `version = "1.2.3"` -def set_current_package_version(version: str) -> None: - with open(PYPROJECT_TOML_FILE_PATH, 'r+', encoding='utf-8') as pyproject_toml_file: - updated_pyproject_toml_file_lines = [] - version_string_found = False - for line in pyproject_toml_file: - if line.startswith('version = '): - version_string_found = True - line = f'version = "{version}"\n' # noqa: PLW2901 - updated_pyproject_toml_file_lines.append(line) - - if not version_string_found: - raise RuntimeError('Unable to find version string.') - - pyproject_toml_file.seek(0) - pyproject_toml_file.write(''.join(updated_pyproject_toml_file_lines)) - pyproject_toml_file.truncate() - - -# Load the version numbers of the currently published versions from PyPI -def get_published_package_versions() -> list: - package_info_url = f'https://pypi.org/pypi/{PACKAGE_NAME}/json' - try: - package_data = json.load(urlopen(package_info_url)) # noqa: S310 - published_versions = list(package_data['releases'].keys()) - # If the URL returns 404, it means the package has no releases yet (which is okay in our case) - except HTTPError as exc: - if exc.code != 404: - raise - published_versions = [] - return published_versions