diff --git a/.github/workflows/check_async_docstrings.yaml b/.github/workflows/_async_docstrings_check.yaml similarity index 55% rename from .github/workflows/check_async_docstrings.yaml rename to .github/workflows/_async_docstrings_check.yaml index 81a5af47..4cb8f7bc 100644 --- a/.github/workflows/check_async_docstrings.yaml +++ b/.github/workflows/_async_docstrings_check.yaml @@ -1,11 +1,14 @@ -name: Check async docstrings +name: Async docstrings check on: workflow_call: +env: + PYTHON_VERSION: 3.12 + jobs: - check_async_docstrings: - name: Check whether doctrings for async methods are up-to-date + async_docstring_check: + name: Async docstrings check runs-on: ubuntu-latest steps: @@ -15,12 +18,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - - name: Check async docstrings + - name: Run async docstrings check run: make check-async-docstrings diff --git a/.github/workflows/check_version_availability.yaml b/.github/workflows/_changelog_entry_check.yaml similarity index 53% rename from .github/workflows/check_version_availability.yaml rename to .github/workflows/_changelog_entry_check.yaml index 5d7d0ac5..78ebefc5 100644 --- a/.github/workflows/check_version_availability.yaml +++ b/.github/workflows/_changelog_entry_check.yaml @@ -1,11 +1,14 @@ -name: Check package version availability +name: Changelog entry check on: workflow_call: +env: + PYTHON_VERSION: 3.12 + jobs: - check_version_availability: - name: Check version availability + check_changelog_entry: + name: Changelog entry check runs-on: ubuntu-latest if: (!startsWith(github.event.pull_request.title, 'docs:')) @@ -16,12 +19,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - - name: Check version availability - run: make check-version-availability + - name: Run changelog entry check + run: make check-changelog-entry diff --git a/.github/workflows/_version_conflict_check.yaml b/.github/workflows/_version_conflict_check.yaml new file mode 100644 index 00000000..e02341b4 --- /dev/null +++ b/.github/workflows/_version_conflict_check.yaml @@ -0,0 +1,30 @@ +name: Version conflict check + +on: + workflow_call: + +env: + PYTHON_VERSION: 3.12 + +jobs: + check_version_conflict: + name: Version conflict check + runs-on: ubuntu-latest + if: (!startsWith(github.event.pull_request.title, 'docs:')) + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + - name: Run version conflict check + run: make check-version-conflict diff --git a/.github/workflows/docs.yaml b/.github/workflows/build_and_deploy_docs.yaml similarity index 86% rename from .github/workflows/docs.yaml rename to .github/workflows/build_and_deploy_docs.yaml index d64e0f35..671e64b3 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/build_and_deploy_docs.yaml @@ -6,6 +6,10 @@ on: - master workflow_dispatch: +env: + NODE_VERSION: 20 + PYTHON_VERSION: 3.12 + jobs: build_and_deploy_docs: environment: @@ -22,14 +26,14 @@ jobs: with: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - name: Set up Node.js + - name: Set up Node uses: actions/setup-node@v4 with: - node-version: 18 + node-version: ${{ env.NODE_VERSION }} cache: npm cache-dependency-path: website/package-lock.json - - name: Install Node.js dependencies + - name: Install Node dependencies run: | npm install npm update @apify/docs-theme @@ -48,11 +52,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies + - name: Install Python dependencies run: | - pipx install poetry + pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - name: Build generated API reference diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml deleted file mode 100644 index de71a105..00000000 --- a/.github/workflows/integration_tests.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: Integration tests - -on: - workflow_call: - secrets: - APIFY_TEST_USER_PYTHON_SDK_API_TOKEN: - description: API token of the Python SDK testing user on Apify - required: true - -concurrency: # This is to make sure that only one run of this workflow is running at the same time, to not overshoot the test user limits - group: integration_tests - -jobs: - integration_tests: - name: Run integration tests - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Run integration tests - run: make INTEGRATION_TESTS_CONCURRENCY=8 integration-tests - env: - APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }} diff --git a/.github/workflows/lint_and_type_checks.yaml b/.github/workflows/lint_and_type_checks.yaml deleted file mode 100644 index 5e7ad416..00000000 --- a/.github/workflows/lint_and_type_checks.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Lint and type checks - -on: - workflow_call: - -jobs: - lint_and_type_checks: - name: Lint and type checks - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Run lint - run: make lint - - - name: Run type checks - run: make type-check diff --git a/.github/workflows/run_checks.yaml b/.github/workflows/run_checks.yaml deleted file mode 100644 index acaf2b17..00000000 --- a/.github/workflows/run_checks.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Code quality checks - -on: - pull_request: - -jobs: - check_version_availability: - name: Check version availability - uses: ./.github/workflows/check_version_availability.yaml - - lint_and_type_checks: - name: Run lint and type checks - uses: ./.github/workflows/lint_and_type_checks.yaml - - unit_tests: - name: Run unit tests - needs: [lint_and_type_checks] - uses: ./.github/workflows/unit_tests.yaml - - check_async_docstrings: - name: Check async dostrings - uses: ./.github/workflows/check_async_docstrings.yaml - - integration_tests: - name: Run integration tests - needs: [lint_and_type_checks, unit_tests, check_async_docstrings] - uses: ./.github/workflows/integration_tests.yaml - secrets: inherit diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml new file mode 100644 index 00000000..684ef60f --- /dev/null +++ b/.github/workflows/run_code_checks.yaml @@ -0,0 +1,44 @@ +name: Run code checks + +on: + # Trigger code checks on opening a new pull request. + # Secrets are only made available to the integration tests job, with a manual approval + # step required for PRs from forks. This prevents their potential exposure. + pull_request: + +jobs: + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + async_docstrings: + name: Async dostrings check + uses: ./.github/workflows/_async_docstrings_check.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + changelog_entry_check: + name: Changelog entry check + uses: ./.github/workflows/_changelog_entry_check.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + version_conflict_check: + name: Version conflict check + uses: ./.github/workflows/_version_conflict_check.yaml + + docs_check: + name: Docs check + uses: apify/workflows/.github/workflows/python_docs_check.yaml@main + + integration_tests: + name: Integration tests + needs: [lint_check, type_check, unit_tests] + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit diff --git a/.github/workflows/release.yaml b/.github/workflows/run_release.yaml similarity index 68% rename from .github/workflows/release.yaml rename to .github/workflows/run_release.yaml index 995ffb3b..a984cac3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/run_release.yaml @@ -1,16 +1,18 @@ -name: Check & Release +name: Run release on: - # Push to master will publish a beta version + # Push to master will publish a beta version. push: branches: - master tags-ignore: - "**" - # A release via GitHub releases will publish a stable version + + # A release via GitHub releases will publish a stable version. release: types: [published] - # Workflow dispatch will publish whatever you choose + + # Workflow dispatch will publish whatever you choose. workflow_dispatch: inputs: release_type: @@ -23,6 +25,9 @@ on: - beta - final +env: + PYTHON_VERSION: 3.12 + jobs: should_release: name: Check whether to release @@ -30,39 +35,61 @@ jobs: runs-on: ubuntu-latest steps: - name: Dummy step - run: true + run: "true" - lint_and_type_checks: - name: Run lint and type_checks + lint_check: + name: Lint check needs: [should_release] - uses: ./.github/workflows/lint_and_type_checks.yaml + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + needs: [should_release] + uses: apify/workflows/.github/workflows/python_type_check.yaml@main unit_tests: - name: Run unit tests + name: Unit tests + needs: [should_release] + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + changelog_entry_check: + name: Changelog entry check needs: [should_release] - uses: ./.github/workflows/unit_tests.yaml + uses: ./.github/workflows/_changelog_entry_check.yaml - check_async_docstrings: - name: Check whether async dostrings are up to date + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + version_conflict_check: + name: Version conflict check needs: [should_release] - uses: ./.github/workflows/check_async_docstrings.yaml + uses: ./.github/workflows/_version_conflict_check.yaml integration_tests: - name: Run integration tests + name: Integration tests needs: [should_release] - uses: ./.github/workflows/integration_tests.yaml + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main secrets: inherit publish_to_pypi: name: Publish to PyPI - needs: [should_release, lint_and_type_checks, unit_tests, check_async_docstrings, integration_tests] + needs: + [ + should_release, + lint_check, + type_check, + unit_tests, + async_docstrings, + changelog_entry_check, + version_conflict_check, + integration_tests, + ] runs-on: ubuntu-latest permissions: contents: write id-token: write environment: name: pypi - url: https://pypi.org/p/apify-client + url: https://pypi.org/project/apify-client/ steps: - name: Checkout repository @@ -71,11 +98,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies + - name: Install Python dependencies run: | - pipx install poetry + pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - # Determine if this is a prerelease or latest release diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml deleted file mode 100644 index 00582607..00000000 --- a/.github/workflows/unit_tests.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Unit tests - -on: - workflow_call: - -jobs: - unit_tests: - name: Run unit tests - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Run unit tests - run: make unit-tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d342e21..5f2c3500 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,14 +13,8 @@ repos: language: system pass_filenames: false - - id: unit-tests - name: Run unit tests - entry: make unit-tests - language: system - pass_filenames: false - - id: check-async-docstrings - name: Check whether async docstrings are aligned with sync ones + name: Check async docstrings entry: make check-async-docstrings language: system pass_filenames: false @@ -30,3 +24,9 @@ repos: entry: make check-changelog-entry language: system pass_filenames: false + + - id: check-version-conflict + name: Check version conflict + entry: make check-version-conflict + language: system + pass_filenames: false diff --git a/Makefile b/Makefile index a1172f6a..0f8f6542 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ -.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests format check-code check-async-docstrings fix-async-docstrings check-version-availability check-changelog-entry build-api-reference run-doc +.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ + integration-tests format check-code check-async-docstrings fix-async-docstrings \ + check-version-availability check-changelog-entry check-version-conflict build-api-reference \ + run-docs DIRS_WITH_CODE = src tests scripts @@ -40,24 +43,27 @@ format: poetry run ruff check --fix $(DIRS_WITH_CODE) poetry run ruff format $(DIRS_WITH_CODE) -# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks -# and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests - check-async-docstrings: poetry run python scripts/check_async_docstrings.py -fix-async-docstrings: - poetry run python scripts/fix_async_docstrings.py +check-changelog-entry: + poetry run python scripts/check_changelog_entry.py -check-version-availability: - poetry run python scripts/check_version_availability.py +check-version-conflict: + poetry run python scripts/check_version_conflict.py -check-changelog-entry: - poetry run python scripts/check_version_in_changelog.py +# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks +# and the run_checks.yaml GitHub Actions workflow. +check-code: lint type-check unit-tests check-async-docstrings check-changelog-entry check-version-conflict + +fix-async-docstrings: + poetry run python scripts/fix_async_docstrings.py build-api-reference: cd website && poetry run ./build_api_reference.sh -run-doc: build-api-reference +build-docs: build-api-reference + cd website && npm clean-install && npm run build + +run-docs: build-api-reference cd website && npm clean-install && npm run start diff --git a/pyproject.toml b/pyproject.toml index d69c9a98..7f2eb97c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,7 @@ max-branches = 18 [tool.pytest.ini_options] addopts = "-ra" +asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" timeout = 1200 diff --git a/scripts/check_async_docstrings.py b/scripts/check_async_docstrings.py index cbeed8c1..fa8bc16b 100755 --- a/scripts/check_async_docstrings.py +++ b/scripts/check_async_docstrings.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +"""Check if async docstrings are the same as sync.""" + import re import sys from pathlib import Path diff --git a/scripts/check_version_in_changelog.py b/scripts/check_changelog_entry.py similarity index 100% rename from scripts/check_version_in_changelog.py rename to scripts/check_changelog_entry.py diff --git a/scripts/check_version_availability.py b/scripts/check_version_conflict.py similarity index 100% rename from scripts/check_version_availability.py rename to scripts/check_version_conflict.py