From a5dbad8023f4a77358d03e2653551fa684d4bd8e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 10:52:13 +0200 Subject: [PATCH 1/8] ci: use the same setup as in Crawlee --- ...ility.yaml => _check_changelog_entry.yaml} | 19 ++-- .github/workflows/_check_docs_build.yaml | 50 +++++++++++ .../workflows/_check_version_conflict.yaml | 29 +++++++ ...ion_tests.yaml => _integration_tests.yaml} | 12 +-- .github/workflows/_linting.yaml | 32 +++++++ .github/workflows/_publish_to_pypi.yaml | 87 +++++++++++++++++++ ...d_type_checks.yaml => _type_checking.yaml} | 16 ++-- .../{unit_tests.yaml => _unit_tests.yaml} | 9 +- .github/workflows/docs.yaml | 13 +-- .github/workflows/run_checks.yaml | 49 ----------- .github/workflows/run_code_checks.yaml | 41 +++++++++ .../{release.yaml => run_release.yaml} | 27 +++--- .pre-commit-config.yaml | 6 +- Makefile | 18 ++-- ..._changelog.py => check_changelog_entry.py} | 0 ...ilability.py => check_version_conflict.py} | 0 16 files changed, 301 insertions(+), 107 deletions(-) rename .github/workflows/{check_version_availability.yaml => _check_changelog_entry.yaml} (53%) create mode 100644 .github/workflows/_check_docs_build.yaml create mode 100644 .github/workflows/_check_version_conflict.yaml rename .github/workflows/{integration_tests.yaml => _integration_tests.yaml} (66%) create mode 100644 .github/workflows/_linting.yaml create mode 100644 .github/workflows/_publish_to_pypi.yaml rename .github/workflows/{lint_and_type_checks.yaml => _type_checking.yaml} (66%) rename .github/workflows/{unit_tests.yaml => _unit_tests.yaml} (73%) delete mode 100644 .github/workflows/run_checks.yaml create mode 100644 .github/workflows/run_code_checks.yaml rename .github/workflows/{release.yaml => run_release.yaml} (87%) rename scripts/{check_version_in_changelog.py => check_changelog_entry.py} (100%) rename scripts/{check_version_availability.py => check_version_conflict.py} (100%) diff --git a/.github/workflows/check_version_availability.yaml b/.github/workflows/_check_changelog_entry.yaml similarity index 53% rename from .github/workflows/check_version_availability.yaml rename to .github/workflows/_check_changelog_entry.yaml index 93d82839..1d872f3d 100644 --- a/.github/workflows/check_version_availability.yaml +++ b/.github/workflows/_check_changelog_entry.yaml @@ -1,17 +1,18 @@ -name: Check package version availability +name: Check changelog entry on: workflow_call: jobs: - check_version_availability: - name: Check version availability + check_changelog_entry: + name: Check changelog entry runs-on: ubuntu-latest if: (!startsWith(github.event.pull_request.title, 'docs:')) + env: + PYTHON_VERSION: 3.12 steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). + # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 with: @@ -20,12 +21,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - - name: Check version availability - run: make check-version-availability + - name: Check changelog entry + run: make check-changelog-entry diff --git a/.github/workflows/_check_docs_build.yaml b/.github/workflows/_check_docs_build.yaml new file mode 100644 index 00000000..d3ca910a --- /dev/null +++ b/.github/workflows/_check_docs_build.yaml @@ -0,0 +1,50 @@ +name: Docs build + +on: + workflow_call: + +jobs: + docs: + name: Docs build + runs-on: ubuntu-latest + env: + NODE_VERSION: 20 + PYTHON_VERSION: 3.12 + + steps: + - name: Checkout Source code + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + - name: Build generated API reference + run: make build-api-reference + + - name: Install dependencies and try to build the website + run: | + # go to website dir + cd website + # install website deps + yarn + # build the docs + yarn build + env: + APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} diff --git a/.github/workflows/_check_version_conflict.yaml b/.github/workflows/_check_version_conflict.yaml new file mode 100644 index 00000000..ee16810a --- /dev/null +++ b/.github/workflows/_check_version_conflict.yaml @@ -0,0 +1,29 @@ +name: Check version conflict + +on: + workflow_call: + +jobs: + check_version_conflict: + name: Check version conflict + runs-on: ubuntu-latest + if: (!startsWith(github.event.pull_request.title, 'docs:')) + env: + PYTHON_VERSION: 3.12 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + - name: Check version conflict + run: make check-version-conflict diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/_integration_tests.yaml similarity index 66% rename from .github/workflows/integration_tests.yaml rename to .github/workflows/_integration_tests.yaml index d96a2dda..f4e620fa 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/_integration_tests.yaml @@ -12,22 +12,16 @@ concurrency: # This is to make sure that only one run of this workflow is runnin jobs: integration_tests: - name: Run integration tests + name: Integration tests runs-on: ubuntu-latest strategy: matrix: - # Run integration tests only on the oldest and newest supported Python versions, - # as these tests are time-consuming and these versions are the most likely to encounter issues. - python-version: ["3.9", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -36,7 +30,7 @@ jobs: - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - name: Run integration tests diff --git a/.github/workflows/_linting.yaml b/.github/workflows/_linting.yaml new file mode 100644 index 00000000..7ea07af0 --- /dev/null +++ b/.github/workflows/_linting.yaml @@ -0,0 +1,32 @@ +name: Linting + +on: + workflow_call: + +jobs: + linting: + name: Linting + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pipx install --python ${{ matrix.python-version }} poetry + make install-dev + + - name: Run linting + run: make lint diff --git a/.github/workflows/_publish_to_pypi.yaml b/.github/workflows/_publish_to_pypi.yaml new file mode 100644 index 00000000..c48c7056 --- /dev/null +++ b/.github/workflows/_publish_to_pypi.yaml @@ -0,0 +1,87 @@ +name: Publish to PyPI + +on: + workflow_call: + inputs: + version_number: + required: true + type: string + +jobs: + publish_to_pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/crawlee + env: + PYTHON_VERSION: 3.12 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + # Determines the release type based on the event that triggered the workflow. + - name: Determine release type + id: determine-release-type + run: | + if [[ ${{ github.event_name }} = release ]]; then + release_type="final" + elif [[ ${{ github.event_name }} = push ]]; then + release_type="beta" + elif [[ ${{ github.event_name }} = workflow_dispatch ]]; then + release_type=${{ github.event.inputs.release_type }} + fi + echo "release_type=${release_type}" >> $GITHUB_OUTPUT + + # Updates the version number for pre-releases in the project's configuration. + - name: Set pre-release version + if: steps.determine-release-type.outputs.release_type != 'final' + run: python ./scripts/update_version_for_prerelease.py ${{ steps.determine-release-type.outputs.release_type }} + + # Builds the package. + - name: Build package + run: make build + + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # If this workflow is not triggered by a GitHub release event, manually create and push a Git tag. + - name: Create Git tag with the published version + if: github.event_name != 'release' + run: | + GIT_TAG=v$(python ./scripts/print_current_package_version.py) + echo "Current package version retrieved: ${GIT_TAG}" + + echo "Creating Git tag: ${GIT_TAG}" + git tag "$GIT_TAG" + echo "Git tag ${GIT_TAG} created successfully." + + echo "Pushing Git tag ${GIT_TAG} to the remote repository." + git push origin tag "$GIT_TAG" + echo "Git tag ${GIT_TAG} pushed successfully." + + # If triggered by a release, upload build artifacts to the associated GitHub release. + - name: Upload the build artifacts to release + if: github.event_name == 'release' + run: | + echo "Uploading build artifacts to GitHub release: ${{ github.ref_name }}" + gh release upload ${{ github.ref_name }} dist/* + echo "Build artifacts uploaded successfully." + + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/lint_and_type_checks.yaml b/.github/workflows/_type_checking.yaml similarity index 66% rename from .github/workflows/lint_and_type_checks.yaml rename to .github/workflows/_type_checking.yaml index a89eca37..b68d8c5c 100644 --- a/.github/workflows/lint_and_type_checks.yaml +++ b/.github/workflows/_type_checking.yaml @@ -1,19 +1,18 @@ -name: Lint and type checks +name: Type checking on: workflow_call: jobs: - lint_and_type_checks: - name: Lint and type checks + type_checking: + name: Type checking runs-on: ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). + # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 with: @@ -26,11 +25,8 @@ jobs: - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - - name: Run lint - run: make lint - - - name: Run type checks + - name: Run type checking run: make type-check diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/_unit_tests.yaml similarity index 73% rename from .github/workflows/unit_tests.yaml rename to .github/workflows/_unit_tests.yaml index ccb5f6ed..b0db84ba 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/_unit_tests.yaml @@ -5,7 +5,7 @@ on: jobs: unit_tests: - name: Run unit tests + name: Unit tests strategy: matrix: os: [ubuntu-latest, windows-latest] @@ -13,8 +13,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). + # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 with: @@ -27,8 +26,10 @@ jobs: - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - name: Run unit tests run: make unit-tests + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 0a231e26..37447a06 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -15,6 +15,9 @@ jobs: pages: write id-token: write runs-on: ubuntu-latest + env: + NODE_VERSION: 18 + PYTHON_VERSION: 3.12 steps: - name: Checkout repository @@ -22,14 +25,14 @@ jobs: with: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - name: Set up Node.js + - name: Set up Node uses: actions/setup-node@v4 with: - node-version: 18 + node-version: ${{ env.NODE_VERSION }} cache: npm cache-dependency-path: website/package-lock.json - - name: Install Node.js dependencies + - name: Install Node dependencies run: | npm install npm update @apify/docs-theme @@ -48,11 +51,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - name: Build generated API reference diff --git a/.github/workflows/run_checks.yaml b/.github/workflows/run_checks.yaml deleted file mode 100644 index 154f30fa..00000000 --- a/.github/workflows/run_checks.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: Code quality checks - -on: - pull_request_target: - -jobs: - check_version_availability: - name: Check version availability - uses: ./.github/workflows/check_version_availability.yaml - - lint_and_type_checks: - name: Run lint and type checks - uses: ./.github/workflows/lint_and_type_checks.yaml - - unit_tests: - name: Run unit tests - needs: [lint_and_type_checks] - uses: ./.github/workflows/unit_tests.yaml - - # If the PR comes from the main repo, run integration tests directly - integration_tests: - if: github.event.pull_request.head.repo.owner.login == 'apify' - name: Run integration tests - needs: [lint_and_type_checks, unit_tests] - uses: ./.github/workflows/integration_tests.yaml - secrets: inherit - - # If the PR comes from a fork, - # we need to approve running the workflow first before allowing it to run, - # so that we can check for any unauthorized access to our secrets. - # We need two workflow jobs for that, - # because jobs calling reusable workflows can't use an environment. - # The first job is a dummy job that just asks for approval to use the `fork-worklows` environment. - integration_tests_fork_approve: - if: github.event.pull_request.head.repo.owner.login != 'apify' - name: Approve integration tests from fork - needs: [lint_and_type_checks, unit_tests] - environment: fork-pr-integration-tests - runs-on: ubuntu-latest - steps: - - name: Dummy step - run: true - - # The second job is the actual integration tests job. - integration_tests_fork: - name: Run integration tests from fork - needs: [integration_tests_fork_approve] - uses: ./.github/workflows/integration_tests.yaml - secrets: inherit diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml new file mode 100644 index 00000000..40214e9e --- /dev/null +++ b/.github/workflows/run_code_checks.yaml @@ -0,0 +1,41 @@ +name: Run code checks + +on: + # Trigger code checks on opening a new pull request. + pull_request_target: + + # Do not trigger code checks on push to the master branch, as they will be triggered + # by the release workflow. + + # Trigger code checks on workflow call (e.g. from run release workflow). + workflow_call: + +jobs: + run_linting: + name: Run linting + uses: ./.github/workflows/_linting.yaml + + run_type_checking: + name: Run type checking + uses: ./.github/workflows/_type_checking.yaml + + run_unit_tests: + name: Run unit tests + uses: ./.github/workflows/_unit_tests.yaml + + check_changelog_entry: + name: Check changelog entry + uses: ./.github/workflows/_check_changelog_entry.yaml + + check_docs_build: + name: Check docs build + uses: ./.github/workflows/_check_docs_build.yaml + + check_version_conflict: + name: Check version conflict + uses: ./.github/workflows/_check_version_conflict.yaml + + run_integration_tests: + name: Run integration tests + uses: ./.github/workflows/_integration_tests.yaml + secrets: inherit diff --git a/.github/workflows/release.yaml b/.github/workflows/run_release.yaml similarity index 87% rename from .github/workflows/release.yaml rename to .github/workflows/run_release.yaml index 9ccaa91e..05e62974 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/run_release.yaml @@ -1,4 +1,4 @@ -name: Check & Release +name: Run release on: # Push to master will publish a beta version @@ -28,36 +28,43 @@ jobs: name: Check whether to release if: (!startsWith(github.event.head_commit.message, 'docs:') || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.12 steps: - name: Dummy step run: true - lint_and_type_checks: - name: Run lint and type_checks + linting: + name: Run linting needs: [should_release] - uses: ./.github/workflows/lint_and_type_checks.yaml + uses: ./.github/workflows/_linting.yaml + + type_checking: + name: Run type checking + needs: [should_release] + uses: ./.github/workflows/_type_checking.yaml unit_tests: name: Run unit tests needs: [should_release] - uses: ./.github/workflows/unit_tests.yaml + uses: ./.github/workflows/_unit_tests.yaml integration_tests: name: Run integration tests needs: [should_release] - uses: ./.github/workflows/integration_tests.yaml + uses: ./.github/workflows/_integration_tests.yaml secrets: inherit publish_to_pypi: name: Publish to PyPI - needs: [should_release, lint_and_type_checks, unit_tests, integration_tests] + needs: [should_release, linting, type_checking, unit_tests, integration_tests] runs-on: ubuntu-latest permissions: contents: write id-token: write environment: name: pypi - url: https://pypi.org/p/apify + url: https://pypi.org/project/apify/ steps: - name: Checkout repository @@ -66,11 +73,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - # Determine if this is a prerelease or latest release diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c1cdf523..f12158e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,8 +19,8 @@ repos: language: system pass_filenames: false - - id: check-version-availability - name: Check version availability - entry: make check-version-availability + - id: check-version-conflict + name: Check version conflict + entry: make check-version-conflict language: system pass_filenames: false diff --git a/Makefile b/Makefile index 768a2c8f..9afade1d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ -.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests format check-code check-version-availability check-changelog-entry build-api-reference run-doc +.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ + integration-tests format check-code check-version-availability check-changelog-entry \ + check-version-conflict build-api-reference run-doc DIRS_WITH_CODE = src tests scripts @@ -40,15 +42,15 @@ format: poetry run ruff check --fix $(DIRS_WITH_CODE) poetry run ruff format $(DIRS_WITH_CODE) -# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks -# and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests +check-changelog-entry: + poetry run python scripts/check_changelog_entry.py -check-version-availability: - poetry run python scripts/check_version_availability.py +check-version-conflict: + poetry run python scripts/check_version_conflict.py -check-changelog-entry: - poetry run python scripts/check_version_in_changelog.py +# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks +# and the run_checks.yaml GitHub Actions workflow. +check-code: lint type-check unit-tests check-changelog-entry check-version-conflict build-api-reference: cd website && poetry run ./build_api_reference.sh diff --git a/scripts/check_version_in_changelog.py b/scripts/check_changelog_entry.py similarity index 100% rename from scripts/check_version_in_changelog.py rename to scripts/check_changelog_entry.py diff --git a/scripts/check_version_availability.py b/scripts/check_version_conflict.py similarity index 100% rename from scripts/check_version_availability.py rename to scripts/check_version_conflict.py From 6cf757e1adf34157b2835dc383aa9b4bddcd4ed4 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 10:55:56 +0200 Subject: [PATCH 2/8] node 20 --- .github/workflows/docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 37447a06..ad39c6e3 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -16,7 +16,7 @@ jobs: id-token: write runs-on: ubuntu-latest env: - NODE_VERSION: 18 + NODE_VERSION: 20 PYTHON_VERSION: 3.12 steps: From 708eda158cf7e3404eaad34c8308969682bf5c8d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 11:04:38 +0200 Subject: [PATCH 3/8] integration tests only on 3.9 and 3.12 --- .github/workflows/_integration_tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_integration_tests.yaml b/.github/workflows/_integration_tests.yaml index f4e620fa..fba022a6 100644 --- a/.github/workflows/_integration_tests.yaml +++ b/.github/workflows/_integration_tests.yaml @@ -16,7 +16,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + # Run integration tests only on the oldest and newest supported Python versions, + # as these tests are time-consuming and these versions are the most likely to encounter issues. + python-version: ["3.9", "3.12"] max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits steps: From 1a1941e8e1b1715717b147b1ba45a192dadaa65b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 11:16:24 +0200 Subject: [PATCH 4/8] rm git checkout with --- .github/workflows/_check_changelog_entry.yaml | 3 --- .github/workflows/_integration_tests.yaml | 6 ++++-- .github/workflows/_linting.yaml | 3 --- .github/workflows/_type_checking.yaml | 3 --- .github/workflows/_unit_tests.yaml | 3 --- 5 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/_check_changelog_entry.yaml b/.github/workflows/_check_changelog_entry.yaml index 1d872f3d..7e93fab3 100644 --- a/.github/workflows/_check_changelog_entry.yaml +++ b/.github/workflows/_check_changelog_entry.yaml @@ -12,11 +12,8 @@ jobs: PYTHON_VERSION: 3.12 steps: - # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/_integration_tests.yaml b/.github/workflows/_integration_tests.yaml index fba022a6..eefe034c 100644 --- a/.github/workflows/_integration_tests.yaml +++ b/.github/workflows/_integration_tests.yaml @@ -4,10 +4,12 @@ on: workflow_call: secrets: APIFY_TEST_USER_PYTHON_SDK_API_TOKEN: - description: API token of the Python SDK testing user on Apify + description: API token of the Python SDK & client testing user on Apify required: true -concurrency: # This is to make sure that only one run of this workflow is running at the same time, to not overshoot the test user limits +# This is to make sure that only one run of this workflow is running at the same time, +# to not overshoot the test user limits. +concurrency: group: integration_tests jobs: diff --git a/.github/workflows/_linting.yaml b/.github/workflows/_linting.yaml index 7ea07af0..942512a0 100644 --- a/.github/workflows/_linting.yaml +++ b/.github/workflows/_linting.yaml @@ -12,11 +12,8 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/_type_checking.yaml b/.github/workflows/_type_checking.yaml index b68d8c5c..62ca2380 100644 --- a/.github/workflows/_type_checking.yaml +++ b/.github/workflows/_type_checking.yaml @@ -12,11 +12,8 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/_unit_tests.yaml b/.github/workflows/_unit_tests.yaml index b0db84ba..1430eb14 100644 --- a/.github/workflows/_unit_tests.yaml +++ b/.github/workflows/_unit_tests.yaml @@ -13,11 +13,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - # We need to check out the head commit in case of PRs, and the default ref otherwise (during release). - name: Checkout repository uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 From f6106f93ead84cb3e947000a5c4f803007b02b60 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 13:41:33 +0200 Subject: [PATCH 5/8] use shared workflows --- ...entry.yaml => _changelog_entry_check.yaml} | 6 +-- .github/workflows/_check_docs_build.yaml | 50 ------------------- .github/workflows/_integration_tests.yaml | 43 ---------------- .github/workflows/_linting.yaml | 29 ----------- .github/workflows/_type_checking.yaml | 29 ----------- .github/workflows/_unit_tests.yaml | 32 ------------ ...lict.yaml => _version_conflict_check.yaml} | 6 +-- .../{docs.yaml => build_and_deploy_docs.yaml} | 0 .github/workflows/run_code_checks.yaml | 46 +++++++++-------- .github/workflows/run_release.yaml | 33 ++++++++---- Makefile | 7 ++- 11 files changed, 59 insertions(+), 222 deletions(-) rename .github/workflows/{_check_changelog_entry.yaml => _changelog_entry_check.yaml} (84%) delete mode 100644 .github/workflows/_check_docs_build.yaml delete mode 100644 .github/workflows/_integration_tests.yaml delete mode 100644 .github/workflows/_linting.yaml delete mode 100644 .github/workflows/_type_checking.yaml delete mode 100644 .github/workflows/_unit_tests.yaml rename .github/workflows/{_check_version_conflict.yaml => _version_conflict_check.yaml} (84%) rename .github/workflows/{docs.yaml => build_and_deploy_docs.yaml} (100%) diff --git a/.github/workflows/_check_changelog_entry.yaml b/.github/workflows/_changelog_entry_check.yaml similarity index 84% rename from .github/workflows/_check_changelog_entry.yaml rename to .github/workflows/_changelog_entry_check.yaml index 7e93fab3..8ac90673 100644 --- a/.github/workflows/_check_changelog_entry.yaml +++ b/.github/workflows/_changelog_entry_check.yaml @@ -1,11 +1,11 @@ -name: Check changelog entry +name: Changelog entry check on: workflow_call: jobs: check_changelog_entry: - name: Check changelog entry + name: Changelog entry check runs-on: ubuntu-latest if: (!startsWith(github.event.pull_request.title, 'docs:')) env: @@ -25,5 +25,5 @@ jobs: pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - - name: Check changelog entry + - name: Execute changelog entry check run: make check-changelog-entry diff --git a/.github/workflows/_check_docs_build.yaml b/.github/workflows/_check_docs_build.yaml deleted file mode 100644 index d3ca910a..00000000 --- a/.github/workflows/_check_docs_build.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docs build - -on: - workflow_call: - -jobs: - docs: - name: Docs build - runs-on: ubuntu-latest - env: - NODE_VERSION: 20 - PYTHON_VERSION: 3.12 - - steps: - - name: Checkout Source code - uses: actions/checkout@v4 - - - name: Set up Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - - name: Build generated API reference - run: make build-api-reference - - - name: Install dependencies and try to build the website - run: | - # go to website dir - cd website - # install website deps - yarn - # build the docs - yarn build - env: - APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} diff --git a/.github/workflows/_integration_tests.yaml b/.github/workflows/_integration_tests.yaml deleted file mode 100644 index eefe034c..00000000 --- a/.github/workflows/_integration_tests.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: Integration tests - -on: - workflow_call: - secrets: - APIFY_TEST_USER_PYTHON_SDK_API_TOKEN: - description: API token of the Python SDK & client testing user on Apify - required: true - -# This is to make sure that only one run of this workflow is running at the same time, -# to not overshoot the test user limits. -concurrency: - group: integration_tests - -jobs: - integration_tests: - name: Integration tests - runs-on: ubuntu-latest - strategy: - matrix: - # Run integration tests only on the oldest and newest supported Python versions, - # as these tests are time-consuming and these versions are the most likely to encounter issues. - python-version: ["3.9", "3.12"] - max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install --python ${{ matrix.python-version }} poetry - make install-dev - - - name: Run integration tests - run: make INTEGRATION_TESTS_CONCURRENCY=8 integration-tests - env: - APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }} diff --git a/.github/workflows/_linting.yaml b/.github/workflows/_linting.yaml deleted file mode 100644 index 942512a0..00000000 --- a/.github/workflows/_linting.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Linting - -on: - workflow_call: - -jobs: - linting: - name: Linting - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install --python ${{ matrix.python-version }} poetry - make install-dev - - - name: Run linting - run: make lint diff --git a/.github/workflows/_type_checking.yaml b/.github/workflows/_type_checking.yaml deleted file mode 100644 index 62ca2380..00000000 --- a/.github/workflows/_type_checking.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Type checking - -on: - workflow_call: - -jobs: - type_checking: - name: Type checking - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install --python ${{ matrix.python-version }} poetry - make install-dev - - - name: Run type checking - run: make type-check diff --git a/.github/workflows/_unit_tests.yaml b/.github/workflows/_unit_tests.yaml deleted file mode 100644 index 1430eb14..00000000 --- a/.github/workflows/_unit_tests.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Unit tests - -on: - workflow_call: - -jobs: - unit_tests: - name: Unit tests - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install --python ${{ matrix.python-version }} poetry - make install-dev - - - name: Run unit tests - run: make unit-tests - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/_check_version_conflict.yaml b/.github/workflows/_version_conflict_check.yaml similarity index 84% rename from .github/workflows/_check_version_conflict.yaml rename to .github/workflows/_version_conflict_check.yaml index ee16810a..df4580a5 100644 --- a/.github/workflows/_check_version_conflict.yaml +++ b/.github/workflows/_version_conflict_check.yaml @@ -1,11 +1,11 @@ -name: Check version conflict +name: Version conflict check on: workflow_call: jobs: check_version_conflict: - name: Check version conflict + name: Version conflict check runs-on: ubuntu-latest if: (!startsWith(github.event.pull_request.title, 'docs:')) env: @@ -25,5 +25,5 @@ jobs: pipx install --python ${{ env.PYTHON_VERSION }} poetry make install-dev - - name: Check version conflict + - name: Execute version conflict check run: make check-version-conflict diff --git a/.github/workflows/docs.yaml b/.github/workflows/build_and_deploy_docs.yaml similarity index 100% rename from .github/workflows/docs.yaml rename to .github/workflows/build_and_deploy_docs.yaml diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index 40214e9e..09d3d82f 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -11,31 +11,35 @@ on: workflow_call: jobs: - run_linting: - name: Run linting - uses: ./.github/workflows/_linting.yaml + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python/lint_check.yaml - run_type_checking: - name: Run type checking - uses: ./.github/workflows/_type_checking.yaml + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python/type_check.yaml - run_unit_tests: - name: Run unit tests - uses: ./.github/workflows/_unit_tests.yaml + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python/unit_tests.yaml + secrets: inherit - check_changelog_entry: - name: Check changelog entry - uses: ./.github/workflows/_check_changelog_entry.yaml + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + changelog_entry_check: + name: Changelog entry check + uses: ./.github/workflows/_changelog_entry_check.yaml - check_docs_build: - name: Check docs build - uses: ./.github/workflows/_check_docs_build.yaml + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + version_conflict_check: + name: Version conflict check + uses: ./.github/workflows/_version_conflict_check.yaml - check_version_conflict: - name: Check version conflict - uses: ./.github/workflows/_check_version_conflict.yaml + docs_check: + name: Docs check + uses: apify/workflows/.github/workflows/python/docs_check.yaml + secrets: inherit - run_integration_tests: - name: Run integration tests - uses: ./.github/workflows/_integration_tests.yaml + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python/integration_tests.yaml secrets: inherit diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml index 05e62974..a14c21a9 100644 --- a/.github/workflows/run_release.yaml +++ b/.github/workflows/run_release.yaml @@ -34,25 +34,38 @@ jobs: - name: Dummy step run: true - linting: - name: Run linting + lint_check: + name: Lint check needs: [should_release] - uses: ./.github/workflows/_linting.yaml + uses: apify/workflows/.github/workflows/python/lint_check.yaml - type_checking: - name: Run type checking + type_check: + name: Type check needs: [should_release] - uses: ./.github/workflows/_type_checking.yaml + uses: apify/workflows/.github/workflows/python/type_check.yaml unit_tests: - name: Run unit tests + name: Unit tests needs: [should_release] - uses: ./.github/workflows/_unit_tests.yaml + uses: apify/workflows/.github/workflows/python/unit_tests.yaml + secrets: inherit + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + changelog_entry_check: + name: Changelog entry check + needs: [should_release] + uses: ./.github/workflows/_changelog_entry_check.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + version_conflict_check: + name: Version conflict check + needs: [should_release] + uses: ./.github/workflows/_version_conflict_check.yaml integration_tests: - name: Run integration tests + name: Integration tests needs: [should_release] - uses: ./.github/workflows/_integration_tests.yaml + uses: apify/workflows/.github/workflows/python/integration_tests.yaml secrets: inherit publish_to_pypi: diff --git a/Makefile b/Makefile index 9afade1d..b51c1572 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ integration-tests format check-code check-version-availability check-changelog-entry \ - check-version-conflict build-api-reference run-doc + check-version-conflict build-api-reference run-docs DIRS_WITH_CODE = src tests scripts @@ -55,5 +55,8 @@ check-code: lint type-check unit-tests check-changelog-entry check-version-confl build-api-reference: cd website && poetry run ./build_api_reference.sh -run-doc: build-api-reference +build-docs: build-api-reference + cd website && yarn && yarn build + +run-docs: build-api-reference cd website && npm clean-install && npm run start From 5e695ad18ddb951a406f0efb537fec7d4ae92058 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 13:46:14 +0200 Subject: [PATCH 6/8] run release needs fix --- .github/workflows/run_release.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml index a14c21a9..167bd90a 100644 --- a/.github/workflows/run_release.yaml +++ b/.github/workflows/run_release.yaml @@ -70,7 +70,16 @@ jobs: publish_to_pypi: name: Publish to PyPI - needs: [should_release, linting, type_checking, unit_tests, integration_tests] + needs: + [ + should_release, + lint_check, + type_check, + unit_tests, + changelog_entry_check, + version_conflict_check, + integration_tests, + ] runs-on: ubuntu-latest permissions: contents: write From 5b7be54970a9b5bb3b922c4c7641943dd6ecceb7 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 13:55:17 +0200 Subject: [PATCH 7/8] unit tests do not use gh token --- .github/workflows/run_code_checks.yaml | 1 - .github/workflows/run_release.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index 09d3d82f..ef98ad61 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -22,7 +22,6 @@ jobs: unit_tests: name: Unit tests uses: apify/workflows/.github/workflows/python/unit_tests.yaml - secrets: inherit # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved changelog_entry_check: diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml index 167bd90a..b2982b75 100644 --- a/.github/workflows/run_release.yaml +++ b/.github/workflows/run_release.yaml @@ -48,7 +48,6 @@ jobs: name: Unit tests needs: [should_release] uses: apify/workflows/.github/workflows/python/unit_tests.yaml - secrets: inherit # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved changelog_entry_check: From 66c0943a79e5229313e4920d213df0858da95269 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Oct 2024 15:22:21 +0200 Subject: [PATCH 8/8] docs check do not use secret --- .github/workflows/build_and_deploy_docs.yaml | 2 +- .github/workflows/run_code_checks.yaml | 1 - .github/workflows/run_release.yaml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_deploy_docs.yaml b/.github/workflows/build_and_deploy_docs.yaml index ad39c6e3..e34a428a 100644 --- a/.github/workflows/build_and_deploy_docs.yaml +++ b/.github/workflows/build_and_deploy_docs.yaml @@ -53,7 +53,7 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies + - name: Install Python dependencies run: | pipx install --python ${{ matrix.python-version }} poetry make install-dev diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index ef98ad61..00a381fe 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -36,7 +36,6 @@ jobs: docs_check: name: Docs check uses: apify/workflows/.github/workflows/python/docs_check.yaml - secrets: inherit integration_tests: name: Integration tests diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml index b2982b75..c2759794 100644 --- a/.github/workflows/run_release.yaml +++ b/.github/workflows/run_release.yaml @@ -32,7 +32,7 @@ jobs: PYTHON_VERSION: 3.12 steps: - name: Dummy step - run: true + run: "true" lint_check: name: Lint check @@ -96,7 +96,7 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies + - name: Install Python dependencies run: | pipx install --python ${{ matrix.python-version }} poetry make install-dev