diff --git a/.github/workflows/_changelog_entry_check.yaml b/.github/workflows/_changelog_entry_check.yaml new file mode 100644 index 00000000..8ac90673 --- /dev/null +++ b/.github/workflows/_changelog_entry_check.yaml @@ -0,0 +1,29 @@ +name: Changelog entry check + +on: + workflow_call: + +jobs: + check_changelog_entry: + name: Changelog entry check + runs-on: ubuntu-latest + if: (!startsWith(github.event.pull_request.title, 'docs:')) + env: + PYTHON_VERSION: 3.12 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + - name: Execute changelog entry check + run: make check-changelog-entry diff --git a/.github/workflows/_publish_to_pypi.yaml b/.github/workflows/_publish_to_pypi.yaml new file mode 100644 index 00000000..c48c7056 --- /dev/null +++ b/.github/workflows/_publish_to_pypi.yaml @@ -0,0 +1,87 @@ +name: Publish to PyPI + +on: + workflow_call: + inputs: + version_number: + required: true + type: string + +jobs: + publish_to_pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/crawlee + env: + PYTHON_VERSION: 3.12 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + # Determines the release type based on the event that triggered the workflow. + - name: Determine release type + id: determine-release-type + run: | + if [[ ${{ github.event_name }} = release ]]; then + release_type="final" + elif [[ ${{ github.event_name }} = push ]]; then + release_type="beta" + elif [[ ${{ github.event_name }} = workflow_dispatch ]]; then + release_type=${{ github.event.inputs.release_type }} + fi + echo "release_type=${release_type}" >> $GITHUB_OUTPUT + + # Updates the version number for pre-releases in the project's configuration. + - name: Set pre-release version + if: steps.determine-release-type.outputs.release_type != 'final' + run: python ./scripts/update_version_for_prerelease.py ${{ steps.determine-release-type.outputs.release_type }} + + # Builds the package. + - name: Build package + run: make build + + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # If this workflow is not triggered by a GitHub release event, manually create and push a Git tag. + - name: Create Git tag with the published version + if: github.event_name != 'release' + run: | + GIT_TAG=v$(python ./scripts/print_current_package_version.py) + echo "Current package version retrieved: ${GIT_TAG}" + + echo "Creating Git tag: ${GIT_TAG}" + git tag "$GIT_TAG" + echo "Git tag ${GIT_TAG} created successfully." + + echo "Pushing Git tag ${GIT_TAG} to the remote repository." + git push origin tag "$GIT_TAG" + echo "Git tag ${GIT_TAG} pushed successfully." + + # If triggered by a release, upload build artifacts to the associated GitHub release. + - name: Upload the build artifacts to release + if: github.event_name == 'release' + run: | + echo "Uploading build artifacts to GitHub release: ${{ github.ref_name }}" + gh release upload ${{ github.ref_name }} dist/* + echo "Build artifacts uploaded successfully." + + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/_version_conflict_check.yaml b/.github/workflows/_version_conflict_check.yaml new file mode 100644 index 00000000..df4580a5 --- /dev/null +++ b/.github/workflows/_version_conflict_check.yaml @@ -0,0 +1,29 @@ +name: Version conflict check + +on: + workflow_call: + +jobs: + check_version_conflict: + name: Version conflict check + runs-on: ubuntu-latest + if: (!startsWith(github.event.pull_request.title, 'docs:')) + env: + PYTHON_VERSION: 3.12 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + run: | + pipx install --python ${{ env.PYTHON_VERSION }} poetry + make install-dev + + - name: Execute version conflict check + run: make check-version-conflict diff --git a/.github/workflows/docs.yaml b/.github/workflows/build_and_deploy_docs.yaml similarity index 85% rename from .github/workflows/docs.yaml rename to .github/workflows/build_and_deploy_docs.yaml index 0a231e26..e34a428a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/build_and_deploy_docs.yaml @@ -15,6 +15,9 @@ jobs: pages: write id-token: write runs-on: ubuntu-latest + env: + NODE_VERSION: 20 + PYTHON_VERSION: 3.12 steps: - name: Checkout repository @@ -22,14 +25,14 @@ jobs: with: token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - name: Set up Node.js + - name: Set up Node uses: actions/setup-node@v4 with: - node-version: 18 + node-version: ${{ env.NODE_VERSION }} cache: npm cache-dependency-path: website/package-lock.json - - name: Install Node.js dependencies + - name: Install Node dependencies run: | npm install npm update @apify/docs-theme @@ -48,11 +51,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies + - name: Install Python dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - name: Build generated API reference diff --git a/.github/workflows/check_version_availability.yaml b/.github/workflows/check_version_availability.yaml deleted file mode 100644 index 93d82839..00000000 --- a/.github/workflows/check_version_availability.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Check package version availability - -on: - workflow_call: - -jobs: - check_version_availability: - name: Check version availability - runs-on: ubuntu-latest - if: (!startsWith(github.event.pull_request.title, 'docs:')) - - steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Check version availability - run: make check-version-availability diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml deleted file mode 100644 index d96a2dda..00000000 --- a/.github/workflows/integration_tests.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Integration tests - -on: - workflow_call: - secrets: - APIFY_TEST_USER_PYTHON_SDK_API_TOKEN: - description: API token of the Python SDK testing user on Apify - required: true - -concurrency: # This is to make sure that only one run of this workflow is running at the same time, to not overshoot the test user limits - group: integration_tests - -jobs: - integration_tests: - name: Run integration tests - runs-on: ubuntu-latest - strategy: - matrix: - # Run integration tests only on the oldest and newest supported Python versions, - # as these tests are time-consuming and these versions are the most likely to encounter issues. - python-version: ["3.9", "3.12"] - max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits - - steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Run integration tests - run: make INTEGRATION_TESTS_CONCURRENCY=8 integration-tests - env: - APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }} diff --git a/.github/workflows/lint_and_type_checks.yaml b/.github/workflows/lint_and_type_checks.yaml deleted file mode 100644 index a89eca37..00000000 --- a/.github/workflows/lint_and_type_checks.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: Lint and type checks - -on: - workflow_call: - -jobs: - lint_and_type_checks: - name: Lint and type checks - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Run lint - run: make lint - - - name: Run type checks - run: make type-check diff --git a/.github/workflows/run_checks.yaml b/.github/workflows/run_checks.yaml deleted file mode 100644 index 154f30fa..00000000 --- a/.github/workflows/run_checks.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: Code quality checks - -on: - pull_request_target: - -jobs: - check_version_availability: - name: Check version availability - uses: ./.github/workflows/check_version_availability.yaml - - lint_and_type_checks: - name: Run lint and type checks - uses: ./.github/workflows/lint_and_type_checks.yaml - - unit_tests: - name: Run unit tests - needs: [lint_and_type_checks] - uses: ./.github/workflows/unit_tests.yaml - - # If the PR comes from the main repo, run integration tests directly - integration_tests: - if: github.event.pull_request.head.repo.owner.login == 'apify' - name: Run integration tests - needs: [lint_and_type_checks, unit_tests] - uses: ./.github/workflows/integration_tests.yaml - secrets: inherit - - # If the PR comes from a fork, - # we need to approve running the workflow first before allowing it to run, - # so that we can check for any unauthorized access to our secrets. - # We need two workflow jobs for that, - # because jobs calling reusable workflows can't use an environment. - # The first job is a dummy job that just asks for approval to use the `fork-worklows` environment. - integration_tests_fork_approve: - if: github.event.pull_request.head.repo.owner.login != 'apify' - name: Approve integration tests from fork - needs: [lint_and_type_checks, unit_tests] - environment: fork-pr-integration-tests - runs-on: ubuntu-latest - steps: - - name: Dummy step - run: true - - # The second job is the actual integration tests job. - integration_tests_fork: - name: Run integration tests from fork - needs: [integration_tests_fork_approve] - uses: ./.github/workflows/integration_tests.yaml - secrets: inherit diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml new file mode 100644 index 00000000..00a381fe --- /dev/null +++ b/.github/workflows/run_code_checks.yaml @@ -0,0 +1,43 @@ +name: Run code checks + +on: + # Trigger code checks on opening a new pull request. + pull_request_target: + + # Do not trigger code checks on push to the master branch, as they will be triggered + # by the release workflow. + + # Trigger code checks on workflow call (e.g. from run release workflow). + workflow_call: + +jobs: + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python/lint_check.yaml + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python/type_check.yaml + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python/unit_tests.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + changelog_entry_check: + name: Changelog entry check + uses: ./.github/workflows/_changelog_entry_check.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + version_conflict_check: + name: Version conflict check + uses: ./.github/workflows/_version_conflict_check.yaml + + docs_check: + name: Docs check + uses: apify/workflows/.github/workflows/python/docs_check.yaml + + integration_tests: + name: Integration tests + uses: apify/workflows/.github/workflows/python/integration_tests.yaml + secrets: inherit diff --git a/.github/workflows/release.yaml b/.github/workflows/run_release.yaml similarity index 75% rename from .github/workflows/release.yaml rename to .github/workflows/run_release.yaml index 9ccaa91e..c2759794 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/run_release.yaml @@ -1,4 +1,4 @@ -name: Check & Release +name: Run release on: # Push to master will publish a beta version @@ -28,36 +28,64 @@ jobs: name: Check whether to release if: (!startsWith(github.event.head_commit.message, 'docs:') || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.12 steps: - name: Dummy step - run: true + run: "true" - lint_and_type_checks: - name: Run lint and type_checks + lint_check: + name: Lint check needs: [should_release] - uses: ./.github/workflows/lint_and_type_checks.yaml + uses: apify/workflows/.github/workflows/python/lint_check.yaml + + type_check: + name: Type check + needs: [should_release] + uses: apify/workflows/.github/workflows/python/type_check.yaml unit_tests: - name: Run unit tests + name: Unit tests + needs: [should_release] + uses: apify/workflows/.github/workflows/python/unit_tests.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + changelog_entry_check: + name: Changelog entry check + needs: [should_release] + uses: ./.github/workflows/_changelog_entry_check.yaml + + # TODO: remove this once https://github.com/apify/apify-sdk-python/issues/241 is resolved + version_conflict_check: + name: Version conflict check needs: [should_release] - uses: ./.github/workflows/unit_tests.yaml + uses: ./.github/workflows/_version_conflict_check.yaml integration_tests: - name: Run integration tests + name: Integration tests needs: [should_release] - uses: ./.github/workflows/integration_tests.yaml + uses: apify/workflows/.github/workflows/python/integration_tests.yaml secrets: inherit publish_to_pypi: name: Publish to PyPI - needs: [should_release, lint_and_type_checks, unit_tests, integration_tests] + needs: + [ + should_release, + lint_check, + type_check, + unit_tests, + changelog_entry_check, + version_conflict_check, + integration_tests, + ] runs-on: ubuntu-latest permissions: contents: write id-token: write environment: name: pypi - url: https://pypi.org/p/apify + url: https://pypi.org/project/apify/ steps: - name: Checkout repository @@ -66,11 +94,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies + - name: Install Python dependencies run: | - pipx install poetry + pipx install --python ${{ matrix.python-version }} poetry make install-dev - # Determine if this is a prerelease or latest release diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml deleted file mode 100644 index ccb5f6ed..00000000 --- a/.github/workflows/unit_tests.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Unit tests - -on: - workflow_call: - -jobs: - unit_tests: - name: Run unit tests - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ matrix.os }} - - steps: - # We need to check out the head commit in case of PRs, - # and the default ref otherwise (during release). - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}" - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pipx install poetry - make install-dev - - - name: Run unit tests - run: make unit-tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c1cdf523..f12158e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,8 +19,8 @@ repos: language: system pass_filenames: false - - id: check-version-availability - name: Check version availability - entry: make check-version-availability + - id: check-version-conflict + name: Check version conflict + entry: make check-version-conflict language: system pass_filenames: false diff --git a/Makefile b/Makefile index 768a2c8f..b51c1572 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ -.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests format check-code check-version-availability check-changelog-entry build-api-reference run-doc +.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov \ + integration-tests format check-code check-version-availability check-changelog-entry \ + check-version-conflict build-api-reference run-docs DIRS_WITH_CODE = src tests scripts @@ -40,18 +42,21 @@ format: poetry run ruff check --fix $(DIRS_WITH_CODE) poetry run ruff format $(DIRS_WITH_CODE) -# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks -# and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests +check-changelog-entry: + poetry run python scripts/check_changelog_entry.py -check-version-availability: - poetry run python scripts/check_version_availability.py +check-version-conflict: + poetry run python scripts/check_version_conflict.py -check-changelog-entry: - poetry run python scripts/check_version_in_changelog.py +# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks +# and the run_checks.yaml GitHub Actions workflow. +check-code: lint type-check unit-tests check-changelog-entry check-version-conflict build-api-reference: cd website && poetry run ./build_api_reference.sh -run-doc: build-api-reference +build-docs: build-api-reference + cd website && yarn && yarn build + +run-docs: build-api-reference cd website && npm clean-install && npm run start diff --git a/scripts/check_version_in_changelog.py b/scripts/check_changelog_entry.py similarity index 100% rename from scripts/check_version_in_changelog.py rename to scripts/check_changelog_entry.py diff --git a/scripts/check_version_availability.py b/scripts/check_version_conflict.py similarity index 100% rename from scripts/check_version_availability.py rename to scripts/check_version_conflict.py