ci: use shared workflows and refactor them

.github/workflows/check_version_availability.yaml → .github/workflows/_check_changelog_entry.yaml

@@ -1,17 +1,18 @@
-name: Check package version availability
+name: Check changelog entry
 
 on:
   workflow_call:
 
 jobs:
-  check_version_availability:
-    name: Check version availability
+  check_changelog_entry:
+    name: Check changelog entry
     runs-on: ubuntu-latest
     if: (!startsWith(github.event.pull_request.title, 'docs:'))
+    env:
+      PYTHON_VERSION: 3.12
 
     steps:
-      # We need to check out the head commit in case of PRs,
-      # and the default ref otherwise (during release).
+      # We need to check out the head commit in case of PRs, and the default ref otherwise (during release).
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -20,12 +21,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install dependencies
         run: |
-          pipx install poetry
+          pipx install --python ${{ env.PYTHON_VERSION }} poetry
           make install-dev
 
-      - name: Check version availability
-        run: make check-version-availability
+      - name: Check changelog entry
+        run: make check-changelog-entry

.github/workflows/_check_docs_build.yaml

@@ -0,0 +1,50 @@
+name: Docs build
+
+on:
+  workflow_call:
+
+jobs:
+  docs:
+    name: Docs build
+    runs-on: ubuntu-latest
+    env:
+      NODE_VERSION: 20
+      PYTHON_VERSION: 3.12
+
+    steps:
+      - name: Checkout Source code
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Enable corepack
+        run: |
+          corepack enable
+          corepack prepare yarn@stable --activate
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          pipx install --python ${{ env.PYTHON_VERSION }} poetry
+          make install-dev
+
+      - name: Build generated API reference
+        run: make build-api-reference
+
+      - name: Install dependencies and try to build the website
+        run: |
+          # go to website dir
+          cd website
+          # install website deps
+          yarn
+          # build the docs
+          yarn build
+        env:
+          APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}

.github/workflows/_check_version_conflict.yaml

@@ -0,0 +1,29 @@
+name: Check version conflict
+
+on:
+  workflow_call:
+
+jobs:
+  check_version_conflict:
+    name: Check version conflict
+    runs-on: ubuntu-latest
+    if: (!startsWith(github.event.pull_request.title, 'docs:'))
+    env:
+      PYTHON_VERSION: 3.12
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          pipx install --python ${{ env.PYTHON_VERSION }} poetry
+          make install-dev
+
+      - name: Check version conflict
+        run: make check-version-conflict

.github/workflows/integration_tests.yaml → .github/workflows/_integration_tests.yaml

@@ -12,22 +12,16 @@ concurrency: # This is to make sure that only one run of this workflow is runnin
 
 jobs:
   integration_tests:
-    name: Run integration tests
+    name: Integration tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # Run integration tests only on the oldest and newest supported Python versions,
-        # as these tests are time-consuming and these versions are the most likely to encounter issues.
-        python-version: ["3.9", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
       max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits
 
     steps:
-      # We need to check out the head commit in case of PRs,
-      # and the default ref otherwise (during release).
       - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}"
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -36,7 +30,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pipx install poetry
+          pipx install --python ${{ matrix.python-version }} poetry
           make install-dev
 
       - name: Run integration tests

.github/workflows/_linting.yaml

@@ -0,0 +1,32 @@
+name: Linting
+
+on:
+  workflow_call:
+
+jobs:
+  linting:
+    name: Linting
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      # We need to check out the head commit in case of PRs, and the default ref otherwise (during release).
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pipx install --python ${{ matrix.python-version }} poetry
+          make install-dev
+
+      - name: Run linting
+        run: make lint

.github/workflows/_publish_to_pypi.yaml

@@ -0,0 +1,87 @@
+name: Publish to PyPI
+
+on:
+  workflow_call:
+    inputs:
+      version_number:
+        required: true
+        type: string
+
+jobs:
+  publish_to_pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write # Required for OIDC authentication.
+    environment:
+      name: pypi
+      url: https://pypi.org/project/crawlee
+    env:
+      PYTHON_VERSION: 3.12
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          pipx install --python ${{ env.PYTHON_VERSION }} poetry
+          make install-dev
+
+      # Determines the release type based on the event that triggered the workflow.
+      - name: Determine release type
+        id: determine-release-type
+        run: |
+          if [[ ${{ github.event_name }} = release ]]; then
+            release_type="final"
+          elif [[ ${{ github.event_name }} = push ]]; then
+            release_type="beta"
+          elif [[ ${{ github.event_name }} = workflow_dispatch ]]; then
+            release_type=${{ github.event.inputs.release_type }}
+          fi
+          echo "release_type=${release_type}" >> $GITHUB_OUTPUT
+
+      # Updates the version number for pre-releases in the project's configuration.
+      - name: Set pre-release version
+        if: steps.determine-release-type.outputs.release_type != 'final'
+        run: python ./scripts/update_version_for_prerelease.py ${{ steps.determine-release-type.outputs.release_type }}
+
+      # Builds the package.
+      - name: Build package
+        run: make build
+
+      # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication.
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+      # If this workflow is not triggered by a GitHub release event, manually create and push a Git tag.
+      - name: Create Git tag with the published version
+        if: github.event_name != 'release'
+        run: |
+          GIT_TAG=v$(python ./scripts/print_current_package_version.py)
+          echo "Current package version retrieved: ${GIT_TAG}"
+
+          echo "Creating Git tag: ${GIT_TAG}"
+          git tag "$GIT_TAG"
+          echo "Git tag ${GIT_TAG} created successfully."
+
+          echo "Pushing Git tag ${GIT_TAG} to the remote repository."
+          git push origin tag "$GIT_TAG"
+          echo "Git tag ${GIT_TAG} pushed successfully."
+
+      # If triggered by a release, upload build artifacts to the associated GitHub release.
+      - name: Upload the build artifacts to release
+        if: github.event_name == 'release'
+        run: |
+          echo "Uploading build artifacts to GitHub release: ${{ github.ref_name }}"
+          gh release upload ${{ github.ref_name }} dist/*
+          echo "Build artifacts uploaded successfully."
+
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/lint_and_type_checks.yaml → .github/workflows/_type_checking.yaml

@@ -1,19 +1,18 @@
-name: Lint and type checks
+name: Type checking
 
 on:
   workflow_call:
 
 jobs:
-  lint_and_type_checks:
-    name: Lint and type checks
+  type_checking:
+    name: Type checking
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
-      # We need to check out the head commit in case of PRs,
-      # and the default ref otherwise (during release).
+      # We need to check out the head commit in case of PRs, and the default ref otherwise (during release).
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -26,11 +25,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pipx install poetry
+          pipx install --python ${{ matrix.python-version }} poetry
           make install-dev
 
-      - name: Run lint
-        run: make lint
-
-      - name: Run type checks
+      - name: Run type checking
         run: make type-check

.github/workflows/unit_tests.yaml → .github/workflows/_unit_tests.yaml

@@ -5,16 +5,15 @@ on:
 
 jobs:
   unit_tests:
-    name: Run unit tests
+    name: Unit tests
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
 
     steps:
-      # We need to check out the head commit in case of PRs,
-      # and the default ref otherwise (during release).
+      # We need to check out the head commit in case of PRs, and the default ref otherwise (during release).
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -27,8 +26,10 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pipx install poetry
+          pipx install --python ${{ matrix.python-version }} poetry
           make install-dev
 
       - name: Run unit tests
         run: make unit-tests
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -15,21 +15,24 @@ jobs:
       pages: write
       id-token: write
     runs-on: ubuntu-latest
+    env:
+      NODE_VERSION: 18
+      PYTHON_VERSION: 3.12
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
 
-      - name: Set up Node.js
+      - name: Set up Node
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: ${{ env.NODE_VERSION }}
           cache: npm
           cache-dependency-path: website/package-lock.json
 
-      - name: Install Node.js dependencies
+      - name: Install Node dependencies
         run: |
           npm install
           npm update @apify/docs-theme
@@ -48,11 +51,11 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install dependencies
         run: |
-          pipx install poetry
+          pipx install --python ${{ matrix.python-version }} poetry
           make install-dev
 
       - name: Build generated API reference