Rewrite documentation building to use Docusaurus (#149)

Author: fnesveda

.flake8

@@ -1,11 +1,9 @@
 [flake8]
 filename =
-    ./docs/*.py,
     ./scripts/*.py,
     ./src/*.py,
     ./tests/*.py
 per-file-ignores =
-    docs/*: D
     scripts/*: D
     tests/*: D
 

.github/workflows/check_docs.yaml renamed to .github/workflows/check_async_docstrings.yaml

@@ -1,11 +1,11 @@
-name: Check documentation status
+name: Check async docstrings
 
 on:
   workflow_call:
 
 jobs:
-  check_docs:
-    name: Check whether documentation is up-to-date
+  check_async_docstrings:
+    name: Check whether doctrings for async methods are up-to-date
     runs-on: ubuntu-latest
 
     steps:
@@ -22,6 +22,3 @@ jobs:
 
       - name: Check async docstrings
         run: make check-async-docstrings
-
-      - name: Check docs building
-        run: make check-docs

.github/workflows/docs.yaml

@@ -0,0 +1,77 @@
+name: Build and deploy docs
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  build_and_deploy_docs:
+    environment:
+      name: github-pages
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: npm
+          cache-dependency-path: website/package-lock.json
+
+      - name: Install Node.js dependencies
+        run: |
+          npm install
+          npm update @apify/docs-theme
+        working-directory: ./website
+
+      # We do this as early as possible to prevent conflicts if someone else would push something in the meantime
+      - name: Commit the updated package.json and lockfile
+        run: |
+          git config user.name 'GitHub Actions'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+          git add website/package.json
+          git add website/package-lock.json
+          git diff-index --quiet HEAD || git commit -m 'chore: Automatic docs theme update [skip ci]' || true
+          git push
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+
+      - name: Install Python dependencies
+        run: make install-dev
+
+      - name: Build generated API reference
+        run: make build-api-reference
+
+      - name: Build Docusaurus docs
+        run: npm run build
+        working-directory: ./website
+
+      - name: Set up GitHub Pages
+        uses: actions/configure-pages@v3
+
+      - name: Upload GitHub Pages artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: ./website/build
+
+      - name: Deploy artifact to GitHub Pages
+        uses: actions/deploy-pages@v2
+
+      - name: Invalidate CloudFront cache
+        run: gh workflow run invalidate.yaml --repo apify/apify-docs-private
+        env:
+          GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -24,26 +24,38 @@ on:
           - final
 
 jobs:
+  should_release:
+    name: Check whether to release
+    if: (!startsWith(github.event.head_commit.message, 'docs:') || github.event_name == 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Dummy step
+        run: true
+
   lint_and_type_checks:
     name: Run lint and type_checks
+    needs: [should_release]
     uses: ./.github/workflows/lint_and_type_checks.yaml
 
   unit_tests:
     name: Run unit tests
+    needs: [should_release]
     uses: ./.github/workflows/unit_tests.yaml
 
+  check_async_docstrings:
+    name: Check whether async dostrings are up to date
+    needs: [should_release]
+    uses: ./.github/workflows/check_async_docstrings.yaml
+
   integration_tests:
     name: Run integration tests
+    needs: [should_release]
     uses: ./.github/workflows/integration_tests.yaml
     secrets: inherit
 
-  check_docs:
-    name: Check whether the documentation is up to date
-    uses: ./.github/workflows/check_docs.yaml
-
   publish_to_pypi:
     name: Publish to PyPI
-    needs: [lint_and_type_checks, unit_tests, integration_tests, check_docs]
+    needs: [should_release, lint_and_type_checks, unit_tests, check_async_docstrings, integration_tests]
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/run_checks.yaml

@@ -13,12 +13,12 @@ jobs:
     needs: [lint_and_type_checks]
     uses: ./.github/workflows/unit_tests.yaml
 
-  check_docs:
-    name: Check whether the documentation is up to date
-    uses: ./.github/workflows/check_docs.yaml
+  check_async_docstrings:
+    name: Check async dostrings
+    uses: ./.github/workflows/check_async_docstrings.yaml
 
   integration_tests:
     name: Run integration tests
-    needs: [lint_and_type_checks, unit_tests, check_docs]
+    needs: [lint_and_type_checks, unit_tests, check_async_docstrings]
     uses: ./.github/workflows/integration_tests.yaml
     secrets: inherit

.gitignore

@@ -15,3 +15,5 @@ build/
 .vscode
 .idea
 .DS_Store
+
+docs/changelog.md

.isort.cfg

@@ -4,4 +4,4 @@ line_length = 150
 use_parentheses = True
 multi_line_output = 3
 sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
-known_first_party = apify_client, apify_shared
+known_first_party = apify,apify_client,apify_shared

.markdownlint.yaml

@@ -0,0 +1,5 @@
+default: true
+line-length:
+  line_length: 150
+ul-style: dash
+no-inline-html: false

.pre-commit-config.yaml

@@ -25,12 +25,6 @@ repos:
         language: system
         pass_filenames: false
 
-      - id: check-docs
-        name: Check whether docs are up-to-date
-        entry: make check-docs
-        language: system
-        pass_filenames: false
-
       - id: check-changelog
         name: Check whether current version is mentioned in changelog
         entry: make check-changelog-entry

CONTRIBUTING.md

@@ -30,6 +30,15 @@ We use `autopep8` and `isort` to automatically format the code to a common forma
 
 We use `flake8` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `make check-code`.
 
+## Integration tests
+
+We have integration tests which send API requests using the API client to the Apify Platform.
+To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests,
+and then start them with `make integration-tests`.
+
+If you want to run the integration tests on a different environment than the main Apify Platform,
+you need to set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use.
+
 ## Documentation
 
 We use the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting the code.