Merge branch 'main' into scripts_refactor

Author: Blargian

.github/workflows/build-search.yml

@@ -0,0 +1,44 @@
+name: Update Algolia Search
+
+on:
+  pull_request:
+    types:
+      - closed
+
+  workflow_dispatch:
+  
+  schedule:
+    - cron: '0 4 * * *'
+
+env:
+  PYTHONUNBUFFERED: 1 # Force the stdout and stderr streams to be unbuffered
+
+jobs:
+  update-search:
+    if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'update search') && github.event.pull_request.base.ref == 'main'
+    #if: contains(github.event.pull_request.labels.*.name, 'update search') # Updated to trigger directly on PRs with the label
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20' 
+
+      - name: Run Prep from Master
+        run: yarn copy-clickhouse-repo-docs
+
+      - name: Run Auto Generate Settings
+        run: yarn auto-generate-settings
+
+      - name: Run Indexer
+        run: yarn run-indexer
+        env:
+          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
+          ALGOLIA_APP_ID: 5H9UG7CX5W
+
+      - name: Verify Completion
+        run: echo "All steps completed successfully!"

.github/workflows/table_of_contents.yml

@@ -0,0 +1,74 @@
+# This GitHub Action is used for triggering updates of
+# the toc.json files present in any directory that
+# needs an automatically generated table of contents.
+
+name: Generate Table of Contents files
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  generate_toc_formats:
+    runs-on: ubuntu-latest
+    steps:
+      # Step 1: Check out the repository
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+       # Step 2 - Setup Python
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
+
+      # Step 3: Install Python dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r 'scripts/table-of-contents-generator/requirements.txt'
+
+      # Step 4 -  Pull main repo docs, run script to generate TOCs:
+      - name: Generate TOCs
+        run: |
+          yarn prep-from-master
+          python -u ./scripts/table-of-contents-generator/toc_gen.py --dir="docs/en/interfaces/formats" --single-toc --out="table-of-contents-files" --ignore "_snippets"
+
+      # Step 5 - Fail the workflow if script returns exit code 1
+      - name: Check exit code
+        run: |
+          if [[ "${{ steps.toc_gen.outcome }}" == "failure" ]]; then
+            echo "Ran into trouble generating a table of contents. See the logs for details."
+            exit 1
+          fi
+
+      # Step 6 - Check if anything was actually updated
+      - name: Check for Changes
+        id: check_changes
+        run: |
+          git status -u
+          if [[ -n "$(git diff --exit-code)" ]]; then
+            echo "Changes detected."
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          else
+            echo "No changes detected."
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          fi
+
+      # Step 7 - Commit and Push generated Table Of Contents files
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          commit_message: "Autogenerate table of contents files from GitHub action - $(date '+%Y-%m-%d %H:%M:%S')"
+          file_pattern: 'table-of-contents-files/*'
+          branch: generate_table_of_contents
+          create_branch: true

.gitignore

@@ -49,3 +49,6 @@ docs/en/cloud/manage/api/services-api-reference.md
 .vscode
 .aspell.en.prepl
 *.md.bak
+
+# Don't ignore generated table of contents files
+!toc.json

copyClickhouseRepoDocs.sh

@@ -31,10 +31,13 @@ echo "[$SCRIPT_NAME] Copying completed"
 
 echo "[$SCRIPT_NAME] Generate changelog"
 cp docs/en/_placeholders/changelog/_index.md docs/en/whats-new/changelog/index.md
-if grep -q '^# $(date +%Y) Changelog' ClickHouse/CHANGELOG.md; then
-  sed '/^# $(date +%Y) Changelog/d' ClickHouse/CHANGELOG.md > temp.txt
-  cat >> docs/en/whats-new/changelog/index.md
-  rm temp.txt
+if grep -q "^# $(date +%Y) Changelog" ClickHouse/CHANGELOG.md; then
+  echo "Generating $(date +%Y) Changelog..."
+  sed "/^# $(date +%Y) Changelog/d" ClickHouse/CHANGELOG.md > temp.txt
+  echo "Changelog copied to temp.txt"
+  cat temp.txt >> docs/en/whats-new/changelog/index.md
+  echo "Changelog written to docs/en/whats-new/changelog/index.md"
+  rm -f temp.txt
   echo "$(date +%Y) Changelog was updated."
 else
   current_year="$(date +%Y)"