Various changes, experiments and updates

Author: Blargian

.github/workflows/table_of_contents.yml

@@ -19,43 +19,61 @@ jobs:
   generate_toc_formats:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      # Step 1: Check out the repository
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          repository: ClickHouse/clickhouse-docs
+          ref: ${{ github.ref }} # checkout the current branch
 
-      # Step 1 - Cache directory contents
-      - name: Cache directory contents # Generating the TOC if there are no files added/removed is wasteful
-        uses: actions/cache@v3
+       # Step 2 - Setup Python
+      - name: Set up Python
+        uses: actions/setup-python@v3
         with:
-          path: |
-            docs/en/interfaces/formats
-          key: toc-cache-${{ hashFiles('docs/en/interfaces/formats/**')}}
-
-      # Step 2 - Check if Cache was hit (files have not changed) generate the TOC
-      - name: Generate Format Table Of Contents
-        if: steps.Cache.outputs.cache-hit != 'true' # If there's no changes
-        id: toc_gen
+          python-version: '3.x'
+
+      # Step 3: Install Python dependencies
+      - name: Install dependencies
         run: |
-          # Step 2.1 - Setup Python
-          - name: Set up Python
-            uses: actions/setup-python@v3
-            with:
-              python-version: '3.x'
-
-          # Step 2.2: Install Python dependencies
-          - name: Install dependencies
-            run: |
-              python -m pip install --upgrade pip
-              pip install -r 'scripts/knowledgebase-checker/requirements.txt'
-          
-          # Step 2.3: Run scripts to generate TOCs:
-          - name: Generate TOCs
-            run: |
-              ./scripts/table-of-contents-generator/toc_gen.py --kb-dir="docs/en/interfaces/formats" --single-toc
-            continue-on-error: true
-
-      # Step 6: Fail the build if any script returns exit code 1
+          python -m pip install --upgrade pip
+          pip install -r 'scripts/table-of-contents-generator/requirements.txt'
+
+      # Step 4 -  Pull main repo docs, run script to generate TOCs:
+      - name: Generate TOCs
+        run: |
+          yarn prep-from-master
+          python -u ./scripts/table-of-contents-generator/toc_gen.py --dir="docs/en/interfaces/formats" --single-toc --out="table-of-contents-files" --ignore "_snippets"
+
+      # Step 5 - Fail the workflow if script returns exit code 1
       - name: Check exit code
         run: |
           if [[ "${{ steps.toc_gen.outcome }}" == "failure" ]]; then
             echo "Ran into trouble generating a table of contents. See the logs for details."
             exit 1
-          fi
+          fi
+
+      # Step 6 - Check if anything was actually updated
+      - name: Check for Changes
+        id: check_changes
+        run: |
+          git status -u
+          if [[ -n "$(git diff --exit-code)" ]]; then
+            echo "Changes detected."
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          else
+            echo "No changes detected."
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          fi
+
+      # Step 7 - Commit and Push generated Table Of Contents files
+      - name: Commit and Push Changes
+        if: steps.check_changes.outputs.has_changes == 'true'
+        run: |
+          # configure the user
+          git config --global user.name "${{ github.actor }}"
+          git config --global user.email "${{ github.actor}}@users.noreply.github.com"
+          # standard git flow
+          git checkout 
+          git add table-of-contents-files/*
+          git commit -m "Autogenerate table of contents files from GitHub action - $(date '+%Y-%m-%d %H:%M:%S')"
+          git push origin HEAD:update_table_of_contents

.gitignore

@@ -49,3 +49,6 @@ docs/en/cloud/manage/api/services-api-reference.md
 .vscode
 .aspell.en.prepl
 *.md.bak
+
+# Don't ignore generated table of contents files
+!toc.json

scripts/table-of-contents-generator/requirements.txt

@@ -0,0 +1 @@
+PyYAML==6.0.2

scripts/table-of-contents-generator/toc_gen.py

@@ -1,14 +1,16 @@
+#!/usr/bin/env python3
+
 """
 This script can be used to automatically generate a table of contents (JSON file) from the markdown files in a directory,
 or multiple directories.
 """
 
-#!/usr/bin/env python3
-
 import json
 import os
 import argparse
 import sys
+from collections import defaultdict
+import yaml
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
@@ -20,43 +22,82 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Generates a single TOC for all files in all sub-directories of provided directory. By default, generates TOC per folder.",
     )
+    parser.add_argument(
+        "--out",
+        default=None,
+        help="Path to output the resulting table of contents file to (by default it is output to the provided directory - file is named according to --dir)"
+    )
     parser.add_argument(
         "--dir",
         help="Path to a folder containing markdown (.md, .mdx) documents containing YAML with title, description, slug."
     )
+    parser.add_argument('--ignore', metavar='S', type=str, nargs='+',
+                        help='Directory names to ignore. E.g --ignore _snippets images')
     return parser.parse_args()
 
 def extract_title_description_slug(filename):
-    with open(filename, "r") as f:
-        lines = f.readlines()
-
-    title, description, slug = None, None, None
-    for line in lines:
-        if line.startswith("title:"):
-            title = line.strip().split(": ")[1]
-        if line.startswith("description:"):
-            description = line.strip().split(": ")[1]
-        elif line.startswith("slug:"):
-            slug = line.strip().split(": ")[1]
-    if title and slug and description:
-        return {"title": title, "description": description, "slug": slug, "dir": filename}
-    return None
-
-def walk_dirs(root_dir):
+    data = defaultdict(str)
+    missing_fields = []
+    frontmatter_data = {}
+
+    try:
+        with open(filename, "r") as f:
+            content = f.read()
+            # find the first frontmatter tag
+            frontmatter_start = content.find('---\n')
+            if frontmatter_start != -1:
+                # find the second frontmatter tag
+                frontmatter_end = content.find('---\n', frontmatter_start + 4)
+                if frontmatter_start != -1:
+                    # find the second frontmatter tag
+                    frontmatter_end = content.find('---\n', frontmatter_start + 4)
+                    if frontmatter_end != -1:
+                        frontmatter_str = content[frontmatter_start+4:frontmatter_end]
+                        frontmatter_data = yaml.safe_load(frontmatter_str) or {}
+
+        data.update(frontmatter_data)
+
+        if missing_fields:
+            print(f"Warning: {filename} is missing some fields:")
+            for field in missing_fields:
+                print(f"- {field}")
+
+        return frontmatter_data
+    except OSError as e:
+        print(f"Ran into a problem reading frontmatter: {e}")
+        sys.exit(1)
+def walk_dirs(root_dir, ignore_dirs=[]):
     for root, dirs, files in os.walk(root_dir):
+        # Modify the 'dirs' list in-place to remove ignored directories
+        dirs[:] = [d for d in dirs if d not in ignore_dirs
+                   and not any(d.startswith(ig) for ig in ignore_dirs)]
         yield root
 
-def write_to_file(json_array, output_path):
+def write_to_file(json_items, directory, output=None):
+
+    if output is not None:
+        # output to the given path the toc.json file
+        # If dir='docs/en/interfaces/formats' the file is called docs_en_interfaces_formats_toc.json
+        output_path = output+"/"+directory.replace("/", "_")
+    else:
+        output_path = directory
     try:
         os.makedirs(os.path.dirname(output_path), exist_ok=True)  # Create directories if they don't exist
         with open(output_path, "w") as f:
-            json.dump(json_array, f, indent=4)
+            json.dump(json_items, f, indent=4)
             f.write('\n')
+            print(f"Wrote {output_path}")
     except OSError as e:
         if e.errno == 21:
             print(f"Directory already exists: {e}")
         else:
             print(f"An error occurred creating directory: {e}")
+def write_file(json_items, args, directory):
+    print(args)
+    if args.out is not None:
+        write_to_file(json_items, directory+"/toc.json", args.out)
+    elif args.out is None:
+        write_to_file(json_items, directory+"/toc.json")
 
 def main():
 
@@ -66,13 +107,15 @@ def main():
     if root_dir is None:
         print("Please provide a directory with argument --dir")
         sys.exit(1)
-
-    if args.single_toc:
+    if os.path.lexists(root_dir) is False:
+        print("Path provided does not exist")
+        sys.exit(1)
+    if args.single_toc is True:
         json_items = [] # single list for all directories
 
-    for directory in walk_dirs(root_dir): # Walk directories
+    for directory in walk_dirs(root_dir, args.ignore): # Walk directories
 
-        if not args.single_toc:
+        if args.single_toc is False:
             json_items = [] # new list for each directory
 
         for filename in os.listdir(directory): # for each directory
@@ -85,19 +128,24 @@ def main():
                     result = extract_title_description_slug(full_path)
                     if result is not None:
                         json_items.append(result)
-
-        if not args.single_toc:
-            json_array = sorted(json_items, key=lambda x: x.get("title"))
-
+                        if args.single_toc is False:
+                            # don't write toc.json for empty folders
+                            if len(json_items) != 0:
+                                json_items = sorted(json_items, key=lambda x: x.get("title"))
+                                # output to the specified directory if arg --out is provided
+                                write_file(json_items, args, directory)
+                            else:
+                                print("Ran into an issue trying to extract YAML: empty result")
+
+        if args.single_toc is True:
             # don't write toc.json for empty folders
             if len(json_items) != 0:
-                write_to_file(json_items, directory+"/toc.json")
-
-    if args.single_toc:
-        json_array = sorted(json_items, key=lambda x: x.get("title"))
-        # don't write toc.json for empty folders
-        if len(json_items) != 0:
-            write_to_file(json_items, root_dir+"/toc.json")
+                json_array = sorted(json_items, key=lambda x: x.get("title"))
+                # output to the specified directory if arg --out is provided
+                write_file(json_items, args, directory)
+                sys.exit(0)
+            else:
+                sys.exit(1)
 
 if __name__ == "__main__":
     main()