Merge pull request #3133 from Blargian/formats_action

Blargian · web-flow · commit dbf34ba09dbd · 2025-01-24T23:22:28.000+01:00
Add Github action for autogenerating table of contents json files
diff --git a/.github/workflows/table_of_contents.yml b/.github/workflows/table_of_contents.yml
@@ -0,0 +1,74 @@
+# This GitHub Action is used for triggering updates of
+# the toc.json files present in any directory that
+# needs an automatically generated table of contents.
+
+name: Generate Table of Contents files
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:
+  pull_request:
+    types:
+      - synchronize
+      - reopened
+      - opened
+
+permissions: write-all
+
+jobs:
+  generate_toc_formats:
+    runs-on: ubuntu-latest
+    steps:
+      # Step 1: Check out the repository
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+       # Step 2 - Setup Python
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
+
+      # Step 3: Install Python dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r 'scripts/table-of-contents-generator/requirements.txt'
+
+      # Step 4 -  Pull main repo docs, run script to generate TOCs:
+      - name: Generate TOCs
+        run: |
+          yarn prep-from-master
+          python -u ./scripts/table-of-contents-generator/toc_gen.py --dir="docs/en/interfaces/formats" --single-toc --out="table-of-contents-files" --ignore "_snippets"
+
+      # Step 5 - Fail the workflow if script returns exit code 1
+      - name: Check exit code
+        run: |
+          if [[ "${{ steps.toc_gen.outcome }}" == "failure" ]]; then
+            echo "Ran into trouble generating a table of contents. See the logs for details."
+            exit 1
+          fi
+
+      # Step 6 - Check if anything was actually updated
+      - name: Check for Changes
+        id: check_changes
+        run: |
+          git status -u
+          if [[ -n "$(git diff --exit-code)" ]]; then
+            echo "Changes detected."
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          else
+            echo "No changes detected."
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          fi
+
+      # Step 7 - Commit and Push generated Table Of Contents files
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          commit_message: "Autogenerate table of contents files from GitHub action - $(date '+%Y-%m-%d %H:%M:%S')"
+          file_pattern: 'table-of-contents-files/*'
+          create_branch: true
diff --git a/.gitignore b/.gitignore
@@ -49,3 +49,6 @@ docs/en/cloud/manage/api/services-api-reference.md
 .vscode
 .aspell.en.prepl
 *.md.bak
+
+# Don't ignore generated table of contents files
+!toc.json
diff --git a/scripts/autogenerate_table_of_contents.py b/scripts/autogenerate_table_of_contents.py
diff --git a/scripts/table-of-contents-generator/requirements.txt b/scripts/table-of-contents-generator/requirements.txt
@@ -0,0 +1 @@
+PyYAML==6.0.2
diff --git a/scripts/table-of-contents-generator/toc_gen.py b/scripts/table-of-contents-generator/toc_gen.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+"""
+This script can be used to automatically generate a table of contents (JSON file) from the markdown files in a directory,
+or multiple directories.
+"""
+
+import json
+import os
+import argparse
+import sys
+from collections import defaultdict
+import yaml
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Script to generate .json table of contents from YAML frontmatter title, description and slug",
+    )
+    parser.add_argument(
+        "--single-toc",
+        action="store_true",
+        help="Generates a single TOC for all files in all sub-directories of provided directory. By default, generates TOC per folder.",
+    )
+    parser.add_argument(
+        "--out",
+        default=None,
+        help="Path to output the resulting table of contents file to (by default it is output to the provided directory - file is named according to --dir)"
+    )
+    parser.add_argument(
+        "--dir",
+        help="Path to a folder containing markdown (.md, .mdx) documents containing YAML with title, description, slug."
+    )
+    parser.add_argument('--ignore', metavar='S', type=str, nargs='+',
+                        help='Directory names to ignore. E.g --ignore _snippets images')
+    return parser.parse_args()
+
+def extract_title_description_slug(filename):
+    data = defaultdict(str)
+    missing_fields = []
+    frontmatter_data = {}
+
+    try:
+        with open(filename, "r") as f:
+            content = f.read()
+            # find the first frontmatter tag
+            frontmatter_start = content.find('---\n')
+            if frontmatter_start != -1:
+                # find the second frontmatter tag
+                frontmatter_end = content.find('---\n', frontmatter_start + 4)
+                if frontmatter_start != -1:
+                    # find the second frontmatter tag
+                    frontmatter_end = content.find('---\n', frontmatter_start + 4)
+                    if frontmatter_end != -1:
+                        frontmatter_str = content[frontmatter_start+4:frontmatter_end]
+                        frontmatter_data = yaml.safe_load(frontmatter_str) or {}
+
+        data.update(frontmatter_data)
+
+        if missing_fields:
+            print(f"Warning: {filename} is missing some fields:")
+            for field in missing_fields:
+                print(f"- {field}")
+
+        return frontmatter_data
+    except OSError as e:
+        print(f"Ran into a problem reading frontmatter: {e}")
+        sys.exit(1)
+def walk_dirs(root_dir, ignore_dirs=[]):
+    for root, dirs, files in os.walk(root_dir):
+        # Modify the 'dirs' list in-place to remove ignored directories
+        dirs[:] = [d for d in dirs if d not in ignore_dirs
+                   and not any(d.startswith(ig) for ig in ignore_dirs)]
+        yield root
+
+def write_to_file(json_items, directory, output=None):
+
+    if output is not None:
+        # output to the given path the toc.json file
+        # If dir='docs/en/interfaces/formats' the file is called docs_en_interfaces_formats_toc.json
+        output_path = output+"/"+directory.replace("/", "_")
+    else:
+        output_path = directory
+    try:
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)  # Create directories if they don't exist
+        with open(output_path, "w") as f:
+            json.dump(json_items, f, indent=4)
+            f.write('\n')
+            print(f"Wrote {output_path}")
+    except OSError as e:
+        if e.errno == 21:
+            print(f"Directory already exists: {e}")
+        else:
+            print(f"An error occurred creating directory: {e}")
+def write_file(json_items, args, directory):
+    print(args)
+    if args.out is not None:
+        write_to_file(json_items, directory+"/toc.json", args.out)
+    elif args.out is None:
+        write_to_file(json_items, directory+"/toc.json")
+
+def main():
+
+    # Extract script arguments
+    args = parse_args()
+    root_dir = args.dir
+    if root_dir is None:
+        print("Please provide a directory with argument --dir")
+        sys.exit(1)
+    if os.path.lexists(root_dir) is False:
+        print("Path provided does not exist")
+        sys.exit(1)
+    if args.single_toc is True:
+        json_items = [] # single list for all directories
+
+    for directory in walk_dirs(root_dir, args.ignore): # Walk directories
+
+        if args.single_toc is False:
+            json_items = [] # new list for each directory
+
+        for filename in os.listdir(directory): # for each directory
+            full_path = os.path.join(directory, filename)
+            if os.path.isfile(full_path) is False:
+                continue
+            else:
+                # index.md is ignored as we expect this to be the page for the table of contents
+                if (filename.endswith(".md") or filename.endswith(".mdx")) and filename != "index.md":
+                    result = extract_title_description_slug(full_path)
+                    if result is not None:
+                        json_items.append(result)
+                        if args.single_toc is False:
+                            # don't write toc.json for empty folders
+                            if len(json_items) != 0:
+                                json_items = sorted(json_items, key=lambda x: x.get("title"))
+                                # output to the specified directory if arg --out is provided
+                                write_file(json_items, args, directory)
+                            else:
+                                print("Ran into an issue trying to extract YAML: empty result")
+
+        if args.single_toc is True:
+            # don't write toc.json for empty folders
+            if len(json_items) != 0:
+                json_array = sorted(json_items, key=lambda x: x.get("title"))
+                # output to the specified directory if arg --out is provided
+                write_file(json_items, args, directory)
+                sys.exit(0)
+            else:
+                sys.exit(1)
+
+if __name__ == "__main__":
+    main()
+
diff --git a/table-of-contents-files/docs_en_interfaces_formats_toc.json b/table-of-contents-files/docs_en_interfaces_formats_toc.json
