Merge pull request #3275 from Blargian/autogen-toc

Generate table of contents file for system settings

Author: Blargian

contribute/contrib-writing-guide.md

@@ -233,7 +233,7 @@ Backup and restore are very different across Cloud and Self-managed as the targe
 ---
 sidebar_label: Backups
 sidebar_position: 1
-slug: /en/manage/backups
+slug: /manage/backups
 ---
 
 import Tabs from '@theme/Tabs';
@@ -261,7 +261,7 @@ Sometimes most of the content is the same across operating systems.  For example
 
 ```md
 ---
-slug: /en/integrations/sql-clients/clickhouse-client-local
+slug: /integrations/sql-clients/clickhouse-client-local
 sidebar_label: clickhouse-client
 title: clickhouse-client and clickhouse-local
 ---
@@ -548,7 +548,7 @@ If a search is not finding the page that you expect, then have a look at the Mar
 
 ```md
 ---
-slug: /en/sql-reference/statements/create/function
+slug: /sql-reference/statements/create/function
 sidebar_position: 38
 sidebar_label: FUNCTION
 ---
@@ -562,7 +562,7 @@ And this was the change to improve the search results (add the expected search t
 
 ```md
 ---
-slug: /en/sql-reference/statements/create/function
+slug: /sql-reference/statements/create/function
 sidebar_position: 38
 sidebar_label: FUNCTION
 ---

docs/guides/sre/index.md

@@ -0,0 +1,10 @@
+---
+slug: /security-and-authentication
+title: "Security and Authentication"
+description: "Landing page for Security and Authentication"
+---
+
+| Page                                                                   | Description                                                                                             |
+|------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|
+| [Users and Roles](/docs/operations/access-rights)                   | Learn more about how ClickHouse supports access control management based on RBAC approach.              |
+| [External Authenticators](/docs/operations/external-authenticators) | Learn more about how OSS ClickHouse supports authenticating and managing users using external services. |

docs/migrations/index.md

@@ -1,5 +1,5 @@
 ---
-slug: en/migrations
+slug: migrations
 title: Migrations
 pagination_prev: null
 pagination_next: null

docs/migrations/postgres/index.md

@@ -1,5 +1,5 @@
 ---
-slug: en/migrations/postgresql
+slug: migrations/postgresql
 pagination_prev: null
 pagination_next: null
 title: PostgreSQL

package.json

@@ -7,7 +7,7 @@
     "autogen_needed_files": "src/Core/FormatFactorySettings.h src/Core/Settings.cpp CHANGELOG.md"
   },
   "scripts": {
-    "build": "yarn copy-clickhouse-repo-docs && yarn run-markdown-linter && yarn generate-changelog && yarn autogenerate-settings && yarn build-api-doc && yarn build-swagger && docusaurus build",
+    "build": "yarn copy-clickhouse-repo-docs && yarn run-markdown-linter && yarn generate-changelog && yarn autogenerate-settings && yarn autogenerate-table-of-contents && yarn build-api-doc && yarn build-swagger && GENERATE_SOURCEMAP=false docusaurus build",
     "clear": "docusaurus clear && bash ./placeholderReset.sh",
     "deploy": "docusaurus deploy",
     "docusaurus": "docusaurus",
@@ -17,6 +17,7 @@
     "build-swagger": "npx @redocly/cli build-docs https://api.clickhouse.cloud/v1 --output build/cloud/manage/api/swagger.html",
     "prep-from-local": "bash ./scripts/copy-clickhouse-repo-docs.sh -l $1",
     "autogenerate-settings": "bash ./scripts/settings/autogenerate-settings.sh",
+    "autogenerate-table-of-contents": "bash ./scripts/autogenerate-table-of-contents.sh",
     "generate-changelog": "bash ./scripts/generate-changelog.sh",
     "start": "docusaurus start",
     "swizzle": "docusaurus swizzle",

scripts/autogenerate-table-of-contents.sh

@@ -16,7 +16,18 @@ pip install -r scripts/table-of-contents-generator/requirements.txt
 
 # Add runs of the script below for any table of contents files that need to be generated
 # You can run toc_gen.py --help for descriptions of the parameters
+
+# TO DO: add a batch mode option to the script so that it takes in a file with the arguments
+#        instead of having to repeat the script each time
+
 python3 scripts/table-of-contents-generator/toc_gen.py --dir="knowledgebase" --single-toc --out="static" --ignore images
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/operations/system-tables" --md="docs/operations/system-tables/index.md"
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/operations/settings" --md="docs/operations/settings/index.md"
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/engines/database-engines" --md="docs/engines/database-engines/index.md"
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/engines/table-engines/mergetree-family" --md="docs/engines/table-engines/mergetree-family/index.md"
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/engines/table-engines/integrations" --md="docs/engines/table-engines/integrations/index.md"
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/engines/table-engines/special" --md="docs/engines/table-engines/special/index.md"
+python3 scripts/table-of-contents-generator/toc_gen.py --single-toc --dir="docs/getting-started/example-datasets" --md="docs/getting-started/index.md" --ignore images
 
 deactivate
 rm -r venv

scripts/table-of-contents-generator/toc_gen.py

@@ -11,6 +11,7 @@
 import sys
 from collections import defaultdict
 import yaml
+import re
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
@@ -27,6 +28,11 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="Path to output the resulting table of contents file to (by default it is output to the provided directory - file is named according to --dir)"
     )
+    parser.add_argument(
+        "--md",
+        default=None,
+        help="Path to markdown file to append the table of contents to"
+    )
     parser.add_argument(
         "--dir",
         help="Path to a folder containing markdown (.md, .mdx) documents containing YAML with title, description, slug."
@@ -69,10 +75,35 @@ def extract_title_description_slug(filename):
 def walk_dirs(root_dir, ignore_dirs=[]):
     for root, dirs, files in os.walk(root_dir):
         # Modify the 'dirs' list in-place to remove ignored directories
-        dirs[:] = [d for d in dirs if d not in ignore_dirs
-                   and not any(d.startswith(ig) for ig in ignore_dirs)]
+        if (ignore_dirs is not None):
+            dirs[:] = [d for d in dirs if d not in ignore_dirs
+                    and not any(d.startswith(ig) for ig in ignore_dirs)]
         yield root
 
+def write_md_to_file(json_items, path_to_md_file):
+    try:
+        with open(path_to_md_file, encoding='utf-8') as pre_check:
+            existing_content = pre_check.read()
+            if "| Page | Description |" in existing_content:
+                print(f"Markdown table already exists in {path_to_md_file}. Skipping.")
+                return
+
+        with open(path_to_md_file, 'a', encoding='utf-8') as f:
+
+            f.write("| Page | Description |\n")
+            f.write("|-----|-----|\n")
+
+            for item in json_items:
+                title = item.get('title', '')
+                slug = item.get('slug', '')
+                description = item.get('description','')
+                link = f"[{title}](/docs{slug})" if slug else title
+                f.write(f"| {link} | {description} |\n")
+
+        print(f"Markdown table appended to {path_to_md_file}")
+
+    except Exception as e:
+        print(f"An error occurred: {e}")
 def write_to_file(json_items, directory, output=None):
 
     if output is not None:
@@ -94,10 +125,21 @@ def write_to_file(json_items, directory, output=None):
             print(f"An error occurred creating directory: {e}")
 def write_file(json_items, args, directory):
     print(args)
-    if args.out is not None:
+    if (args.out is not None) and (args.md is None):
         write_to_file(json_items, directory+"/toc.json", args.out)
-    elif args.out is None:
+    elif (args.out is None) and (args.md is None):
         write_to_file(json_items, directory+"/toc.json")
+    elif (args.out is None) and (args.md is not None):
+        write_md_to_file(json_items, args.md)
+def sort_by_title_before_underscore(json_items):
+    def sort_key(item):
+        title = item.get("title", "")
+        if "_" in title:
+            return title.lower().split("_")[0]  # Sort by part before underscore
+        else:
+            return title.lower()  # Sort by whole title if no underscore
+
+    return sorted(json_items, key=sort_key)
 
 def main():
 
@@ -131,7 +173,7 @@ def main():
                         if args.single_toc is False:
                             # don't write toc.json for empty folders
                             if len(json_items) != 0:
-                                json_items = sorted(json_items, key=lambda x: x.get("title"))
+                                json_items = sort_by_title_before_underscore(json_items)
                                 # output to the specified directory if arg --out is provided
                                 write_file(json_items, args, directory)
                             else:
@@ -140,7 +182,7 @@ def main():
         if args.single_toc is True:
             # don't write toc.json for empty folders
             if len(json_items) != 0:
-                json_array = sorted(json_items, key=lambda x: x.get("title"))
+                json_array = sort_by_title_before_underscore(json_items)
                 # output to the specified directory if arg --out is provided
                 write_file(json_items, args, directory)
                 sys.exit(0)

sidebars.js

@@ -149,8 +149,8 @@ const sidebars = {
       className: "top-nav-item",
       collapsed: true,
       collapsible: true,
+      link: {type: "doc", id: "getting-started/index"},
       items: [
-        "getting-started/index",
         "getting-started/example-datasets/amazon-reviews",
         "getting-started/example-datasets/amplab-benchmark",
         "getting-started/example-datasets/brown-benchmark",
@@ -508,13 +508,14 @@ const sidebars = {
       label: "Functions",
       collapsed: false,
       collapsible: false,
+      link: {type: "doc", id: "sql-reference/functions/index"},
       items: [
         {
           type: "category",
           label: "Regular Functions",
           collapsed: true,
           collapsible: true,
-          link: {type: "doc", id: "sql-reference/functions/index"},
+          link: {type: "doc", id: "sql-reference/functions/regular-functions-index"},
           items: [
             {
               type: "autogenerated",
@@ -1359,6 +1360,7 @@ const sidebars = {
       label: "Security and Authentication",
       collapsed: false,
       collapsible: false,
+      link: {type: "doc", id: "guides/sre/index"},
       items: [
         "guides/sre/user-management/index",
         {
@@ -1576,7 +1578,7 @@ const sidebars = {
           type: "link",
           label: "Migration Guides",
           description: "Migrate your database to ClickHouse",
-          href: "/migrations/en/migrations"
+          href: "/migrations/migrations"
         },
         {
           type: "link",
@@ -1699,13 +1701,13 @@ const sidebars = {
           type: "link",
           label: "Deployments and Scaling",
           description: "How to deploy ClickHouse",
-          href: "/architecture/introduction"
+          href: "/deployment-guides/index"
         },
         {
           type: "link",
           label: "Security and Authentication",
           description: "Secure your ClickHouse deployment",
-          href: "/operations/external-authenticators"
+          href: "/security-and-authentication"
         },
         {
           type: "link",