Merge branch 'documentation-reorg' into final-changes

Author: dishaprakash

.ci/lint-docs-source-page.sh

@@ -2,16 +2,36 @@
 set -e
 
 python3 - << 'EOF'
+"""
+MCP TOOLBOX: SOURCE PAGE LINTER
+This script enforces a standardized structure for integration Source pages 
+(_index.md files). It ensures users can predictably find 
+information across all database integrations.
+
+MAINTENANCE GUIDE:
+------------------
+1. TO ADD A NEW HEADING: 
+   Add the exact heading text to the 'ALLOWED_ORDER' list in the desired 
+   sequence.
+
+2. TO MAKE A HEADING MANDATORY: 
+   Add the heading text to the 'REQUIRED' set.
+
+3. TO IGNORE NEW CONTENT TYPES:
+   Update the regex in the 'clean_body' variable (Step 3) to strip out 
+   Markdown before linting.
+
+4. SCOPE:
+   This script ignores top-level directory files and only targets 
+   integrations/{provider}/_index.md.
+"""
+
 import os
 import re
 import sys
 from pathlib import Path
 
-integration_dir = Path("./docs/en/integrations")
-if not integration_dir.exists():
-    print("Info: Directory './docs/en/integrations' not found. Skipping linting.")
-    sys.exit(0)
-
+# --- CONFIGURATION ---
 ALLOWED_ORDER = [
     "About",
     "Available Tools",
@@ -23,95 +43,78 @@ ALLOWED_ORDER = [
     "Additional Resources"
 ]
 REQUIRED = {"About", "Example", "Reference"}
-
-# Regex to catch any variation of the list-tools shortcode, including parameters
 SHORTCODE_PATTERN = r"\{\{<\s*list-tools.*?>\}\}"
+# ---------------------
+
+integration_dir = Path("./docs/en/integrations")
+if not integration_dir.exists():
+    print("Info: Directory './docs/en/integrations' not found. Skipping linting.")
+    sys.exit(0)
 
 has_errors = False
 
-# Find all _index.md files inside the subdirectories of integrations/
 for filepath in integration_dir.rglob("_index.md"):
-    # Skip the top-level integrations/_index.md if it exists
     if filepath.parent == integration_dir:
         continue
 
     with open(filepath, "r", encoding="utf-8") as f:
         content = f.read()
 
-    # Separate YAML frontmatter from the markdown body
     match = re.match(r'^\s*---\s*\n(.*?)\n---\s*(.*)', content, re.DOTALL)
     if match:
-        frontmatter = match.group(1)
-        body = match.group(2)
+        frontmatter, body = match.group(1), match.group(2)
     else:
-        frontmatter = ""
-        body = content
+        frontmatter, body = "", content
 
-    # If the file has no markdown content (metadata placeholder only), skip it entirely
     if not body.strip():
         continue
 
     file_errors = False
 
-    # 1. Check Frontmatter Title
-    title_source = frontmatter if frontmatter else content
-    title_match = re.search(r"^title:\s*[\"']?(.*?)[\"']?\s*$", title_source, re.MULTILINE)
+    # 1. Frontmatter Title Check
+    title_match = re.search(r"^title:\s*[\"']?(.*?)[\"']?\s*$", frontmatter if frontmatter else content, re.MULTILINE)
     if not title_match or not title_match.group(1).strip().endswith("Source"):
-        found_title = title_match.group(1) if title_match else "None"
-        print(f"[{filepath}] Error: Frontmatter title must end with 'Source'. Found: '{found_title}'")
+        print(f"[{filepath}] Error: Title must end with 'Source'.")
         file_errors = True
 
-    # 2. Check Shortcode Placement ONLY IF "Available Tools" heading is present
-    tools_section_match = re.search(r"^##\s+Available Tools\s*(.*?)(?=^##\s|\Z)", body, re.MULTILINE | re.DOTALL)
-    if tools_section_match:
-        if not re.search(SHORTCODE_PATTERN, tools_section_match.group(1)):
-            print(f"[{filepath}] Error: The list-tools shortcode must be placed under the '## Available Tools' heading.")
-            file_errors = True
-    else:
-        # Prevent edge case where shortcode is used but the heading was forgotten
-        if re.search(SHORTCODE_PATTERN, body):
-            print(f"[{filepath}] Error: A list-tools shortcode was found, but the '## Available Tools' heading is missing.")
+    # 2. Shortcode Placement Check
+    tools_section = re.search(r"^##\s+Available Tools\s*(.*?)(?=^##\s|\Z)", body, re.MULTILINE | re.DOTALL)
+    if tools_section:
+        if not re.search(SHORTCODE_PATTERN, tools_section.group(1)):
+            print(f"[{filepath}] Error: {{< list-tools >}} must be under '## Available Tools'.")
             file_errors = True
+    elif re.search(SHORTCODE_PATTERN, body):
+        print(f"[{filepath}] Error: {{< list-tools >}} found, but '## Available Tools' heading is missing.")
+        file_errors = True
 
-    # 3. Strip code blocks from body to avoid linting example markdown headings
+    # 3. Heading Linting (Stripping code blocks first)
     clean_body = re.sub(r"```.*?```", "", body, flags=re.DOTALL)
 
-    # 4. Check H1 Headings
     if re.search(r"^#\s+\w+", clean_body, re.MULTILINE):
-        print(f"[{filepath}] Error: H1 headings (#) are forbidden in the body.")
+        print(f"[{filepath}] Error: H1 (#) headings are forbidden in the body.")
         file_errors = True
 
-    # 5. Check H2 Headings
-    h2s = re.findall(r"^##\s+(.*)", clean_body, re.MULTILINE)
-    h2s = [h2.strip() for h2 in h2s]
+    h2s = [h.strip() for h in re.findall(r"^##\s+(.*)", clean_body, re.MULTILINE)]
 
-    # Missing Required
-    missing = REQUIRED - set(h2s)
-    if missing:
-        print(f"[{filepath}] Error: Missing required H2 headings: {missing}")
+    # 4. Required & Unauthorized Check
+    if missing := (REQUIRED - set(h2s)):
+        print(f"[{filepath}] Error: Missing required H2s: {missing}")
         file_errors = True
 
-    # Unauthorized Headings
-    unauthorized = set(h2s) - set(ALLOWED_ORDER)
-    if unauthorized:
-        print(f"[{filepath}] Error: Unauthorized H2 headings found: {unauthorized}")
+    if unauthorized := (set(h2s) - set(ALLOWED_ORDER)):
+        print(f"[{filepath}] Error: Unauthorized H2s found: {unauthorized}")
         file_errors = True
 
-    # Strict Ordering
-    filtered_h2s = [h for h in h2s if h in ALLOWED_ORDER]
-    expected_order = [h for h in ALLOWED_ORDER if h in h2s]
-    if filtered_h2s != expected_order:
-        print(f"[{filepath}] Error: Headings are out of order.")
-        print(f"  Expected: {expected_order}")
-        print(f"  Found:    {filtered_h2s}")
+    # 5. Order Check
+    if [h for h in h2s if h in ALLOWED_ORDER] != [h for h in ALLOWED_ORDER if h in h2s]:
+        print(f"[{filepath}] Error: Headings out of order. Reference: {ALLOWED_ORDER}")
         file_errors = True
 
-    if file_errors:
-        has_errors = True
+    if file_errors: has_errors = True
 
 if has_errors:
-    print("Linting failed. Please fix the structure errors above.")
+    print("Linting failed. Fix structure errors above.")
     sys.exit(1)
-else:
-    print("Success: All Source pages passed structure validation.")
-EOF
+print("Success: Source pages validated.")
+sys.exit(0)
+EOF

.ci/lint-docs-tool-page.sh

@@ -2,16 +2,34 @@
 set -e
 
 python3 - << 'EOF'
+"""
+MCP TOOLBOX: TOOL PAGE LINTER
+This script enforces a standardized structure for individual Tool pages 
+(e.g., integrations/postgres/postgres-sql.md). It ensures that LLM agents 
+can parse tool capabilities and parameter definitions reliably.
+
+MAINTENANCE GUIDE:
+------------------
+1. TO ADD A NEW HEADING: 
+   Add the exact heading text to the 'ALLOWED_ORDER' list in the desired 
+   sequence.
+
+2. TO MAKE A HEADING MANDATORY: 
+   Add the heading text to the 'REQUIRED' set.
+
+3. TO UPDATE SHORTCODE LOGIC:
+   If the shortcode name changes, update 'SHORTCODE_PATTERN'.
+
+4. SCOPE:
+   This script targets all .md files in integrations/ EXCEPT _index.md files.
+"""
+
 import os
 import re
 import sys
 from pathlib import Path
 
-integration_dir = Path("./docs/en/integrations")
-if not integration_dir.exists():
-    print("Info: Directory './docs/en/integrations' not found. Skipping linting.")
-    sys.exit(0)
-
+# --- CONFIGURATION ---
 ALLOWED_ORDER = [
     "About",
     "Compatible Sources",
@@ -25,9 +43,13 @@ ALLOWED_ORDER = [
     "Additional Resources"
 ]
 REQUIRED = {"About", "Example"}
-
-# Regex to catch any variation of the compatible-sources shortcode
 SHORTCODE_PATTERN = r"\{\{<\s*compatible-sources.*?>\}\}"
+# ---------------------
+
+integration_dir = Path("./docs/en/integrations")
+if not integration_dir.exists():
+    print("Info: Directory './docs/en/integrations' not found. Skipping linting.")
+    sys.exit(0)
 
 has_errors = False
 
@@ -48,7 +70,6 @@ for filepath in integration_dir.rglob("*.md"):
         frontmatter = ""
         body = content
 
-    # If the file has no markdown content (metadata placeholder only), skip it entirely
     if not body.strip():
         continue
 
@@ -66,35 +87,30 @@ for filepath in integration_dir.rglob("*.md"):
     sources_section_match = re.search(r"^##\s+Compatible Sources\s*(.*?)(?=^##\s|\Z)", body, re.MULTILINE | re.DOTALL)
     if sources_section_match:
         if not re.search(SHORTCODE_PATTERN, sources_section_match.group(1)):
-            print(f"[{filepath}] Error: The compatible-sources shortcode must be placed under the '## Compatible Sources' heading.")
-            file_errors = True
-    else:
-        # Prevent edge case where shortcode is used but the heading was forgotten
-        if re.search(SHORTCODE_PATTERN, body):
-            print(f"[{filepath}] Error: A compatible-sources shortcode was found, but the '## Compatible Sources' heading is missing.")
+            print(f"[{filepath}] Error: The compatible-sources shortcode must be placed under '## Compatible Sources'.")
             file_errors = True
+    elif re.search(SHORTCODE_PATTERN, body):
+        print(f"[{filepath}] Error: Shortcode found, but '## Compatible Sources' heading is missing.")
+        file_errors = True
 
-    # 3. Strip code blocks from body to avoid linting example markdown headings
-    clean_body = re.sub(r"```.*?```", "", body, flags=re.DOTALL)
+    # 3. Strip code blocks
+    clean_body = re.sub(r"^(?:```|~~~).*?^(?:```|~~~)", "", body, flags=re.DOTALL | re.MULTILINE)
 
     # 4. Check H1 Headings
     if re.search(r"^#\s+\w+", clean_body, re.MULTILINE):
         print(f"[{filepath}] Error: H1 headings (#) are forbidden in the body.")
         file_errors = True
 
     # 5. Check H2 Headings
-    h2s = re.findall(r"^##\s+(.*)", clean_body, re.MULTILINE)
-    h2s = [h2.strip() for h2 in h2s]
+    h2s = [h.strip() for h in re.findall(r"^##\s+(.*)", clean_body, re.MULTILINE)]
 
     # Missing Required
-    missing = REQUIRED - set(h2s)
-    if missing:
+    if missing := (REQUIRED - set(h2s)):
         print(f"[{filepath}] Error: Missing required H2 headings: {missing}")
         file_errors = True
 
     # Unauthorized Headings
-    unauthorized = set(h2s) - set(ALLOWED_ORDER)
-    if unauthorized:
+    if unauthorized := (set(h2s) - set(ALLOWED_ORDER)):
         print(f"[{filepath}] Error: Unauthorized H2 headings found: {unauthorized}")
         file_errors = True
 
@@ -115,4 +131,4 @@ if has_errors:
     sys.exit(1)
 else:
     print("Success: All Tool pages passed structure validation.")
-EOF
+EOF

.github/workflows/deploy_previous_version_docs.yaml

@@ -111,4 +111,4 @@ jobs:
           publish_branch: versioned-gh-pages
           keep_files: true
           allow_empty_commit: true
-          commit_message: "deploy: docs to root for ${{ github.event.inputs.version_tag }}"
+          commit_message: "deploy: docs to root for ${{ github.event.inputs.version_tag }}"

.github/workflows/deploy_previous_version_docs_to_cf.yaml

@@ -73,7 +73,7 @@ jobs:
           HUGO_RELATIVEURLS: false
           HUGO_PARAMS_VERSION: ${{ github.event.inputs.version_tag }}
 
-      - name: Build Pagefind Index (Archived Version)
+      - name: Build Pagefind Index (Root)
         run: npx pagefind --site public
         working-directory: .hugo
 

.github/workflows/docs_lint.yaml

@@ -23,7 +23,7 @@ on:
 
 jobs:
   lint-source-pages:
-    name: Validate Source Page Structure
+    name: Lint Documentation
     runs-on: ubuntu-latest
 
     steps:
@@ -35,26 +35,19 @@ jobs:
         with:
           python-version: '3.x'
 
-      - name: Check for large files in docs/
+      - name: Check for large files (>24MB)
         run: |
-          if [ -d "docs" ]; then
-            LARGE_FILES=$(find docs/ -type f -size +24M)
-            if [ -n "$LARGE_FILES" ]; then
-              echo "Error: The following files in the docs/ directory exceed the 24MB size limit:"
-              echo "$LARGE_FILES"
-              exit 1
-            else
-              echo "Success: No files in docs/ exceed 24MB."
-            fi
-          else
-            echo "Info: docs/ directory not found. Skipping file size check."
+          LARGE_FILES=$(find docs/ -type f -size +24M)
+          if [ -n "$LARGE_FILES" ]; then
+            echo "Error: Files exceed 24MB limit: $LARGE_FILES"
+            exit 1
           fi
 
       - name: Make scripts executable
         run: chmod +x .ci/lint-docs-*.sh
 
       - name: Run Structure Linter for Source Pages
-        run: .ci/lint-docs-source-page.sh
+        run: bash .ci/lint-docs-source-page.sh
 
       - name: Run Structure Linter for Tool Pages
-        run: .ci/lint-docs-tool-page.sh
+        run: bash .ci/lint-docs-tool-page.sh

.github/workflows/docs_preview_deploy_cf.yaml

@@ -12,23 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: "CF: Deploy Docs Preview"
-
-on:
-  workflow_run:
-    workflows: ["CF: Build Docs Preview"]
-    types:
-      - completed
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: 'PR Number to deploy (Manual override)'
-        required: true
-        type: string
-      build_run_id:
-        description: 'The Run ID from the successful "CF: Build Docs Preview" workflow'
-        required: true
-        type: string
+name: "CF: Deploy PR Preview"
 
 permissions:
   contents: read