Automatic checking of glossary terms on CI. Warning for now

Author: dhtclk

.github/workflows/check-build.yml

@@ -16,15 +16,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        check_type: [spellcheck, kbcheck, md-lint]
+        check_type: [spellcheck, kbcheck, md-lint, glossary-check]
     steps:
       # Add setup steps per check here
       - uses: actions/checkout@v4
       - name: Install Aspell
         if: matrix.check_type == 'spellcheck'
         run: sudo apt-get update && sudo apt-get install -y aspell aspell-en
       - name: Set up Python
-        if: matrix.check_type == 'kbcheck'
+        if: matrix.check_type == 'kbcheck' || matrix.check_type == 'glossary-check'
         run: |
           curl -Ls https://astral.sh/uv/install.sh | sh
           uv clean
@@ -51,6 +51,12 @@ jobs:
           elif [[ "${{ matrix.check_type }}" == "md-lint" ]]; then
             yarn check-markdown
           exit_code=$?
+          elif [[ "${{ matrix.check_type }}" == "glossary-check" ]]; then
+            echo "Extracting glossary from markdown..."
+            python3 scripts/glossary/extract_glossary.py
+            echo "Checking glossary coverage..."
+            python3 scripts/glossary/wrap_glossary_terms.py --check || echo "::warning::Glossary check found unwrapped terms (non-blocking)"
+            exit_code=0  # Always succeed for glossary check
           fi
           
           if [[ $exit_code -ne 0 ]]; then
@@ -74,5 +80,4 @@ jobs:
         if: needs.stylecheck.result != 'success'
         run: |
           echo "::error::One or more checks of the style check failed."
-          exit 1
-          
+          exit 1

scripts/wrap-glossary-terms/README.md renamed to scripts/glossary/README.md

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import re
+import json
+import argparse
+
+def extract_glossary_terms(markdown_content):
+    """Extract terms and definitions from ClickHouse glossary markdown"""
+    
+    # Pattern to match: ## Term {#anchor} followed by definition paragraph(s)
+    pattern = r'^## ([^{]+?)\s*\{#[^}]+\}\s*\n\n(.*?)(?=\n## |\Z)'
+    
+    matches = re.findall(pattern, markdown_content, re.MULTILINE | re.DOTALL)
+    
+    glossary = {}
+    
+    for term, definition in matches:
+        # Clean up the term
+        term = term.strip()
+        
+        # Clean up the definition
+        definition = definition.strip()
+        
+        # Remove extra whitespace and normalize line breaks
+        definition = re.sub(r'\n+', ' ', definition)
+        definition = re.sub(r'\s+', ' ', definition)
+        
+        glossary[term] = definition
+    
+    return glossary
+
+def main():
+    parser = argparse.ArgumentParser(description='Convert ClickHouse glossary.md to JSON')
+    parser.add_argument('--input', '-i', default='./docs/concepts/glossary.md',
+                       help='Input markdown file')
+    parser.add_argument('--output', '-o', default='./src/components/GlossaryTooltip/glossary.json',
+                       help='Output JSON file')
+    
+    args = parser.parse_args()
+    
+    # Read the markdown file
+    try:
+        with open(args.input, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except FileNotFoundError:
+        print(f"❌ Input file not found: {args.input}")
+        return
+    except Exception as e:
+        print(f"❌ Error reading file: {e}")
+        return
+    
+    # Extract glossary terms
+    glossary = extract_glossary_terms(content)
+    
+    if not glossary:
+        print("❌ No glossary terms found")
+        return
+    
+    print(f"✅ Extracted {len(glossary)} terms:")
+    for term in sorted(glossary.keys()):
+        print(f"   - {term}")
+    
+    # Write JSON file
+    try:
+        with open(args.output, 'w', encoding='utf-8') as f:
+            json.dump(glossary, f, indent=2, ensure_ascii=False)
+        
+        print(f"💾 Saved to: {args.output}")
+        
+    except Exception as e:
+        print(f"❌ Error writing JSON file: {e}")
+
+if __name__ == '__main__':
+    main()

scripts/glossary/extract-glossary-terms.py

@@ -173,10 +173,12 @@ def main():
     parser.add_argument('--glossary', default='./src/components/GlossaryTooltip/glossary.json', help='Glossary JSON file')
     parser.add_argument('--dry-run', action='store_true', help='Show changes without writing files')
     parser.add_argument('--force', action='store_true', help='Process files even if they already have glossary syntax')
+    parser.add_argument('--check', action='store_true', help='Check for unwrapped terms and show warnings (non-blocking)')
 
     args = parser.parse_args()
 
-    print("🚀 Starting Glossary Term Wrapper...\n")
+    if not args.check:
+        print("🚀 Starting Glossary Term Wrapper...\n")
 
     # Load glossary
     if not os.path.exists(args.glossary):
@@ -192,45 +194,67 @@ def main():
     # Filter out skip patterns
     files = [f for f in all_files if not should_skip_file(f)]
 
-    print(f"📁 Found {len(all_files)} MDX files, processing {len(files)} files")
-    print(f"⏭️  Skipped {len(all_files) - len(files)} files based on skip patterns")
-    
-    if args.force:
-        print("💪 FORCE MODE - Processing files even with existing glossary syntax")
-    
-    if args.dry_run:
-        print("🔍 DRY RUN MODE - No files will be modified")
-    
-    print()
+    if not args.check:
+        print(f"📁 Found {len(all_files)} MDX files, processing {len(files)} files")
+        print(f"⏭️  Skipped {len(all_files) - len(files)} files based on skip patterns")
+        
+        if args.force:
+            print("💪 FORCE MODE - Processing files even with existing glossary syntax")
+        
+        if args.dry_run:
+            print("🔍 DRY RUN MODE - No files will be modified")
+        
+        print()
 
     # Process files
     stats = {'modified': 0, 'unchanged': 0, 'skipped': 0, 'error': 0, 'terms_wrapped': 0}
+    file_details = []  # Track which files had terms for warning display
 
     for file_path in files:
         rel_path = os.path.relpath(file_path, args.docs_dir)
-        status, changes = process_file(file_path, terms, args.dry_run, args.force)
+        # For check mode, always use dry_run=True to avoid writing files
+        status, changes = process_file(file_path, terms, args.dry_run or args.check, args.force)
 
-        if status == 'modified':
-            print(f"✅ Modified {rel_path} ({changes} terms)")
-        elif status == 'unchanged':
+        if status == 'modified' and changes > 0:
+            file_details.append((rel_path, changes))
+            if not args.check:
+                print(f"✅ Modified {rel_path} ({changes} terms)")
+        elif status == 'unchanged' and not args.check:
             print(f"➖ No changes needed for {rel_path}")
-        elif status == 'skipped':
+        elif status == 'skipped' and not args.check:
             print(f"⏭️  Skipped {rel_path} (already has glossary syntax)")
 
         stats[status] += 1
         stats['terms_wrapped'] += changes
 
-    # Print summary
-    print(f"\n📊 Summary:")
-    print(f"   Files processed: {stats['modified'] + stats['unchanged']}")
-    print(f"   Files modified: {stats['modified']}")
-    print(f"   Files skipped: {stats['skipped']}")
-    print(f"   Terms wrapped: {stats['terms_wrapped']}")
-    
-    if args.dry_run:
-        print("\n💡 Run without --dry-run to apply changes")
-    if not args.force and stats['skipped'] > 0:
-        print("💡 Use --force to process files with existing glossary syntax")
+    # Show results
+    if args.check:
+        # Check mode: show warning if terms found
+        if stats['terms_wrapped'] > 0:
+            print(f"⚠️  GLOSSARY WARNING: Found {stats['terms_wrapped']} unwrapped glossary terms in {len(file_details)} files")
+            print("💡 Run 'python3 scripts/wrap_glossary_terms.py' to add glossary tooltips")
+            
+            # Show files with opportunities (limit to top 10 to avoid spam)
+            if file_details:
+                print("   Files with unwrapped terms:")
+                for rel_path, count in sorted(file_details, key=lambda x: x[1], reverse=True)[:10]:
+                    print(f"   - {rel_path} ({count} terms)")
+                if len(file_details) > 10:
+                    print(f"   ... and {len(file_details) - 10} more files")
+        else:
+            print("✅ All glossary terms are properly wrapped")
+    else:
+        # Normal mode: show detailed summary
+        print(f"\n📊 Summary:")
+        print(f"   Files processed: {stats['modified'] + stats['unchanged']}")
+        print(f"   Files modified: {stats['modified']}")
+        print(f"   Files skipped: {stats['skipped']}")
+        print(f"   Terms wrapped: {stats['terms_wrapped']}")
+        
+        if args.dry_run:
+            print("\n💡 Run without --dry-run to apply changes")
+        if not args.force and stats['skipped'] > 0:
+            print("💡 Use --force to process files with existing glossary syntax")
 
 if __name__ == '__main__':
     main()