Apply suggestions from Copilot and Gemini review

Author: josh-wong

.github/workflows/deploy.yml

@@ -23,6 +23,9 @@ jobs:
 
       - name: Install dependencies
         run: npm ci
+      - name: Install Python dependencies
+        run: |
+          python3 -m pip install --user gitingest
       - name: Build website
         run: npm run build
 

.github/workflows/test-deploy.yml

@@ -23,5 +23,7 @@ jobs:
 
       - name: Install dependencies
         run: npm ci
+      - name: Install Python dependencies
+        run: python3 -m pip install --user gitingest
       - name: Test build website
         run: npm run build

package.json

@@ -12,7 +12,7 @@
     "serve": "docusaurus serve",
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
-    "generate-llms-full": "(.venv/bin/python scripts/generate-llms-full.py) || (python3 scripts/generate-llms-full.py) || (python scripts/generate-llms-full.py)"
+    "generate-llms-full": "python3 scripts/generate-llms-full.py"
   },
   "dependencies": {
     "@docusaurus/core": "^3.7.0",

scripts/README.md

@@ -31,18 +31,21 @@ python scripts/generate-llms-full.py
 ### Requirements
 
 - Python 3.8+
-- gitingest package (automatically installed by using `pip` if not present)
+- gitingest package
+
+> [!NOTE]
+>
+> For local development, install gitingest manually by using `pip install --user gitingest` or `pipx install gitingest`. For GitHub Actions, gitingest is automatically installed in the workflow for building and deploying the docs site at `.github/workflows/deploy.yml`.
 
 ### What the `generate-llms-full.py` script does
 
 1. Uses gitingest to analyze the `docs` directory.
-2. Automatically installs gitingest if not already available.
-3. Includes only .mdx documentation files (`docs/*.mdx`, `docs/**/*.mdx`, and `src/components/en-us`).
-4. Focuses on the latest version of English documentation.
-5. Excludes build artifacts, node_modules, and other irrelevant files.
-6. Generates a comprehensive AI-friendly text digest.
-7. Adds a custom header for ScalarDB documentation context.
-8. Outputs to `build/llms-full.txt`.
+2. Includes only .mdx documentation files (`docs/*.mdx`, `docs/**/*.mdx`, and `src/components/en-us`).
+3. Focuses on the latest version of English documentation.
+4. Excludes build artifacts, node_modules, and other irrelevant files.
+5. Generates a comprehensive AI-friendly text digest.
+6. Adds a custom header for ScalarDB documentation context.
+7. Outputs to `build/llms-full.txt`.
 
 ### Configuration
 

scripts/generate-llms-full.py

@@ -4,49 +4,20 @@
 """
 
 import asyncio
-import os
-import subprocess
 import sys
+import textwrap
 from pathlib import Path
 
-# Add the project root to the Python path.
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-def install_gitingest():
-    """Install gitingest package using pip."""
-    try:
-        print("📦 Installing gitingest...")
-        subprocess.check_call([sys.executable, "-m", "pip", "install", "gitingest"])
-        print("✅ gitingest installed successfully")
-        return True
-    except subprocess.CalledProcessError as e:
-        print(f"❌ Failed to install gitingest: {e}")
-        return False
-
-# Try to import gitingest, install if not available
 try:
     from gitingest import ingest_async
 except ImportError:
-    print("⚠️ gitingest not found, attempting to install...")
-    if install_gitingest():
-        try:
-            from gitingest import ingest_async
-            print("✅ gitingest imported successfully after installation")
-        except ImportError:
-            print("❌ Failed to import gitingest even after installation")
-            print("Manual installation may be required:")
-            print("  pip install gitingest")
-            print("  # or")
-            print("  pipx install gitingest")
-            sys.exit(1)
-    else:
-        print("❌ gitingest installation failed")
-        print("Manual installation required:")
-        print("  pip install gitingest")
-        print("  # or")
-        print("  pipx install gitingest")
-        sys.exit(1)
+    print("❌ gitingest not found. Please install it first:")
+    print("  pip install --user gitingest")
+    print("  # or")
+    print("  pipx install gitingest")
+    print("")
+    print("For GitHub Actions, this should be installed automatically in the workflow.")
+    sys.exit(1)
 
 
 async def generate_llms_full():
@@ -72,19 +43,20 @@ async def generate_llms_full():
         # Generate content by using gitingest.
         summary, tree, content = await ingest_async(
             str(repo_path),
-            max_file_size=102400, # 100KB max file size
+            max_file_size=100000, # 100 KB max file size
             include_patterns=include_patterns,
             exclude_patterns=exclude_patterns,
             include_gitignored=False
         )
 
         # Create a header that matches your current format.
-        header = """# ScalarDB Documentation - Full Repository Context
-        # Generated by using GitIngest for AI/LLM consumption
-        # Cloud-native universal transaction manager
-        # Website: https://scalardb.scalar-labs.com
+        header = textwrap.dedent("""\
+            # ScalarDB Documentation - Full Repository Context
+            # Generated by using GitIngest for AI/LLM consumption
+            # Cloud-native universal transaction manager
+            # Website: https://scalardb.scalar-labs.com
 
-        """
+            """)
 
         # Combine all sections.
         full_content = header + summary + "\n\n" + tree + "\n\n" + content