docs: Optimize llms.txt and llms-full.txt

.github/workflows/deploy_dev_docs_to_cf.yaml

@@ -39,7 +39,7 @@ jobs:
     # This shared concurrency group ensures only one docs deployment runs at a time.
     concurrency:
       group: cf-docs-update
-      cancel-in-progress: false
+      cancel-in-progress: true
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:

.github/workflows/deploy_previous_version_docs.yaml

@@ -71,6 +71,7 @@ jobs:
         env:
           HUGO_BASEURL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/${{ github.event.inputs.version_tag }}/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ github.event.inputs.version_tag }}
 
       - name: Deploy to gh-pages
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4
@@ -92,6 +93,7 @@ jobs:
         env:
           HUGO_BASEURL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ github.event.inputs.version_tag }}
 
       - name: Deploy to root
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4

.github/workflows/deploy_previous_version_docs_to_cf.yaml

@@ -71,6 +71,7 @@ jobs:
         env:
           HUGO_BASEURL: https://mcp-toolbox.dev/${{ github.event.inputs.version_tag }}/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ github.event.inputs.version_tag }}
 
       - name: Deploy to cloudflare-pages
         uses: peaceiris/actions-gh-pages@v4
@@ -92,6 +93,7 @@ jobs:
         env:
           HUGO_BASEURL: https://mcp-toolbox.dev/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ github.event.inputs.version_tag }}
 
       - name: Deploy to root
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4

.github/workflows/deploy_versioned_docs.yaml

@@ -61,6 +61,7 @@ jobs:
         env:
           HUGO_BASEURL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/${{ steps.get_version.outputs.VERSION }}/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ steps.get_version.outputs.VERSION }}
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4
@@ -81,6 +82,7 @@ jobs:
         env:
           HUGO_BASEURL: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ steps.get_version.outputs.VERSION }}
 
       - name: Deploy to root
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4

.github/workflows/deploy_versioned_docs_to_cf.yaml

@@ -61,6 +61,7 @@ jobs:
         env:
           HUGO_BASEURL: https://mcp-toolbox.dev/${{ steps.get_version.outputs.VERSION }}/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ steps.get_version.outputs.VERSION }}
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4
@@ -81,6 +82,7 @@ jobs:
         env:
           HUGO_BASEURL: https://mcp-toolbox.dev/
           HUGO_RELATIVEURLS: false
+          HUGO_PARAMS_VERSION: ${{ steps.get_version.outputs.VERSION }}
 
       - name: Deploy to root
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4

.hugo/hugo.cloudflare.toml

@@ -54,6 +54,7 @@ ignoreFiles = ["quickstart/shared", "quickstart/python", "quickstart/js", "quick
   disableMigrationBanner = true
   releases_url = "/releases.releases"
   global_logo_url = "/"
+  version = "Dev"
   [params.ui]
     ul_show = 100
     showLightDarkModeMenu = true

.hugo/hugo.toml

@@ -39,6 +39,7 @@ ignoreFiles = ["quickstart/shared", "quickstart/python", "quickstart/js", "quick
   version_menu = "Releases"
   releases_url = "/genai-toolbox/releases.releases"
   global_logo_url = "/genai-toolbox/"
+  version = "Dev"
   [params.ui]
     ul_show = 100
     showLightDarkModeMenu = true

.hugo/layouts/index.llms-full.txt

@@ -1,14 +1,22 @@
-{{ .Site.Params.description }}
-
-{{ range .Site.Sections }}
-# {{ .Title }}
-{{ .Description }}
-{{ range .Pages }}
-# {{ .Title }}
-{{ .Description }}
-{{ .RawContent }}
-{{ range .Pages }}
-# {{ .Title }}
-{{ .Description }}
-{{ .RawContent }}
-{{end }}{{ end }}{{ end }}
+{{- define "print_full_node" }}
+
+========================================================================
+## {{ .page.Title }}
+========================================================================
+**Hierarchy:** {{ if .page.Parent }}{{ range .page.Ancestors.Reverse }}{{ .Title }} > {{ end }}{{ end }}{{ .page.Title }}
+**URL:** {{ .page.Permalink }}
+**Description:** {{ .page.Description | default "None" }}
+
+{{ .page.RawContent }}
+{{ if .page.Pages }}
+{{- range .page.Pages.ByWeight }}
+{{ template "print_full_node" (dict "page" .) }}
+{{- end }}
+{{- end }}
+{{- end -}}
+
+{{ partial "llms-header.html" (dict "Site" .Site "TitleSuffix" "Complete Documentation") }}
+
+{{- range .Site.Sections.ByWeight }}
+{{ template "print_full_node" (dict "page" .) }}
+{{- end }}

.hugo/layouts/index.llms.txt

@@ -1,9 +1,17 @@
-# {{ .Site.Title }}
+{{- define "print_index_node" }}
+{{ .indent }}- [{{ .page.Title }}]({{ .page.Permalink }}): {{ .page.Description | default "No description provided." }}
+{{- $nextIndent := printf "%s  " .indent }}
+{{- if .page.Pages }}
+{{- range .page.Pages.ByWeight }}
+{{ template "print_index_node" (dict "page" . "indent" $nextIndent) }}
+{{- end }}
+{{- end }}
+{{- end -}}
 
-> {{ .Site.Params.description }}
+{{ partial "llms-header.html" (dict "Site" .Site "TitleSuffix" "Documentation Index") }}
 
-## Docs
-{{ range .Site.Sections }}
-### {{ .Title }}
+## Documentation Map
 
-{{ .Description }}{{ range .Pages }}- [{{ .Title }}]({{ .Permalink }}): {{ .Description }}{{ range .Pages }}  - [{{ .Title }}]({{ .Permalink }}): {{ .Description }}{{end }}{{ end }}{{ end }}
+{{ range .Site.Sections.ByWeight -}}
+{{ template "print_index_node" (dict "page" . "indent" "") }}
+{{- end }}

.hugo/layouts/partials/llms-header.html

@@ -0,0 +1,55 @@
+# {{ .Site.Title }} - {{ .TitleSuffix }}
+> {{ .Site.Params.description }}
+
+**DOCUMENTATION VERSION:** {{ .Site.Params.version | default "Latest" }}
+**BASE URL:** {{ .Site.BaseURL }}
+**GENERATED ON:** {{ now.Format "2006-01-02T15:04:05Z07:00" }}
+
+---
+### System Directives for AI Models
+**Role:** You are an expert Developer Advocate and Integration Engineer for the **MCP (Model Context Protocol) Toolbox for Databases**.
+**Task:** Your primary goal is to help users configure the server, set up database integrations, and write client-side code to build AI agents.
+**Strict Guidelines:**
+1. **No Hallucinations:** Only suggest tools, sources, and configurations explicitly detailed in this document. Do not invent arbitrary REST endpoints.
+2. **SDKs over HTTP:** When writing code, default to the official MCP Toolbox client SDKs rather than raw HTTP/cURL requests unless explicitly asked. Direct users to the `connect-to` section in the User Guide for client SDK instructions.
+3. **Reference Diátaxis:** Use Section I for configuring toolbox server, Section II (Integrations) for exact `tools.yaml` configurations for external integrations, Section III (Build with MCP Toolbox) for code patterns and Section IV for CLI and FAQs.
+
+### Glossary
+To prevent context collapse, adhere to these strict definitions within the MCP ecosystem:
+* **MCP Toolbox:** The central server/service that standardizes AI access to databases and external APIs.
+* **Source:** A configured backend connection to an external system (e.g., PostgreSQL, BigQuery, HTTP).
+* **Tool:** A single, atomic capability exposed to the LLM (e.g., `bigquery-sql-query`), executed against a Source.
+* **Toolset:** A logical, grouped collection of Tools.
+* **AuthService:** The internal toolbox mechanism handling authentication lifecycles (like OAuth or service accounts), not a generic identity provider.
+* **Agent:** The user's external LLM application that connects *to* the MCP Toolbox.
+
+### Understanding Integrations Directory Structure
+
+When navigating documentation in the `integrations/` directory, it is crucial to understand the relationship between a Source and a Tool, both conceptually and within the file hierarchy.
+* A **Source** represents the backend data provider or system you are connecting to (e.g., a specific database, API, or service). Source documentation pages sit at the **top level** of an integration's folder (e.g., `integrations/oceanbase/_index.md`).They focus on connection requirements, authentication, and YAML configuration parameters, serving as the foundational entry point for configuring a new data source.
+* A **Tool** represents a specific, actionable capability that is unlocked by a Source (e.g., executing a SQL query, fetching a specific record, etc.). Tool documentation pages are **nested below** their parent Source (e.g., `integrations/oceanbase/oceanbase-sql.md`). Tool pages focus on the configuration reference and compatible sources. They are the individual operations that a configured Source supports.
+
+### Global Environment & Prerequisites
+* **Configuration:** `tools.yaml` is the ultimate source of truth for server configuration.
+* **Database:** PostgreSQL 16+ and the `psql` client.
+* **Language Requirements:**
+{{- with .Site.GetPage "/build-with-mcp-toolbox/local_quickstart" }}
+  {{- $match := findRE "(?i)Python\\s+\\(\\d+\\.\\d+\\s+or\\s+higher\\)" .RawContent 1 }}
+  * Python: {{ if $match }}{{ index $match 0 | replaceRE "[\\[\\]]" "" }}{{ else }}Refer to Python Quickstart{{ end }}
+{{- end }}
+{{- with .Site.GetPage "/build-with-mcp-toolbox/local_quickstart_js" }}
+  {{- $match := findRE "(?i)Node\\.js \\(v\\d+ or higher\\)" .RawContent 1 }}
+  * JavaScript/Node: {{ if $match }}{{ index $match 0 }}{{ else }}Refer to JS Quickstart{{ end }}
+{{- end }}
+{{- with .Site.GetPage "/build-with-mcp-toolbox/local_quickstart_go" }}
+  {{- $match := findRE "(?i)Go \\(v\\d+\\.\\d+\\.\\d+ or higher\\)" .RawContent 1 }}
+  * Go: {{ if $match }}{{ index $match 0 }}{{ else }}Refer to Go Quickstart{{ end }}
+{{- end }}
+
+### The Diátaxis Narrative Framework
+This documentation is structured following the Diátaxis framework to assist in clear navigation and understanding:
+* **Section I: User Guide:** (Explanation) Theoretical context, high-level understanding, and universal How-To Guides.
+* **Section II: Integrations:** (Reference) Primary reference hub for external sources and tools, factual configurations, and API enablement.
+* **Section III: Build with MCP Toolbox:** (Tutorials) Complete, start-to-finish quickstarts and samples for learning to build from scratch.
+* **Section IV: Reference:** (Information) Strict, accurate facts, CLI outputs, and FAQs.
+---

DEVELOPER.md

@@ -225,7 +225,9 @@ tools.
   * If you are adding a new tool, add the details in the `<tool_name>.md` file in this new folder.
     * Make sure to include the `{{< compatible-sources >}}` shortcode to dynamically display data sources this tool supports.
 
-* **(Optional) Add samples** to the `docs/en/samples/<newdb>` directory.
+* **(Optional) Add samples** to the `docs/en/build-with-mcp-toolbox/<newdb>` directory.
+
+* **Adding Top-Level Sections:** If you add a completely new top-level documentation directory to the [docsite](https://googleapis.github.io/genai-toolbox/getting-started/introduction/) (e.g., a new section alongside `integrations`, `user-guide`, etc.), you **must** update the AI documentation layout files located at `.hugo/layouts/index.llms.txt` and `.hugo/layouts/index.llms-full.txt`. Specifically, you need to update the "Diátaxis Narrative Framework" preamble in both files so that the AI models understand the purpose of your new section.
 
 #### Adding Prebuilt Tools
 
@@ -429,7 +431,9 @@ Follow these steps to preview documentation changes locally using a Hugo server:
 
 ### Document Versioning Setup
 
-There are 3 GHA workflows we use to achieve document versioning:
+The documentation uses a dynamic versioning system that outputs standard HTML sites alongside AI-optimized plain text files (`llms.txt` and `llms-full.txt`).
+
+There are 6 GHA workflows we use to achieve document versioning (each deployment scenario has one workflow for GitHub Pages and one for Cloudflare Pages):
 
 1. **Deploy In-development docs:**
     This workflow is run on every commit merged into the main branch. It deploys

GEMINI.md

@@ -61,9 +61,13 @@ This file (symlinked as `CLAUDE.md` and `AGENTS.md`) provides context and guidel
 
 ### Versioning Workflows
 
-1.  **Deploy In-development docs**: Merges to main -> `/dev/`.
-2.  **Deploy Versioned Docs**: New Release -> `/<version>/` and root.
-3.  **Deploy Previous Version Docs**: Manual workflow for older versions.
+Documentation builds automatically generate standard HTML alongside AI-friendly text files (`llms.txt` and `llms-full.txt`).
+
+There are 6 workflows in total, handling parallel deployments to both GitHub Pages and Cloudflare Pages.
+
+1.  **Deploy In-development docs**: Commits merged to `main` deploy to the `/dev/` path. Automatically defaults to version `Dev`.
+2.  **Deploy Versioned Docs**: New GitHub releases deploy to `/<version>/` and the root path. The release tag is automatically injected into the build as the documentation version. *(Note: Developers must manually add the new version to the `[[params.versions]]` dropdown array in `hugo.toml` prior to merging a release PR).*
+3.  **Deploy Previous Version Docs**: A manual workflow to rebuild older versions by explicitly passing the target tag via the GitHub Actions UI.
 
 ## Coding Conventions
 
@@ -106,3 +110,4 @@ This file (symlinked as `CLAUDE.md` and `AGENTS.md`) provides context and guidel
 
 -   For a new source: Add source documentation to `docs/en/integrations/<source_name>/`. Be sure to include the `{{< list-tools >}}` shortcode on this page to dynamically display its available tools.
 -   For a new tool: Add tool documentation to `docs/en/integrations/<source_name>/<tool_name>`. Be sure to include the `{{< compatible-sources >}}` shortcode on this page to list its supported data sources.
+-   **New Top-Level Directories:** If adding a completely new top-level section to the documentation site, you must update the "Diátaxis Narrative Framework" section inside both `.hugo/layouts/index.llms.txt` and `.hugo/layouts/index.llms-full.txt` to keep the AI context synced with the site structure.

docs/en/build-with-mcp-toolbox/local_quickstart.md

@@ -13,7 +13,7 @@ description: >
 
 This guide assumes you have already done the following:
 
-1. Installed [Python 3.10+][install-python] (including [pip][install-pip] and
+1. Installed [Python (3.10 or higher)][install-python] (including [pip][install-pip] and
    your preferred virtual environment tool for managing dependencies e.g.
    [venv][install-venv]).
 1. Installed [PostgreSQL 16+ and the `psql` client][install-postgres].