DOC-13760 produce markdown per page (#863)

* DOC-13760 produce markup file per published page

* DOC-13760 test on staging chatbot

* DOC-13760 remove unneeded async

copilot review suggests this may introduce unneeded overhead.
As we're getting Node memory errors, then we may as well try!

* DOC-13760 save ONLY markdown summary

Reduce memory usage by *only* saving the Markdown version in this run.
We can do this as a separate build, and scp the output to the same bucket.

e.g. this content would be overlaid on the *standard* build (which would
only need to add the <link href="..."> URL feature.

Rewrite relative hrefs to .md target.

* DOC-13760 tidy up

fix page.pub.url to ensure that the site nav is also updated

* DOC-13760 create an initial llms.txt

* tidy

* tidy

* reduce memory usage by explicitly freeing DOM

* change markdown parsing dependency

Use the one used by https://github.com/cerbos/antora-llm-generator

* remove chunking, fix links

* workaround for mangled noopener links in nav

* Admonition handling

* fixes

Author: osfameron

antora-playbook-staging-chatbot.diff.yml

@@ -1,6 +1,7 @@
 antora:
   extensions:
   - ./lib/report-tree.js
+  - ./lib/markdown-for-llm.js
   - ./lib/site-stats-extension.js
   - ./lib/component-stats.js
 

antora-playbook-staging-chatbot.yml

@@ -23,6 +23,7 @@ antora:
     - ./lib/embargo.js
     - ./lib/antora-component-version-rank.js
     - ./lib/report-tree.js
+    - ./lib/markdown-for-llm.js
     - ./lib/site-stats-extension.js
     - ./lib/component-stats.js
 site:
@@ -36,7 +37,6 @@ site:
         { "title": "Server", "startPage": "home::server.adoc", "components": ["server", "enterprise-analytics"] },
         { "title": "Mobile / Edge", "startPage": "home::mobile.adoc", "components": ["couchbase-lite", "couchbase-lite-javascript", "sync-gateway", "couchbase-edge-server"] },
         { "title": "Capella", "startPage": "home::cloud.adoc", "components": ["cloud", "app-services", "ai", "analytics"] },
-        { "title": "Cloud-Native", "components": ["cloud-native-database"] },
         { "title": "Kubernetes Operator", "startPage": "operator::overview.adoc", "components": ["operator"] },
         { "title": "CMOS", "components": ["cmos"] },
         { "title": "Develop", "startPage": "home::developer.adoc",
@@ -160,9 +160,6 @@ content:
       start_path: docs
       branches:
         - master
-    - url: https://github.com/couchbase/docs-cloud-native
-      branches:
-        - cloud-native-2.2
     - url: https://github.com/couchbase/docs-elastic-search
       branches:
         - main
@@ -221,6 +218,7 @@ content:
         - release/7.6.2
         - release/7.6
         - release/7.2
+        - release/8.0.1
     - url: https://github.com/couchbase/docs-sdk-c
       branches:
         - release/3.3
@@ -247,9 +245,9 @@ content:
         - release/1.0
     - url: https://github.com/couchbase/docs-sdk-java
       branches:
+        - release/3.11
         - release/3.10
         - release/3.9
-        - release/3.8
     - url: https://github.com/couchbase/docs-quarkus-extension
       branches:
         - release/1.2
@@ -264,9 +262,9 @@ content:
         - release/1.2
     - url: https://github.com/couchbase/docs-sdk-scala
       branches:
+        - release/3.11
         - release/3.10
         - release/3.9
-        - release/1.8
     - url: https://github.com/couchbase/docs-sdk-nodejs
       branches:
         - temp/4.6

antora-playbook.preview.yml

@@ -23,6 +23,7 @@ antora:
     - ./lib/embargo.js
     - ./lib/antora-component-version-rank.js
     - ./lib/preview.js
+    - ./lib/markdown-for-llm.js
     - ./lib/report-tree.js
     - ./lib/site-stats-extension.js
     - ./lib/component-stats.js

lib/markdown-for-llm.js

@@ -0,0 +1,181 @@
+'use strict'
+
+const { NodeHtmlMarkdown } = require('node-html-markdown')
+const YAML = require('yaml')
+
+let nhm
+const customTranslators = {
+  DIV:  ({ visitor }) => ({
+    surroundingNewlines: 2,
+    postprocess: ({ nodeMetadata, node }) => {
+      // <div class="admonitionblock note">
+      if (node.classList.contains('admonitionblock')) {
+
+        const type = (node
+            .classList.values().
+            find(v => v != 'admonitionblock') || 'NOTE'
+            ).toUpperCase()
+
+        const content = (
+          nhm.translate(
+            node.querySelector('td:nth-child(2)')
+            .innerHTML
+          ).replace(/^/mg, '> '))
+
+        return `> [!${type}]\n${content}`
+      }
+    }
+  })
+}
+nhm = new NodeHtmlMarkdown({}, customTranslators)
+
+const File = require('vinyl')
+
+function markdownify(page, pubDate) {
+  const title = page.title
+  const html = page.contents.toString()
+  const markdown = nhm.translate(html)
+
+  // haha, we now have to translate the .md link back to .html just for this link
+  const orig = page.pub.url.replace(/\.md$/, '.html')
+
+  let output = 
+    `[View original HTML](${orig})\n\n` +
+    `# ${title}\n\n${markdown}`
+
+  const frontmatter = {
+    title,
+    description: page.asciidoc.attributes.description,
+    editUrl: page.src.editUrl,
+    pubDate,
+  }
+
+  output = `---\n${YAML.stringify(frontmatter)}---\n\n${output}`
+
+  page.contents = Buffer.from(output)
+}
+
+module.exports.register = function ({ playbook, config }) {
+  this.once('contextStarted', () => {
+    const { createPageComposer: _delegate } = this.getFunctions()
+
+    this.replaceFunctions({
+      // see https://gitlab.com/antora/antora/-/blob/v3.1.x/packages/page-composer/lib/create-page-composer.js
+      createPageComposer (playbook, contentCatalog, uiCatalog) {
+        function composePage (file, _contentCatalog, _navigationCatalog) {
+          // instead of wrapping the file in a layout, just
+          // return the file as-is
+          return file
+        }
+        const create404Page = (siteAsciiDocConfig) =>
+          composePage({
+            asciidoc: siteAsciiDocConfig,
+            mediaType: 'text/html',
+            out: { path: '404.html' },
+            pub: {},
+            src: { stem: '404' },
+            title: siteAsciiDocConfig?.attributes['404-page-title'] || 'PageNot Found',
+        })
+
+        const ret = Object.assign(composePage, {composePage, create404Page} )
+        return ret
+      }
+    })
+  })
+
+  this.once('contentClassified', async ({ playbook, contentCatalog, siteCatalog }) => {
+    const logger = this.getLogger('llm-summaries')
+
+    const pages = contentCatalog.getPages((page) => page.pub && page.out)
+
+    for (const page of pages) {
+        page.pub.url = page.pub.url.replace(/\.html$/, '.md')
+        page.out.path = page.out.path.replace(/\.html$/, '.md')
+    }
+  })
+
+  this.once('documentsConverted', async ({ playbook, contentCatalog, siteCatalog }) => {
+    const logger = this.getLogger('llm-summaries')
+
+    const pubDate = new Date().toISOString()
+
+    const pages = contentCatalog.getPages(
+      (page) =>
+        page.mediaType === 'text/html'
+        && page.pub
+        && page.out)
+
+    for (const page of pages) {
+      page.mediaType = 'text/markdown'
+      markdownify(page, pubDate)
+    }
+  })
+
+ this.once('beforePublish', async ({ siteCatalog }) => {
+    const nav = siteCatalog.getFiles().find(
+      file => file.path === '_/js/site-navigation-data.js').contents.toString()
+    const match = nav.match(/^siteNavigationData=(.+)$/s);
+    // This file is created by @antora/site-generator-ms
+    // eslint-disable-next-line no-eval
+    const navObj = eval(match[1])
+
+    let output = ''
+
+    output+=`# Couchbase
+
+> This is the official documentation for Couchbase, a leading NoSQL distributed database platform
+> designed for high performance, scalability, and flexibility.
+> The documentation covers comprehensive guides for developers and operations
+> teams, including installation and configuration, development tutorials, API
+> references, administration, security, cloud deployment, and integration with
+> various  programming languages and frameworks.
+> Couchbase enables applications to handle massive data volumes and high
+> concurrency with its memory-first architecture and flexible JSON document
+> model.
+
+
+## Docs\n`
+
+    for (const c of navObj) {
+      const v = c.versions[0]
+      if (! v.sets.length) { continue }
+
+      const version = v.version.match(/\d\.\d/) ? `(${v.version})` : ''
+      output += `\n\n### ${c.title} ${version}\n`
+
+      function process(item, level=0) {
+        if (item.content) {
+
+          const content = item.url ?
+            `[${item.content}](${item.url})` :
+            nhm.translate(item.content)
+
+          // above is due to a quirk in the way the navigation data is generated:
+          // in the case of a link with `^` created with `rel="noopener" target="_blank"`
+          // something in the antora -> antora-site-generator-ms pipeline causes the content 
+          // to be the HTML node, with no url extracted. This still *works* because Markdown
+          // allows HTML snippets, but it's ugly for readers of the raw markdown, so we translate 
+          // the HTML link to markdown ourselves here.
+
+          output+=`${' '.repeat(4*(level))}- ${content}\n`
+        }
+        const indent = item.content ? 1 : 0
+
+        for (const i of item.items || []) {
+          process(i, level+indent)
+        }
+      }
+
+      process({ items: v.sets })
+      const file = new File({
+        contents: Buffer.from(output),
+        mediaType: 'text/markdown',
+        out: { path: 'llms.txt' },
+        path: 'llms.txt',
+        pub: { url: `/llms.txt`, rootPath: '' },
+        src: { stem: 'llms' },
+      })
+      siteCatalog.addFile(file)
+    }
+  })
+}

package-lock.json

@@ -19,6 +19,7 @@
     "gulp-connect": "~5.7",
     "js-yaml": "~4.1",
     "markdown-it": "^13.0.1",
+    "node-html-markdown": "^2.0.0",
     "ora": "^9.0.0",
     "picomatch": "^4.0.3",
     "striptags": "^3.2.0",