fixes

Author: osfameron

lib/markdown-for-llm.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { NodeHtmlMarkdown } = require('node-html-markdown')
+const YAML = require('yaml')
 
 let nhm
 const customTranslators = {
@@ -12,8 +13,8 @@ const customTranslators = {
 
         const type = (node
             .classList.values().
-            find(v => v != 'admonitionblock')
-            .toUpperCase())
+            find(v => v != 'admonitionblock') || 'NOTE'
+            ).toUpperCase()
 
         const content = (
           nhm.translate(
@@ -30,16 +31,28 @@ nhm = new NodeHtmlMarkdown({}, customTranslators)
 
 const File = require('vinyl')
 
-function markdownify(page) {
+function markdownify(page, pubDate) {
+  const title = page.title
   const html = page.contents.toString()
+  const markdown = nhm.translate(html)
 
   // haha, we now have to translate the .md link back to .html just for this link
   const orig = page.pub.url.replace(/\.md$/, '.html')
-  const link = `[View original HTML](${orig})\n\n`
 
-  const markdown = link + nhm.translate(html)
+  let output = 
+    `[View original HTML](${orig})\n\n` +
+    `# ${title}\n\n${markdown}`
 
-  page.contents = Buffer.from(markdown)
+  const frontmatter = {
+    title,
+    description: page.asciidoc.attributes.description,
+    editUrl: page.src.editUrl,
+    pubDate,
+  }
+
+  output = `---\n${YAML.stringify(frontmatter)}---\n\n${output}`
+
+  page.contents = Buffer.from(output)
 }
 
 module.exports.register = function ({ playbook, config }) {
@@ -84,6 +97,8 @@ module.exports.register = function ({ playbook, config }) {
   this.once('documentsConverted', async ({ playbook, contentCatalog, siteCatalog }) => {
     const logger = this.getLogger('llm-summaries')
 
+    const pubDate = new Date().toISOString()
+
     const pages = contentCatalog.getPages(
       (page) =>
         page.mediaType === 'text/html'
@@ -92,7 +107,7 @@ module.exports.register = function ({ playbook, config }) {
 
     for (const page of pages) {
       page.mediaType = 'text/markdown'
-      markdownify(page)
+      markdownify(page, pubDate)
     }
   })
 

scripts/update-snapshot

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# An antora build with the lib/markdown-for-llms.js extension 
+# will output a directory of Markdown files, with YAML front-matter
+# including pubDate.
+# If we first clone the https://github.com/couchbaselabs/docs-markdown-snapshot/
+# repo into `public/` then the Antora build will naturally
+# update those files in place.
+#
+# This script attempts to handle these changes, including
+#  * new files
+#  * deletion of no-longer output files
+#  * preserving files which are unchanged (other than the pubDate)
+
+cd public
+
+# all markdown files
+find . -name *.md \
+  | sed 's|^\./||' \
+  | sort \
+  > md-all
+
+# all markdown that
+# were output by Antora
+# and caused a modification (M) flag
+# (e.g. that have at the very least, a new pubDate:)
+git status --porcelain=v1 \
+  --untracked-files=no \
+  | grep '\.md$' \
+  | grep '^\s*M' \
+  | awk '{ print $2 }' \
+  | sort \
+  > md-output
+
+# all markdown files that have changed more than just pubDate
+# (-I flag ignores changes which *only* contain that pattern)
+git diff --numstat -I^pubDate: \
+  | grep '\.md$' \
+  | awk '{ print $3 }' \
+  | sort \
+  > md-changed
+
+# all markdown that
+# were output by Antora
+# and aren't already in git (??)
+git status --porcelain=v1 \
+  --untracked-files=all \
+  | grep '\.md$' \
+  | grep ^\?\? \
+  | awk '{ print $2 }' \
+  | sort \
+  > md-added
+
+# So we want to categorise the following groups:
+# A. in (md-changed, md-added)                 -> ADD
+# B. in (md-output) but NOT (md-changed)       -> CHECKOUT (revert)
+# C. in (md-all) but NOT (md-added, md-output) -> DELETE
+
+# The useful-but-confusing `comm` utility prints:
+#     (only 1) (only 2) (common)
+# 
+# so `comm -23` prints in 1 but NOT 2 (or common)
+
+# A. in (md-changed, md-added)                 -> ADD
+git add --pathspec-from-file md-changed
+git add --pathspec-from-file md-added
+
+# B. in (md-output) but NOT (md-changed)       -> CHECKOUT (revert)
+comm -23 md-output md-changed > md-checkout
+git checkout --pathspec-from-file md-checkout
+
+# C. in (md-all) but NOT (md-added, md-output) -> DELETE
+comm -23 md-all <(sort md-added md-output) > md-delete
+git rm --pathspec-from-file md-delete
+
+git commit -m "Updating snapshot"
+git push