Fix snippet-final-v5-(2).jpg

Author: gewenyu99

gatsby/onCreateNode.ts

@@ -6,7 +6,6 @@ import slugify from 'slugify'
 import { JSDOM } from 'jsdom'
 import { GatsbyNode } from 'gatsby'
 import { PAGEVIEW_CACHE_KEY } from './onPreBootstrap'
-import { resolveSnippets } from './snippetUtils'
 
 require('dotenv').config({
     path: `.env.${process.env.NODE_ENV}`,
@@ -271,16 +270,15 @@ export const onCreateNode: GatsbyNode['onCreateNode'] = async ({
         }
 
         const contentWithoutFrontmatter = stripFrontmatter(node.rawBody)
-        const contentWithSnippets = resolveSnippets(contentWithoutFrontmatter, node.fileAbsolutePath)
 
         // Prepend title as H1 if it exists
         const title = node.frontmatter?.title
-        const contentWithSnippetsAndTitle = title ? `# ${title}\n\n${contentWithSnippets}` : contentWithSnippets
+        const contentWithTitle = title ? `# ${title}\n\n${contentWithoutFrontmatter}` : contentWithoutFrontmatter
 
         createNodeField({
             node,
             name: `contentWithSnippets`,
-            value: contentWithSnippetsAndTitle,
+            value: contentWithTitle,
         })
     }
 

gatsby/onPostBuild.ts

@@ -482,7 +482,7 @@ const createOrUpdateStrapiPosts = async (posts, roadmaps) => {
     )
 }
 
-export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql }) => {
+export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql, reporter }) => {
     // Generate API spec markdown files first
     try {
         const openApiSpecUrl = process.env.POSTHOG_OPEN_API_SPEC_URL || 'https://app.posthog.com/api/schema/'
@@ -555,26 +555,24 @@ export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql }) => {
         generateSdkReferencesMarkdown(node)
     })
 
-    // Generate markdown files for llms.txt file and LLM ingestion (after API spec files exist)
-    const markdownQuery = await graphql(`
-        query pagesForMarkdown {
-            allMdx {
+    // Generate markdown files for llms.txt file and LLM ingestion (after pages are built)
+    // Convert HTML files to markdown using turndown
+    const docsQuery = (await graphql(`
+        query {
+            allMdx(filter: { fields: { slug: { regex: "/^/docs/" } } }) {
                 nodes {
-                    frontmatter {
-                        title
-                        date
-                    }
-                    rawBody
                     fields {
                         slug
-                        contentWithSnippets
+                    }
+                    frontmatter {
+                        title
                     }
                 }
             }
         }
-    `)
+    `)) as { data: { allMdx: { nodes: Array<{ fields: { slug: string }; frontmatter: { title: string } }> } } }
 
-    const filteredPages = await generateRawMarkdownPages(markdownQuery.data.allMdx.nodes)
+    const filteredPages = await generateRawMarkdownPages(docsQuery.data.allMdx.nodes, reporter)
     generateLlmsTxt(filteredPages)
 
     if (process.env.AWS_CODEPIPELINE !== 'true') {

gatsby/rawMarkdownUtils.ts

@@ -2,12 +2,20 @@ import path from 'path'
 import fs from 'fs'
 import { SdkReferenceData } from '../src/templates/sdk/SdkReference'
 import { getLanguageFromSdkId } from '../src/components/SdkReferences/utils'
+import {
+    createTurndownService,
+    extractTitleFromHtml,
+    extractMainContent,
+    postProcessMarkdown,
+    preprocessHtmlForTabs,
+} from './turndownService'
+
+export const generateRawMarkdownPages = async (
+    docsNodes: Array<{ fields: { slug: string }; frontmatter: { title: string } }>,
+    reporter?: any
+) => {
+    const publicPath = path.resolve(__dirname, '../public')
 
-// Function to generate raw markdown files
-export const generateRawMarkdownPages = async (pages) => {
-    console.log('Generating markdown files for LLMs...')
-
-    // Filter out any pages with certain slugs
     const excludeTerms = [
         '/_snippets',
         '/snippets/',
@@ -23,53 +31,63 @@ export const generateRawMarkdownPages = async (pages) => {
         '/startups',
         '/example-components',
     ]
-    const filteredPages = pages.filter((doc) => !excludeTerms.some((term) => doc.fields.slug.includes(term)))
 
-    console.log(`Found ${filteredPages.length} docs to generate markdown for (filtered from ${pages.length} total)`)
+    const filteredNodes = docsNodes.filter((node) => {
+        return !excludeTerms.some((term) => node.fields.slug.includes(term))
+    })
+
+    const processedPages: Array<{ slug: string; title: string }> = []
 
-    for (const doc of filteredPages) {
+    for (const node of filteredNodes) {
         try {
-            const { slug, contentWithSnippets } = doc.fields
-            const { title } = doc.frontmatter
-            const body = contentWithSnippets || doc.rawBody
-
-            // Create the frontmatter, so it always has the page title
-            let markdownContent = `---\ntitle: ${title}\nslug: ${slug}\n---\n`
-
-            // Add the content
-            if (body) {
-                // Process internal links to point to .md equivalents
-                let processedBody = body.replace(/\[([^\]]+)\]\(\/([^)]+)\)/g, (match, text, path) => {
-                    // Only convert if the path doesn't already end with .md
-                    if (!path.endsWith('.md')) {
-                        return `[${text}](/${path}.md)`
-                    }
-                    return match
-                })
+            const { slug } = node.fields
+            const htmlFilePath = path.join(publicPath, slug, 'index.html')
 
-                markdownContent += processedBody
+            if (!fs.existsSync(htmlFilePath)) {
+                if (reporter) {
+                    reporter.warn(`HTML file not found: ${htmlFilePath}`)
+                }
+                continue
             }
 
-            // Create the directory structure
-            const publicPath = path.resolve(__dirname, '../public')
-            const filePath = path.join(publicPath, `${slug}.md`)
-            const dirPath = path.dirname(filePath)
+            const html = fs.readFileSync(htmlFilePath, 'utf8')
+            const title = extractTitleFromHtml(html) || node.frontmatter.title || 'Untitled'
+            const mainContent = extractMainContent(html)
+            const preprocessedContent = preprocessHtmlForTabs(mainContent)
+
+            const turndownService = createTurndownService(title)
+            let markdown = turndownService.turndown(preprocessedContent)
+            markdown = postProcessMarkdown(markdown, title)
+
+            const outputPath = path.join(publicPath, `${slug}.md`)
+            const dirPath = path.dirname(outputPath)
 
-            // Ensure directory exists
             if (!fs.existsSync(dirPath)) {
                 fs.mkdirSync(dirPath, { recursive: true })
             }
 
-            // Write the file
-            fs.writeFileSync(filePath, markdownContent, 'utf8')
-            console.log(`Generated: ${slug}.md`)
+            fs.writeFileSync(outputPath, markdown, 'utf8')
+            processedPages.push({ slug, title })
+
+            if (reporter) {
+                reporter.info(`Generated: ${slug}.md`)
+            }
         } catch (error) {
-            console.error(`Error generating markdown for ${doc.fields.slug}:`, error)
+            const errorMsg = `Error generating markdown for ${node.fields.slug}: ${error}`
+            if (reporter) {
+                reporter.error(errorMsg)
+            }
         }
     }
 
-    // Return filtered pages for use in generateLlmsTxt
-    return filteredPages
+    if (reporter) {
+        reporter.info(`Generated ${processedPages.length} markdown files`)
+    }
+
+    return processedPages.map((page) => ({
+        fields: { slug: page.slug },
+        frontmatter: { title: page.title },
+    }))
 }
 
 // Function to generate individual API endpoint markdown files from the OpenAPI spec