fix: markdown converter, replace regex with cheerio for extracting main content div (#37)

walmsles · web-flow · commit 685dc590d7ce · 2025-05-17T00:11:22.000+10:00
diff --git a/package.json b/package.json
@@ -53,6 +53,7 @@
     "@types/cacache": "^17.0.2",
     "@types/node": "^22.14.1",
     "cacache": "^19.0.1",
+    "cheerio": "^1.0.0",
     "lunr": "^2.3.9",
     "lunr-languages": "^1.14.0",
     "make-fetch-happen": "^14.0.3",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/services/markdown/nodeHtmlMarkdownConverter.spec.ts b/src/services/markdown/nodeHtmlMarkdownConverter.spec.ts
@@ -31,6 +31,32 @@ describe('[Markdown-Converter] When using NodeHtmlMarkdownConverter', () => {
       const result = converter.extractContent(html);
       expect(result.content).toContain('Test content');
     });
+    
+    it('should extract complete content from md-content div with nested elements', () => {
+      const html = `
+        <html>
+          <body>
+            <div class="md-content" data-md-component="content">
+              <h2>Section 1</h2>
+              <p>First paragraph</p>
+              <div class="nested">
+                <h3>Subsection</h3>
+                <p>Nested content</p>
+              </div>
+              <h2>Section 2</h2>
+              <p>Final paragraph</p>
+            </div>
+          </body>
+        </html>
+      `;
+      const result = converter.extractContent(html);
+      expect(result.content).toContain('Section 1');
+      expect(result.content).toContain('First paragraph');
+      expect(result.content).toContain('Subsection');
+      expect(result.content).toContain('Nested content');
+      expect(result.content).toContain('Section 2');
+      expect(result.content).toContain('Final paragraph');
+    });
   });
 
   describe('[Markdown-Conversion] When converting HTML to markdown', () => {
diff --git a/src/services/markdown/nodeHtmlMarkdownConverter.ts b/src/services/markdown/nodeHtmlMarkdownConverter.ts
@@ -1,5 +1,6 @@
 import { HtmlToMarkdownConverter } from './types';
 
+import * as cheerio from 'cheerio';
 import { NodeHtmlMarkdown, NodeHtmlMarkdownOptions } from 'node-html-markdown';
 
 /**
@@ -44,24 +45,34 @@ export class NodeHtmlMarkdownConverter implements HtmlToMarkdownConverter {
   }
   
   /**
-   * Extract title and main content from HTML using regex
+   * Extract title and main content from HTML using cheerio DOM parser
    * @param html The HTML content to process
    * @returns Object containing title and main content
    */
   extractContent(html: string): { title: string, content: string } {
-    // Simple regex approach for title extraction
-    const titleMatch = html.match(/<h1[^>]*>(.*?)<\/h1>/i) || 
-                      html.match(/<title[^>]*>(.*?)<\/title>/i);
-    const title = titleMatch ? titleMatch[1].trim() : '';
+    // Load HTML into cheerio
+    const $ = cheerio.load(html);
+    
+    // Extract title - first try h1, then fall back to title tag
+    let title = $('h1').first().text().trim();
+    if (!title) {
+      title = $('title').text().trim();
+    }
     
     // Extract main content - target the md-content container
-    const contentMatch = html.match(/<div class="md-content"[^>]*data-md-component="content"[^>]*>([\s\S]*?)<\/div>/i);
+    let contentHtml = '';
+    
+    // First try to find the main content container
+    const mdContent = $('div.md-content[data-md-component="content"]');
     
-    // If we found the main content container, use it; otherwise use the body
-    const content = contentMatch ? 
-      contentMatch[0] : 
-      html.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[0] || html;
+    if (mdContent.length > 0) {
+      // Get the HTML content of the md-content div
+      contentHtml = mdContent.html() || '';
+    } else {
+      // Fall back to body content if md-content not found
+      contentHtml = $('body').html() || html;
+    }
     
-    return { title, content };
+    return { title, content: contentHtml };
   }
 }
