feat: improve refparser

Author: xlc-dev

refparser.go

@@ -8,49 +8,45 @@ import (
 	"go/parser"
 	"go/printer"
 	"go/token"
+	"html"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 )
 
-// generateReferenceMarkdown creates a Markdown documentation file from Go source code comments.
-// It parses the package found in inputDir and writes the formatted Markdown to outputFile.
+// generateReferenceMarkdown generates a Markdown reference file for the package
+// found in inputDir and writes it to outputFile.
 func generateReferenceMarkdown(inputDir, outputFile string) error {
 	if _, err := os.Stat(inputDir); os.IsNotExist(err) {
 		return fmt.Errorf("input directory does not exist: %s", inputDir)
 	}
 
 	outputDirPath := filepath.Dir(outputFile)
 	if err := os.MkdirAll(outputDirPath, 0755); err != nil {
-		// Log warning but continue, os.Create might still work
 		log.Printf("Warning: Could not ensure output directory %s exists: %v", outputDirPath, err)
 	}
 
 	log.Printf("Generating reference docs from '%s' to '%s'", inputDir, outputFile)
 
 	fset := token.NewFileSet()
 	pkgs, err := parser.ParseDir(fset, inputDir, func(fi os.FileInfo) bool {
-		// Basic filter to ignore test files
 		return !strings.HasSuffix(fi.Name(), "_test.go")
 	}, parser.ParseComments)
-
 	if err != nil {
 		return fmt.Errorf("failed to parse directory %s: %w", inputDir, err)
 	}
 
-	// Find the primary package in the directory
 	var pkg *ast.Package
 	for _, p := range pkgs {
 		pkg = p
-		break // Use the first package found
+		break
 	}
 	if pkg == nil {
 		return fmt.Errorf("no non-test Go package found in directory: %s", inputDir)
 	}
 
-	// Extract documentation (exported symbols only by default)
 	docPkg := doc.New(pkg, pkg.Name, doc.AllDecls)
 
 	out, err := os.Create(outputFile)
@@ -59,217 +55,172 @@ func generateReferenceMarkdown(inputDir, outputFile string) error {
 	}
 	defer out.Close()
 
-	// Add header
 	header := "{{ title: Nova - Reference }}\n\n{{ include-block: doc.html markdown=\"true\" }}\n\n"
-	_, err = out.WriteString(header)
-	if err != nil {
+	if _, err := out.WriteString(header); err != nil {
 		return fmt.Errorf("failed to write header to output file: %w", err)
 	}
 
-	var contentBuf bytes.Buffer
-
-	title := "# Reference\n\n"
-
-	// Package Documentation (if any)
-	if docPkg.Doc != "" {
-		contentBuf.WriteString(fmt.Sprintf("## %s\n\n", "Overview"))
-		contentBuf.WriteString(formatDocText(docPkg.Doc))
-		contentBuf.WriteString("\n\n")
-	}
-
-	// Write Title
-	_, err = out.WriteString(title)
-	if err != nil {
-		return fmt.Errorf("failed to write title to output file: %w", err)
-	}
+	var content bytes.Buffer
+	content.WriteString("# Reference\n\n")
 
-	// Generate and Write Table of Contents
 	toc := generateTOC(docPkg)
 	if toc != "" {
-		_, err = out.WriteString(toc)
-		if err != nil {
-			return fmt.Errorf("failed to write TOC to output file: %w", err)
-		}
-		_, err = out.WriteString("\n\n") // Add separation after TOC
-		if err != nil {
-			return fmt.Errorf("failed to write separator after TOC: %w", err)
-		}
+		content.WriteString(toc)
+		content.WriteString("\n\n")
+	}
+
+	if docPkg.Doc != "" {
+		content.WriteString("## Overview\n\n")
+		content.WriteString(formatDocText(docPkg.Doc))
+		content.WriteString("\n\n")
 	}
 
-	// Constants
 	if len(docPkg.Consts) > 0 {
-		contentBuf.WriteString("## Constants\n\n")
+		content.WriteString("## Constants\n\n")
 		for _, c := range docPkg.Consts {
-			writeDocItem(&contentBuf, fset, c.Doc, c.Names, c.Decl, 3)
+			writeDocItem(&content, fset, c.Doc, c.Names, c.Decl, 3)
 		}
 	}
 
-	// Variables
 	if len(docPkg.Vars) > 0 {
-		contentBuf.WriteString("## Variables\n\n")
+		content.WriteString("## Variables\n\n")
 		for _, v := range docPkg.Vars {
-			writeDocItem(&contentBuf, fset, v.Doc, v.Names, v.Decl, 3)
+			writeDocItem(&content, fset, v.Doc, v.Names, v.Decl, 3)
 		}
 	}
 
-	// Functions
 	if len(docPkg.Funcs) > 0 {
-		contentBuf.WriteString("## Functions\n\n")
+		content.WriteString("## Functions\n\n")
 		for _, f := range docPkg.Funcs {
-			writeDocItem(&contentBuf, fset, f.Doc, []string{f.Name}, f.Decl, 3)
+			writeDocItem(&content, fset, f.Doc, []string{f.Name}, f.Decl, 3)
 		}
 	}
 
-	// Types
 	if len(docPkg.Types) > 0 {
-		contentBuf.WriteString("## Types\n\n")
+		content.WriteString("## Types\n\n")
 		for _, t := range docPkg.Types {
-			writeDocItem(&contentBuf, fset, t.Doc, []string{t.Name}, t.Decl, 3)
+			fmt.Fprintf(&content, "### `%s`\n\n", t.Name)
+			printDeclaration(&content, fset, t.Decl, t.Name)
+			if t.Doc != "" {
+				content.WriteString(formatDocText(t.Doc))
+				content.WriteString("\n\n")
+			}
 
 			if len(t.Consts) > 0 {
-				contentBuf.WriteString("#### Associated Constants\n\n")
+				content.WriteString("#### Associated Constants\n\n")
 				for _, c := range t.Consts {
-					writeDocItem(&contentBuf, fset, c.Doc, c.Names, c.Decl, 4)
+					writeDocItem(&content, fset, c.Doc, c.Names, c.Decl, 4)
 				}
 			}
 			if len(t.Vars) > 0 {
-				contentBuf.WriteString("#### Associated Variables\n\n")
+				content.WriteString("#### Associated Variables\n\n")
 				for _, v := range t.Vars {
-					writeDocItem(&contentBuf, fset, v.Doc, v.Names, v.Decl, 4)
+					writeDocItem(&content, fset, v.Doc, v.Names, v.Decl, 4)
 				}
 			}
 			if len(t.Funcs) > 0 {
-				contentBuf.WriteString("#### Associated Functions\n\n")
+				content.WriteString("#### Associated Functions\n\n")
 				for _, f := range t.Funcs {
-					writeDocItem(&contentBuf, fset, f.Doc, []string{f.Name}, f.Decl, 4)
+					writeDocItem(&content, fset, f.Doc, []string{f.Name}, f.Decl, 4)
 				}
 			}
 			if len(t.Methods) > 0 {
-				contentBuf.WriteString("#### Methods\n\n")
+				content.WriteString("#### Methods\n\n")
 				for _, m := range t.Methods {
-					writeDocItem(&contentBuf, fset, m.Doc, []string{m.Name}, m.Decl, 4)
+					writeDocItem(&content, fset, m.Doc, []string{m.Name}, m.Decl, 4)
 				}
 			}
 		}
 	}
 
-	// Write Main Content
-	_, err = contentBuf.WriteTo(out)
-	if err != nil {
+	if _, err := content.WriteTo(out); err != nil {
 		return fmt.Errorf("failed to write content buffer to output file: %w", err)
 	}
 
-	// Add footer
 	footer := "{{ endinclude }}"
-	_, err = out.WriteString(footer)
-	if err != nil {
+	if _, err := out.WriteString(footer); err != nil {
 		return fmt.Errorf("failed to write footer to output file: %w", err)
 	}
 
-	log.Printf("Successfully generated reference docs with TOC to %s", outputFile)
+	log.Printf("Successfully generated reference docs to %s", outputFile)
 	return nil
 }
 
+// generateTOC builds a Table of Contents using GitHub-style heading anchors.
 func generateTOC(docPkg *doc.Package) string {
 	var tocBuf bytes.Buffer
 	tocBuf.WriteString("## Table of Contents\n\n")
 
 	hasContent := false
 
 	if docPkg.Doc != "" {
-		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Overview", "overview"))
+		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Overview", generateAnchor("Overview")))
 		hasContent = true
 	}
-
 	if len(docPkg.Consts) > 0 {
-		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Constants", "constants"))
+		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Constants", generateAnchor("Constants")))
 		hasContent = true
 	}
-
 	if len(docPkg.Vars) > 0 {
-		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Variables", "variables"))
+		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Variables", generateAnchor("Variables")))
 		hasContent = true
 	}
-
 	if len(docPkg.Funcs) > 0 {
-		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Functions", "functions"))
+		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Functions", generateAnchor("Functions")))
 		hasContent = true
 	}
-
 	if len(docPkg.Types) > 0 {
-		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Types", "types"))
+		tocBuf.WriteString(fmt.Sprintf("- [%s](#%s)\n", "Types", generateAnchor("Types")))
 		hasContent = true
 		for _, t := range docPkg.Types {
 			typeTitle := fmt.Sprintf("`%s`", t.Name)
 			typeAnchor := generateAnchor(t.Name)
 			tocBuf.WriteString(fmt.Sprintf("  - [%s](#%s)\n", typeTitle, typeAnchor))
-
-			if len(t.Consts) > 0 || len(t.Vars) > 0 || len(t.Funcs) > 0 || len(t.Methods) > 0 {
-				if len(t.Consts) > 0 {
-					tocBuf.WriteString(fmt.Sprintf("    - [Associated Constants](#%s-constants)\n", typeAnchor))
-				}
-				if len(t.Vars) > 0 {
-					tocBuf.WriteString(fmt.Sprintf("    - [Associated Variables](#%s-variables)\n", typeAnchor))
-				}
-				if len(t.Funcs) > 0 {
-					tocBuf.WriteString(fmt.Sprintf("    - [Associated Functions](#%s-functions)\n", typeAnchor))
-				}
-				if len(t.Methods) > 0 {
-					tocBuf.WriteString(fmt.Sprintf("    - [Methods](#%s-methods)\n", typeAnchor))
-				}
-			}
 		}
 	}
 
 	if !hasContent {
 		return ""
 	}
-
 	return tocBuf.String()
 }
 
-// writeDocItem formats a single documentation item (const, var, func, type, method)
-// and writes it to the content buffer.
+// writeDocItem writes a documentation item heading, its declaration, and doc text.
 func writeDocItem(contentBuf *bytes.Buffer, fset *token.FileSet, docComment string, names []string, decl ast.Node, level int) {
-	displayNames := strings.Join(names, ", ")
-	itemTitle := fmt.Sprintf("`%s`", displayNames)
-	itemAnchor := generateAnchor(displayNames)
-
-	// Create a clean anchor tag that is compatible with most Markdown renderers
-	fmt.Fprintf(contentBuf, "<a id=\"%s\"></a>\n", itemAnchor)
-	fmt.Fprintf(contentBuf, "%s %s\n\n", strings.Repeat("#", level), itemTitle)
+	displayName := strings.Join(names, ", ")
+	fmt.Fprintf(contentBuf, "%s `%s`\n\n", strings.Repeat("#", level), displayName)
 
-	// Print the declaration (signature) using go/printer
-	var declBuf bytes.Buffer
-	cfg := printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 4}
-	err := cfg.Fprint(&declBuf, fset, decl)
-	if err != nil {
-		log.Printf("Warning: Failed to print declaration for %s: %v", displayNames, err)
-		contentBuf.WriteString("```go\n// Error printing declaration\n```\n\n")
-	} else {
-		contentBuf.WriteString("```go\n")
-		contentBuf.Write(declBuf.Bytes())
-		contentBuf.WriteString("\n```\n\n")
-	}
+	printDeclaration(contentBuf, fset, decl, displayName)
 
-	// Write the documentation comment
 	if docComment != "" {
 		contentBuf.WriteString(formatDocText(docComment))
 		contentBuf.WriteString("\n\n")
 	}
 }
 
+// printDeclaration writes an AST declaration into a Go code fence.
+func printDeclaration(buf *bytes.Buffer, fset *token.FileSet, decl ast.Node, name string) {
+	var declBuf bytes.Buffer
+	cfg := printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 4}
+	if err := cfg.Fprint(&declBuf, fset, decl); err != nil {
+		log.Printf("Warning: Failed to print declaration for %s: %v", name, err)
+		buf.WriteString("```go\n// Error printing declaration\n```\n\n")
+		return
+	}
+	buf.WriteString("```go\n")
+	buf.Write(declBuf.Bytes())
+	buf.WriteString("\n```\n\n")
+}
+
+// formatDocText converts Go doc comments to Markdown, unescaping HTML,
+// turning "Parameters:" into a small header and cleaning code fences.
 func formatDocText(text string) string {
-	// Trim leading/trailing whitespace
 	trimmed := strings.TrimSpace(text)
-
-	// Use doc.ToHTML to handle formatting, then convert to Markdown
 	var buf bytes.Buffer
 	doc.ToHTML(&buf, trimmed, nil)
-	html := buf.String()
+	htmlStr := buf.String()
 
-	// Basic conversion from HTML to Markdown
-	md := strings.ReplaceAll(html, "<p>", "")
+	md := strings.ReplaceAll(htmlStr, "<p>", "")
 	md = strings.ReplaceAll(md, "</p>", "\n\n")
 	md = strings.ReplaceAll(md, "<pre>", "```go\n")
 	md = strings.ReplaceAll(md, "</pre>", "\n```\n\n")
@@ -280,27 +231,32 @@ func formatDocText(text string) string {
 	md = strings.ReplaceAll(md, "<li>", "- ")
 	md = strings.ReplaceAll(md, "</li>", "\n")
 
-	// Clean up extra newlines
 	md = strings.TrimSpace(md)
-	md = regexp.MustCompile(`\n{3,}`).ReplaceAllString(md, "\n\n")
 
-	return md
-}
+	// Unescape HTML entities.
+	md = html.UnescapeString(md)
 
-func generateAnchor(text string) string {
-	// Remove backticks first
-	text = strings.ReplaceAll(text, "`", "")
-	text = strings.ToLower(text)
-	text = strings.ReplaceAll(text, " ", "-")
+	// Convert a "Parameters" paragraph into a small header.
+	md = regexp.MustCompile(`(?m)^\s*Parameters:\s*$`).ReplaceAllString(md, "#### Parameters")
 
-	// Remove any characters that are not letters, numbers, or hyphens
-	reg := regexp.MustCompile("[^a-z0-9-]+")
-	text = reg.ReplaceAllString(text, "")
+	// Ensure code fences don't end up with an extra blank line before the closing fence.
+	reFence := regexp.MustCompile("(?s)```go\\n(.*?)\\n+```")
+	md = reFence.ReplaceAllString(md, "```go\n$1\n```")
 
-	// Replace multiple hyphens with a single hyphen
-	text = regexp.MustCompile("-+").ReplaceAllString(text, "-")
+	// Collapse excessive blank lines.
+	md = regexp.MustCompile(`\n{3,}`).ReplaceAllString(md, "\n\n")
 
-	// Ensure it doesn't start or end with a hyphen
-	text = strings.Trim(text, "-")
-	return text
+	return strings.TrimSpace(md)
+}
+
+// generateAnchor returns a GitHub-style slug for text.
+func generateAnchor(text string) string {
+	s := strings.TrimSpace(text)
+	s = strings.ToLower(s)
+	s = strings.ReplaceAll(s, "`", "")
+	s = regexp.MustCompile(`[^a-z0-9 -]+`).ReplaceAllString(s, "")
+	s = strings.ReplaceAll(s, " ", "-")
+	s = regexp.MustCompile(`-+`).ReplaceAllString(s, "-")
+	s = strings.Trim(s, "-")
+	return s
 }