feat: updated scribe to have better turbopuffer mapping (#241)

Co-authored-by: Kapil Gowru <[email protected]>

Author: kgowru

.github/ISSUE_TEMPLATE/fern_scribe_request.yml

@@ -14,31 +14,31 @@ body:
     id: request_description
     attributes:
       label: "📝 What do you want Fern Scribe (AI Technical Writer) to do?"
-      description: "Describe the documentation changes you need"
-      placeholder: "Example: Update the authentication guide to include new OAuth 2.0 flow, add examples for the new webhook endpoints"
+      description: "Example: Update the authentication guide to include new OAuth 2.0 flow, add examples for the new webhook endpoints"
+      placeholder: "Describe docs changes here..."
     validations:
       required: true
 
   - type: input
     id: slack_thread
     attributes:
       label: "🔗 Link to existing Slack thread"
-      description: "Optional: Link to relevant Slack discussion"
-      placeholder: "https://your-workspace.slack.com/archives/..."
+      description: "Ex. https://your-workspace.slack.com/archives/..."
+      placeholder: "Paste slack link here..."
 
   - type: textarea
     id: existing_instructions
     attributes:
-      label: "📋 Are there existing instructions in that docs for this issue?"
-      description: "Reference any existing documentation or guidelines"
-      placeholder: "See docs/guides/authentication.md for current approach"
+      label: "📋 Do you know of any documentation that needs to be updated?"
+      description: "Ex. See buildwithfern.com/learn/docs/guides/authentication for current approach"
+      placeholder: "Paste docs links here..."
 
   - type: textarea
     id: why_not_work
     attributes:
-      label: "❌ If yes, describe why they didn't work"
-      description: "Explain what's missing or incorrect in current docs"
-      placeholder: "Current docs don't cover the new OAuth scopes introduced in v2.1"
+      label: "❌ If yes, describe why that documentation is incorrect"
+      description: "Ex. Current docs don't cover the new OAuth scopes introduced in v2.1"
+      placeholder: "Describe here..."
 
   - type: checkboxes
     id: changelog_required
@@ -49,21 +49,9 @@ body:
         - label: "Yes, include changelog entry"
         - label: "No changelog needed"
 
-  - type: dropdown
-    id: priority
-    attributes:
-      label: "🚨 Priority Level"
-      options:
-        - "Low - Documentation enhancement"
-        - "Medium - Important update needed"
-        - "High - Critical documentation gap"
-        - "Urgent - Blocks user adoption"
-    validations:
-      required: true
-
   - type: textarea
     id: additional_context
     attributes:
       label: "📎 Additional Context"
       description: "Any other relevant information, links, or requirements"
-      placeholder: "Related API changes, user feedback, etc."
+      placeholder: "Add info here..."

.github/scripts/fern-scribe.js

@@ -573,6 +573,20 @@ Changelog entry:`;
           resolvedPath = `fern/${resolvedPath}`;
         }
 
+        // Determine the effective slug for URL generation
+        let effectiveSlug = null;
+        if (product['skip-slug'] === true) {
+          // Skip slug entirely - pages will be at root level
+          effectiveSlug = '';
+        } else if (product.slug) {
+          // Use explicit slug
+          effectiveSlug = product.slug;
+        } else {
+          // Derive slug from path or display-name
+          const pathBasename = product.path.split('/').pop().replace(/\.yml$/, '');
+          effectiveSlug = pathBasename;
+        }
+        
         // Load product config
         const productContent = await this.fetchFileContent(resolvedPath);
         if (!productContent) continue;
@@ -581,7 +595,7 @@ Changelog entry:`;
         if (!productConfig.navigation) continue;
 
         // Process navigation structure
-        await this.processNavigation(productConfig, product.slug, resolvedPath);
+        await this.processNavigation(productConfig, effectiveSlug, resolvedPath);
       }
 
       this.isPathMappingLoaded = true;
@@ -591,24 +605,64 @@ Changelog entry:`;
     }
   }
 
-  async processNavigation(config, productSlug, configPath) {
+  async processNavigation(config, productSlug, configPath, parentSections = []) {
     if (!config.navigation) return;
 
-    const basePath = configPath.replace(/\/[^\/]+\.yml$/, '').replace('fern/', '');
+    // Extract the directory path from the config file path
+    // e.g., "fern/products/fern-def.yml" -> "products/fern-def"
+    const basePath = configPath.replace(/\.yml$/, '').replace('fern/', '');
 
     for (const navItem of config.navigation) {
       if (navItem.page) {
-        // It's a page
-        const pageFilePath = `fern/${basePath}/pages/${navItem.page}`;
-        const pageUrl = `/${productSlug}/${navItem.page}`;
+        // It's a page - check if it has a custom path
+        let pageFilePath;
+        if (navItem.path) {
+          // Use custom path
+          pageFilePath = `fern/${basePath}/pages/${navItem.path}`;
+        } else {
+          // Default path based on page name
+          pageFilePath = `fern/${basePath}/pages/${navItem.page}`;
+        }
+        
+        // Build URL with parent sections
+        const urlParts = [productSlug, ...parentSections, navItem.page].filter(Boolean);
+        const pageUrl = `/${urlParts.join('/')}`;
         this.dynamicPathMapping.set(pageUrl, pageFilePath);
       } else if (navItem.section) {
-        // It's a section with pages
+        // It's a section with pages - create section-based URLs
+        const sectionSlug = navItem.section.toLowerCase().replace(/\s+/g, '-');
+        const newParentSections = [...parentSections, sectionSlug];
+        
         for (const pageItem of navItem.contents || []) {
           if (pageItem.page) {
-            const pageFilePath = `fern/${basePath}/pages/${pageItem.page}`;
-            const pageUrl = `/${productSlug}/${pageItem.page}`;
-            this.dynamicPathMapping.set(pageUrl, pageFilePath);
+            let pageFilePath;
+            if (pageItem.path) {
+              // Use custom path
+              pageFilePath = `fern/${basePath}/pages/${pageItem.path}`;
+            } else {
+              // Default path based on page name
+              pageFilePath = `fern/${basePath}/pages/${pageItem.page}`;
+            }
+            
+            // Build URL with all parent sections
+            const urlParts = [productSlug, ...newParentSections, pageItem.page].filter(Boolean);
+            const sectionUrl = `/${urlParts.join('/')}`;
+            this.dynamicPathMapping.set(sectionUrl, pageFilePath);
+            
+            // Also create direct URL for backwards compatibility (without parent sections)
+            const directUrlParts = [productSlug, pageItem.page].filter(Boolean);
+            const directUrl = `/${directUrlParts.join('/')}`;
+            if (!this.dynamicPathMapping.has(directUrl)) {
+              this.dynamicPathMapping.set(directUrl, pageFilePath);
+            }
+          } else if (pageItem.section) {
+            // Handle nested sections recursively
+            const nestedSectionSlug = pageItem.section.toLowerCase().replace(/\s+/g, '-');
+            const nestedParentSections = [...newParentSections, nestedSectionSlug];
+            
+            // Create a temporary config object for recursive processing
+            const nestedConfig = { navigation: pageItem.contents || [] };
+            await this.processNavigation(nestedConfig, productSlug, configPath, nestedParentSections);
           }
         }
       }
@@ -620,7 +674,21 @@ Changelog entry:`;
     // Remove /learn prefix and clean up trailing slashes
     let urlPath = turbopufferUrl.replace('/learn', '').replace(/\/$/, '');
 
-    // Extract product and path
+    // First try to use dynamic mapping
+    if (this.dynamicPathMapping.has(urlPath)) {
+      const mappedPath = this.dynamicPathMapping.get(urlPath);
+      // Add .mdx extension if not present and not already a complete path
+      if (!mappedPath.endsWith('.mdx') && !mappedPath.endsWith('/')) {
+        // Check if it's a known directory case (like changelog)
+        if (mappedPath.endsWith('/changelog') || (mappedPath.includes('/changelog') && !mappedPath.includes('.'))) {
+          return mappedPath; // Return as directory
+        }
+        return `${mappedPath}.mdx`;
+      }
+      return mappedPath;
+    }
+    
+    // Fallback to hardcoded logic for backwards compatibility
     const pathParts = urlPath.split('/').filter(p => p);
     if (pathParts.length === 0) return null;
 
@@ -662,7 +730,7 @@ Changelog entry:`;
     // Ensure dynamic mapping is loaded
     await this.loadDynamicPathMapping();
 
-    // Use the new transformation logic
+    // Use the improved transformation logic that prioritizes dynamic mapping
     return this.transformTurbopufferUrlToPath(turbopufferPath) || turbopufferPath;
   }
 

.github/scripts/system-prompt.md

@@ -24,7 +24,7 @@ Follow these key principles from Fern's contribution guide:
 - Use clear, descriptive headings with appropriate markdown hierarchy
 - Include practical code examples with proper syntax highlighting
 - Provide step-by-step instructions for complex procedures
-- Use Fern's callouts and admonitions for important notes, warnings, and tips
+- Use Fern's callouts for important notes, warnings, and tips
 - Maintain consistent formatting and style throughout
 - Include relevant links to related documentation
 - Cross-reference related concepts appropriately
@@ -75,16 +75,16 @@ Based on Fern's documentation standards (influenced by Google's developer docume
 ## Fern-Specific Components
 
 When creating documentation, leverage Fern's documentation components:
-- Callouts for important information
+- Callouts for important information, warnings, and tips
 - Code blocks with proper language tags
 - Tabs for multiple examples
 - Cards for organizing related content
-- Admonitions for warnings and tips
 
 ## Quality Checklist
 
 Before finalizing content, ensure:
-- ✅ All code examples are tested and functional
+- ✅ All code examples are tested and functional, try to only show one example per edit
+- ✅ Prefer making edits to one page and point to that page and section from other pages if needed.  
 - ✅ Links are valid and point to correct destinations
 - ✅ Formatting is consistent with Fern standards
 - ✅ Content is logically organized and flows well
@@ -102,6 +102,6 @@ Remember that Fern documentation:
 - Requires Node.js 16+ and npm
 - Uses Fern's documentation component system
 - Follows a specific file structure and organization
-- Is currently live at fern-api.docs.buildwithfern.com/learn/home
+- Is currently live at buildwithfern.com/learn
 
 Always maintain consistency with existing documentation patterns and structure.