Skip to content

Commit a648993

Browse files
kgowruKapil GowruKapil Gowru
authored
added turbopuffer context for scribe & removed api-definition directory (#301)
Co-authored-by: Kapil Gowru <[email protected]> Co-authored-by: Kapil Gowru <[email protected]>
1 parent f904fe1 commit a648993

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1139
-4915
lines changed

‎.github/scripts/fern-scribe.js‎

Lines changed: 850 additions & 34 deletions
Large diffs are not rendered by default.

‎.github/scripts/fern-url-mapper.js‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ const fs = require('fs').promises;
55
class FernUrlMapper {
66
constructor(githubToken = null, repository = null) {
77
this.dynamicPathMapping = new Map();
8+
this.staticPathMapping = new Map();
89
this.isPathMappingLoaded = false;
10+
this.isStaticMappingLoaded = false;
911

1012
// Initialize GitHub client if credentials provided
1113
if (githubToken && repository) {
@@ -40,6 +42,35 @@ class FernUrlMapper {
4042
}
4143
}
4244

45+
// Load static path mapping from my-mappings.md
46+
async loadStaticPathMapping() {
47+
if (this.isStaticMappingLoaded) return;
48+
49+
try {
50+
const mappingsContent = await fs.readFile('my-mappings.md', 'utf-8');
51+
console.log('Loading static path mappings from my-mappings.md...');
52+
53+
// Parse the markdown file for URL mappings
54+
const lines = mappingsContent.split('\n');
55+
let mappingCount = 0;
56+
57+
for (const line of lines) {
58+
// Look for lines that match the mapping pattern: - `/learn/...` → `fern/...`
59+
const match = line.match(/^-\s+`([^`]+)`\s+→\s+`([^`]+)`/);
60+
if (match) {
61+
const [, url, path] = match;
62+
this.staticPathMapping.set(url, path);
63+
mappingCount++;
64+
}
65+
}
66+
67+
this.isStaticMappingLoaded = true;
68+
console.log(`Loaded ${mappingCount} static path mappings from my-mappings.md`);
69+
} catch (error) {
70+
console.error('Failed to load static path mapping:', error.message);
71+
}
72+
}
73+
4374
// Load dynamic path mapping from Fern docs structure
4475
async loadDynamicPathMapping() {
4576
if (this.isPathMappingLoaded) return;
@@ -222,10 +253,15 @@ class FernUrlMapper {
222253

223254
// Transform Turbopuffer URLs to actual GitHub file paths
224255
transformTurbopufferUrlToPath(turbopufferUrl) {
225-
// Clean up trailing slashes but keep the /learn prefix for dynamic mapping lookup
256+
// Clean up trailing slashes but keep the /learn prefix for mapping lookup
226257
let cleanUrl = turbopufferUrl.replace(/\/$/, '');
227258

228-
// First try to use dynamic mapping with full URL (including /learn)
259+
// First try to use static mapping from my-mappings.md
260+
if (this.staticPathMapping.has(cleanUrl)) {
261+
return this.staticPathMapping.get(cleanUrl);
262+
}
263+
264+
// Second try to use dynamic mapping with full URL (including /learn)
229265
if (this.dynamicPathMapping.has(cleanUrl)) {
230266
const mappedPath = this.dynamicPathMapping.get(cleanUrl);
231267
// Add .mdx extension if not present and not already a complete path
@@ -279,22 +315,33 @@ class FernUrlMapper {
279315
}
280316
}
281317

282-
// Map Turbopuffer URLs to actual GitHub file paths (now using dynamic mapping)
318+
// Map Turbopuffer URLs to actual GitHub file paths (now using static mapping first, then dynamic)
283319
async mapTurbopufferPathToGitHub(turbopufferPath) {
284-
// Ensure dynamic mapping is loaded
320+
// Ensure static mapping is loaded first
321+
await this.loadStaticPathMapping();
322+
// Ensure dynamic mapping is loaded as fallback
285323
await this.loadDynamicPathMapping();
286324

287-
// Use the improved transformation logic that prioritizes dynamic mapping
325+
// Use the improved transformation logic that prioritizes static mapping, then dynamic mapping
288326
return this.transformTurbopufferUrlToPath(turbopufferPath) || turbopufferPath;
289327
}
290328

291329
// Get all mappings as an object for external use
292330
async getAllMappings() {
331+
await this.loadStaticPathMapping();
293332
await this.loadDynamicPathMapping();
294333
const mappings = {};
334+
335+
// Add dynamic mappings first
295336
for (const [url, path] of this.dynamicPathMapping) {
296337
mappings[url] = path;
297338
}
339+
340+
// Override with static mappings (they take priority)
341+
for (const [url, path] of this.staticPathMapping) {
342+
mappings[url] = path;
343+
}
344+
298345
return mappings;
299346
}
300347

@@ -341,6 +388,7 @@ class FernUrlMapper {
341388

342389
// Test specific URL mappings
343390
async testMappings(testUrls = []) {
391+
await this.loadStaticPathMapping();
344392
await this.loadDynamicPathMapping();
345393

346394
console.log('\n=== TESTING URL MAPPINGS ===');

‎.github/scripts/generate-mappings.js‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
const fs = require('fs');
2+
const path = require('path');
3+
const yaml = require('js-yaml');
4+
const { Octokit } = require('@octokit/rest');
5+
6+
const OUTPUT_FILE = path.join(__dirname, 'my-mappings.md');
7+
const GITHUB_TOKEN = process.env.GITHUB_TOKEN;
8+
const REPOSITORY = process.env.REPOSITORY;
9+
const BRANCH = 'main';
10+
11+
if (!GITHUB_TOKEN || !REPOSITORY) {
12+
console.error('GITHUB_TOKEN and REPOSITORY env vars are required.');
13+
process.exit(1);
14+
}
15+
16+
const [owner, repo] = REPOSITORY.split('/');
17+
const octokit = new Octokit({ auth: GITHUB_TOKEN });
18+
19+
async function listDir(pathInRepo) {
20+
try {
21+
const res = await Promise.race([
22+
octokit.repos.getContent({
23+
owner,
24+
repo,
25+
path: pathInRepo,
26+
ref: BRANCH
27+
}),
28+
new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 10000))
29+
]);
30+
return Array.isArray(res.data) ? res.data : [];
31+
} catch (e) {
32+
return [];
33+
}
34+
}
35+
36+
async function getFileContent(pathInRepo) {
37+
try {
38+
const res = await Promise.race([
39+
octokit.repos.getContent({
40+
owner,
41+
repo,
42+
path: pathInRepo,
43+
ref: BRANCH
44+
}),
45+
new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 10000))
46+
]);
47+
if (res.data && res.data.content) {
48+
return Buffer.from(res.data.content, 'base64').toString('utf-8');
49+
}
50+
return null;
51+
} catch (e) {
52+
return null;
53+
}
54+
}
55+
56+
function slugify(str) {
57+
return String(str)
58+
.replace(/(-def|-reference|-docs|-api)$/i, '') // Only remove at the end
59+
.replace(/_/g, '-')
60+
.replace(/\s+/g, '-')
61+
.toLowerCase();
62+
}
63+
64+
async function findDocsYml(productDir) {
65+
// Try <dir>.yml, docs.yml, or any .yml in the product dir
66+
const files = await listDir(`fern/products/${productDir}`);
67+
const candidates = [
68+
`${productDir}.yml`,
69+
`docs.yml`
70+
];
71+
for (const candidate of candidates) {
72+
if (files.find(f => f.name === candidate)) {
73+
return candidate;
74+
}
75+
}
76+
// fallback: first .yml file
77+
const yml = files.find(f => f.name.endsWith('.yml'));
78+
return yml ? yml.name : null;
79+
}
80+
81+
async function findPageFile(productDir, page) {
82+
// Try to find the .mdx file in pages/ recursively using the API
83+
async function walk(dir) {
84+
console.log(`[DEBUG] Listing directory: ${dir}`);
85+
const items = await listDir(dir);
86+
for (const item of items) {
87+
if (item.type === 'dir') {
88+
const found = await walk(item.path);
89+
if (found) return found;
90+
} else if (item.name.replace(/\.mdx$/, '') === page) {
91+
console.log(`[DEBUG] Found page file: ${item.path} for page: ${page}`);
92+
return item.path;
93+
}
94+
}
95+
return null;
96+
}
97+
return await walk(`fern/products/${productDir}/pages`);
98+
}
99+
100+
async function walkNav(nav, parentSlugs, pages, productDir, canonicalSlug, depth = 0) {
101+
for (const item of nav) {
102+
let sectionSlug = '';
103+
if (item['skip-slug']) {
104+
sectionSlug = '';
105+
console.log(`[DEBUG] [${' '.repeat(depth)}] Skipping slug for section: ${item.section || ''}`);
106+
} else if (item.slug === true && item.section) {
107+
sectionSlug = slugify(item.section);
108+
console.log(`[DEBUG] [${' '.repeat(depth)}] Section with slug:true: ${sectionSlug}`);
109+
} else if (typeof item.slug === 'string') {
110+
sectionSlug = slugify(item.slug);
111+
console.log(`[DEBUG] [${' '.repeat(depth)}] Section with explicit slug: ${sectionSlug}`);
112+
} else if (item.section) {
113+
sectionSlug = slugify(item.section);
114+
console.log(`[DEBUG] [${' '.repeat(depth)}] Section with name: ${sectionSlug}`);
115+
}
116+
const newSlugs = sectionSlug ? [...parentSlugs, sectionSlug] : parentSlugs;
117+
if (item.contents) {
118+
console.log(`[DEBUG] [${' '.repeat(depth)}] Entering section: ${sectionSlug || '(no slug)'} with path: /learn/${[canonicalSlug, ...newSlugs].join('/')}`);
119+
await walkNav(item.contents, newSlugs, pages, productDir, canonicalSlug, depth + 1);
120+
console.log(`[DEBUG] [${' '.repeat(depth)}] Exiting section: ${sectionSlug || '(no slug)'}`);
121+
}
122+
if (item.page) {
123+
let pageSlug = typeof item.slug === 'string' ? slugify(item.slug) : slugify(item.page);
124+
// Only add pageSlug if it's not the same as the last section slug
125+
let urlSegments = ['/learn', canonicalSlug, ...newSlugs];
126+
if (newSlugs[newSlugs.length - 1] !== pageSlug) {
127+
urlSegments.push(pageSlug);
128+
}
129+
const learnUrl = urlSegments.filter(Boolean).join('/');
130+
if (item.path) {
131+
// Remove leading './' if present
132+
let repoPath = item.path.replace(/^\.\//, '');
133+
// Always make it relative to fern/products/<productDir>
134+
if (!repoPath.startsWith('fern/products/')) {
135+
repoPath = `fern/products/${productDir}/${repoPath}`;
136+
}
137+
pages.push({ learnUrl, repoPath });
138+
console.log(`[DEBUG] [${' '.repeat(depth)}] Mapping: ${learnUrl} → ${repoPath}`);
139+
} else {
140+
console.warn(`[DEBUG] [${' '.repeat(depth)}] Skipping page: ${item.page} in product: ${productDir} (missing path)`);
141+
}
142+
}
143+
}
144+
}
145+
146+
async function main() {
147+
// Step 1: Find and parse the root docs.yml
148+
const rootDocsYmlContent = await getFileContent('fern/docs.yml');
149+
if (!rootDocsYmlContent) {
150+
console.error('Could not find fern/docs.yml in the repo.');
151+
process.exit(1);
152+
}
153+
const rootDocsYml = yaml.load(rootDocsYmlContent);
154+
155+
// Step 2: Get the root URL subpath (e.g., /learn)
156+
let rootUrlSubpath = '';
157+
if (rootDocsYml.url) {
158+
const url = rootDocsYml.url;
159+
const match = url.match(/https?:\/\/[^/]+(\/.*)/);
160+
rootUrlSubpath = match ? match[1].replace(/\/$/, '') : '';
161+
console.log(`[DEBUG] Root URL subpath: ${rootUrlSubpath}`);
162+
}
163+
if (!rootUrlSubpath) rootUrlSubpath = '/learn';
164+
console.log(`[DEBUG] rootUrlSubpath: "${rootUrlSubpath}"`);
165+
166+
// Step 3: Parse products from root docs.yml
167+
const products = rootDocsYml.products || [];
168+
let rootMappingLines = ['## Product Root Directories', ''];
169+
let slugToDir = {};
170+
let allPages = [];
171+
for (const product of products) {
172+
if (!product.path || !product.slug) {
173+
console.warn(`[DEBUG] Skipping product with missing path or slug: ${JSON.stringify(product)}`);
174+
continue;
175+
}
176+
// product.path is like ./products/openapi-def/openapi-def.yml
177+
const productDir = product.path.split('/')[2];
178+
const productYmlPath = product.path.replace('./', 'fern/');
179+
const ymlContent = await getFileContent(productYmlPath);
180+
if (!ymlContent) {
181+
console.warn(`[DEBUG] Could not fetch product YAML: ${productYmlPath}`);
182+
continue;
183+
}
184+
const productYml = yaml.load(ymlContent);
185+
const canonicalSlug = slugify(product.slug);
186+
slugToDir[canonicalSlug] = productDir;
187+
rootMappingLines.push(`${canonicalSlug}: ${productDir}`);
188+
console.log(`[DEBUG] Product: ${productDir}, Slug: ${canonicalSlug}, YAML: ${productYmlPath}`);
189+
if (productYml && productYml.navigation) {
190+
await walkNav(productYml.navigation, [], allPages, productDir, canonicalSlug);
191+
} else {
192+
console.warn(`[DEBUG] No navigation found in ${productYmlPath}`);
193+
}
194+
}
195+
rootMappingLines.push('');
196+
197+
let lines = [
198+
'# Fern URL Mappings',
199+
'',
200+
`Generated on: ${new Date().toISOString()}`,
201+
'',
202+
...rootMappingLines,
203+
'## Products',
204+
''
205+
];
206+
let total = 0;
207+
for (const slug in slugToDir) {
208+
lines.push(`## ${slug.charAt(0).toUpperCase() + slug.slice(1)}`);
209+
lines.push('');
210+
const pages = allPages.filter(p => p.learnUrl.startsWith(`/learn/${slug}/`));
211+
if (pages.length === 0) {
212+
lines.push('_No .mdx files found for this product._');
213+
}
214+
for (const { learnUrl, repoPath } of pages) {
215+
lines.push(`- \`${learnUrl}\` → \`${repoPath}\``);
216+
console.log(`[DEBUG] Mapping: ${learnUrl} → ${repoPath}`);
217+
total++;
218+
}
219+
lines.push('');
220+
}
221+
lines[3] = `Total mappings: ${total}`;
222+
fs.writeFileSync(OUTPUT_FILE, lines.join('\n'), 'utf-8');
223+
console.log(`Wrote ${total} mappings to ${OUTPUT_FILE}`);
224+
if (total === 0) {
225+
console.warn('Warning: No mappings were generated. Check your repo structure and permissions.');
226+
}
227+
}
228+
229+
main();

‎.github/scripts/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
"dependencies": {
77
"@octokit/rest": "^20.0.2",
88
"@turbopuffer/turbopuffer": "^0.10.14",
9-
"node-fetch": "^3.3.2",
109
"js-yaml": "^4.1.0"
1110
},
1211
"engines": {

‎.github/workflows/fern-scribe.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ jobs:
3232
cd .github/scripts
3333
npm install
3434
35+
# --- NEW STEP: Generate my-mappings.md ---
36+
- name: Generate Fern URL Mappings
37+
run: |
38+
cd .github/scripts
39+
node generate-mappings.js
40+
# -----------------------------------------
41+
3542
- name: Run Fern Scribe
3643
env:
3744
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎fern/docs.yml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,6 @@ products:
4242
image: ./images/product-switcher/product-switcher-askfern-light.png
4343
slug: ask-fern
4444
subtitle: Let users find answers in your documentation instantly
45-
46-
47-
# - display-name: API Definition
48-
# path: ./products/api-definition/api-definition.yml
49-
# icon: fa-regular fa-book
50-
# image: ./images/product-switcher/api-definitions-light.png
51-
# slug: api-definition
5245

5346
- display-name: OpenAPI
5447
path: ./products/openapi-def/openapi-def.yml

‎fern/images/product-switcher/product-switcher-askfern-dark.png‎

-663 Bytes
Loading

‎fern/images/product-switcher/product-switcher-askfern-light.png‎

-76 Bytes
Loading

0 commit comments

Comments
 (0)