diff --git a/packages/commons/docs-loader/src/readonly-docs-loader.ts b/packages/commons/docs-loader/src/readonly-docs-loader.ts index f003c918af..5d4d9ac7b7 100644 --- a/packages/commons/docs-loader/src/readonly-docs-loader.ts +++ b/packages/commons/docs-loader/src/readonly-docs-loader.ts @@ -553,22 +553,58 @@ const getRoot = async ( authConfig: AuthEdgeConfig | undefined, cacheConfig: Required ) => { + const authCacheKey = authState.authed + ? `authed:${authState.user?.roles?.sort().join(",") || "noroles"}` + : "anon"; + console.log(`\nšŸ’¾ [GET_ROOT] Cache MISS for getRoot - generating fresh root`); + console.log(` šŸ”‘ Cache key: [${domainKey}, ${cacheConfig.cacheKeySuffix || '""'}, ${authCacheKey}]`); + console.log(` šŸ‘¤ authState.authed: ${authState.authed}`); + console.log(` āš™ļø authConfig exists: ${!!authConfig}`); + + console.log(`\nšŸŒ² [GET_ROOT] Fetching unpruned root from unsafe_getRootCached`); let root = await unsafe_getRootCached(cacheConfig)(domainKey); + console.log(` āœ… Got unpruned root with id: ${root.id}`); + if (authConfig) { + console.log(`\nšŸ”€ [GET_ROOT] Calling pruneWithAuthState`); + console.log(` šŸ‘¤ authState.authed: ${authState.authed}`); + console.log(` šŸ‘„ authState.user.roles: ${authState.authed ? JSON.stringify(authState.user?.roles || []) : 'N/A'}`); + console.log(` šŸ“‹ authConfig.allowlist: ${authConfig.allowlist?.length || 0}`); + console.log(` šŸ“‹ authConfig.denylist: ${authConfig.denylist?.length || 0}`); + console.log(` šŸ“‹ authConfig.anonymous: ${authConfig.anonymous?.length || 0}`); + root = pruneWithAuthState(authState, authConfig, root); + + console.log(` āœ… [GET_ROOT] Finished pruning, root id: ${root.id}`); + } else { + console.log(`\nāš ļø [GET_ROOT] No authConfig - skipping pruneWithAuthState`); } + FernNavigation.utils.mutableUpdatePointsTo(root); + console.log(`\nāœ… [GET_ROOT] Returning pruned root\n`); return root; }; const getRootCached = (cacheConfig: Required) => cache(async (domainKey: string, authState: AuthState, authConfig: AuthEdgeConfig | undefined) => { - return await unstable_cache( + const authCacheKey = authState.authed + ? `authed:${authState.user?.roles?.sort().join(",") || "noroles"}` + : "anon"; + + console.log(`\nšŸ” [PRUNE_CACHE_CHECK] Checking cache for getRoot`); + console.log(` šŸ”‘ Cache key: [${domainKey}, ${cacheConfig.cacheKeySuffix || '""'}, ${authCacheKey}]`); + console.log(` šŸ‘¤ authState.authed: ${authState.authed}`); + + const result = await unstable_cache( (domainKey: string, authState: AuthState, authConfig: AuthEdgeConfig | undefined) => getRoot(domainKey, authState, authConfig, cacheConfig), [domainKey, cacheConfig.cacheKeySuffix], { tags: [domainKey, "getRoot"] } )(domainKey, authState, authConfig); + + console.log(`\nāœ… [PRUNE_CACHE_RETURN] Returning root (cache hit if no CACHE_MISS log above)\n`); + + return result; }); const getNavigationNode = (cacheConfig: Required) => @@ -1111,17 +1147,35 @@ export const createCachedDocsLoader = async ( await clearKvCache(domainKey); } - const authConfig = options?.skipAuth ? Promise.resolve(undefined) : getAuthConfig(domainKey); + const authConfig = options?.skipAuth + ? (console.log(`\nāš™ļø [AUTH_CONFIG] Using skipAuth mode - no authConfig`), Promise.resolve(undefined)) + : (async () => { + console.log(`\nāš™ļø [AUTH_CONFIG] Fetching authConfig for domainKey: ${domainKey}`); + const config = await getAuthConfig(domainKey); + console.log(` āœ… [AUTH_CONFIG] Fetched authConfig`); + console.log(` šŸ“‹ allowlist: ${config?.allowlist?.length || 0} paths`); + console.log(` šŸ“‹ denylist: ${config?.denylist?.length || 0} paths`); + console.log(` šŸ“‹ anonymous: ${config?.anonymous?.length || 0} paths`); + console.log(` šŸ” auth type: ${config?.type || 'none'}`); + return config; + })(); const metadata = getMetadata(config)(withoutStaging(domainKey)); const getAuthState = options?.skipAuth - ? async (_pathname?: string) => ({ - authed: true as const, - ok: true as const, - user: {}, - partner: "custom" as const - }) + ? async (_pathname?: string) => { + console.log(`\nšŸ”“ [GET_AUTH_STATE] Using skipAuth mode - returning authed=true`); + return { + authed: true as const, + ok: true as const, + user: {}, + partner: "custom" as const + }; + } : cache(async (pathname?: string) => { + console.log(`\nšŸ” [GET_AUTH_STATE] Fetching auth state for domainKey: ${domainKey}`); + console.log(` šŸŽ« fern_token present: ${!!fern_token}`); + console.log(` šŸ“ pathname: ${pathname || 'undefined'}`); + const { getAuthState } = await createGetAuthState( host, domainKey, @@ -1129,7 +1183,14 @@ export const createCachedDocsLoader = async ( await authConfig, await metadata ); - return await getAuthState(pathname); + const authState = await getAuthState(pathname); + + console.log(` āœ… [GET_AUTH_STATE] Result - authed: ${authState.authed}`); + if (authState.authed) { + console.log(` šŸ‘„ roles: ${JSON.stringify(authState.user?.roles || [])}`); + } + + return authState; }); return { @@ -1156,7 +1217,7 @@ export const createCachedDocsLoader = async ( { tags: [domainKey, "endpointByLocator"] } ) ), - getRoot: async () => getRootCached(config)(domainKey, await getAuthState(), await authConfig), + getRoot: async () => getRoot(domainKey, await getAuthState(), await authConfig, config), getNavigationNode: async (id: string) => getNavigationNode(config)(domainKey, id, await getAuthState(), await authConfig), unsafe_getFullRoot: () => unsafe_getRootCached(config)(domainKey), diff --git a/packages/commons/docs-server/src/loadDocsDefinitionFromS3.ts b/packages/commons/docs-server/src/loadDocsDefinitionFromS3.ts index 7ca49dc685..87d53f7ab6 100644 --- a/packages/commons/docs-server/src/loadDocsDefinitionFromS3.ts +++ b/packages/commons/docs-server/src/loadDocsDefinitionFromS3.ts @@ -4,6 +4,7 @@ import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { getSignedUrl as getUncachedSignedUrl } from "@aws-sdk/s3-request-presigner"; import type { FdrAPI } from "@fern-api/fdr-sdk"; import { getS3KeyForV1DocsDefinition } from "@fern-api/fdr-sdk/docs"; +import { writeFile } from "fs/promises"; import { cache } from "react"; import { isLocal } from "./isLocal"; @@ -57,6 +58,17 @@ export const loadDocsDefinitionFromS3 = cache( if (response.ok) { console.debug("Successfully loaded docs definition from S3: ", signedUrl); const json = await response.json(); + + // Write to local file for debugging + try { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `/tmp/docs-definition-${cleanDomain.replace(/[^a-zA-Z0-9]/g, '-')}-${timestamp}.json`; + await writeFile(filename, JSON.stringify(json, null, 2)); + console.log(`šŸ’¾ [DEBUG] Saved docs definition to: ${filename}`); + } catch (writeError) { + console.error("Failed to write docs definition to file:", writeError); + } + return json as FdrAPI.docs.v2.read.LoadDocsForUrlResponse; } throw new Error( diff --git a/packages/commons/docs-server/src/withRbac.ts b/packages/commons/docs-server/src/withRbac.ts index 574216458f..1d06eb71b8 100644 --- a/packages/commons/docs-server/src/withRbac.ts +++ b/packages/commons/docs-server/src/withRbac.ts @@ -76,6 +76,9 @@ function withAllowed Gate>(predicate: T): (...args } export function pruneWithBasicTokenAnonymous(auth: PathnameViewerRules, node: RootNode): RootNode { + console.log(`\nšŸ‘¤ [PRUNE_ANONYMOUS] Starting pruning for ANONYMOUS user`); + console.log(` šŸ“‹ Auth rules - denylist: ${auth.denylist?.length || 0}, allowlist: ${auth.allowlist?.length || 0}, anonymous: ${auth.anonymous?.length || 0}`); + const result = Pruner.from(node) // mark nodes that are authed .authed(withDenied(withBasicTokenAnonymousCheck(auth))) @@ -86,6 +89,7 @@ export function pruneWithBasicTokenAnonymous(auth: PathnameViewerRules, node: Ro throw new Error("Failed to prune navigation tree"); } + console.log(`\nāœ… [PRUNE_ANONYMOUS] Pruning complete for anonymous user\n`); return result; } @@ -107,6 +111,10 @@ export function matchRoles(roles: string[] | "anonymous", filters: string[][]): } export function pruneWithBasicTokenAuthed(auth: PathnameViewerRules, node: RootNode, roles: string[] = []): RootNode { + console.log(`\nšŸ”‘ [PRUNE_AUTHENTICATED] Starting pruning for AUTHENTICATED user`); + console.log(` šŸ‘„ User roles: ${JSON.stringify(roles)}`); + console.log(` šŸ“‹ Auth rules - denylist: ${auth.denylist?.length || 0}, allowlist: ${auth.allowlist?.length || 0}, anonymous: ${auth.anonymous?.length || 0}`); + const result = Pruner.from(node) // apply rbac .keep(withAllowed(rbacViewGate(roles, true))) @@ -121,10 +129,12 @@ export function pruneWithBasicTokenAuthed(auth: PathnameViewerRules, node: RootN throw new Error("Failed to prune navigation tree"); } + console.log(`\nāœ… [PRUNE_AUTHENTICATED] Pruning complete for authenticated user\n`); return result; } export function pruneWithAuthState(authState: AuthState, authConfig: PathnameViewerRules, node: RootNode): RootNode { + console.log(`\nšŸš€ [PRUNE_WITH_AUTH_STATE] authState.authed = ${authState.authed}`); return authState.authed ? pruneWithBasicTokenAuthed(authConfig, node, authState.user.roles) : pruneWithBasicTokenAnonymous(authConfig, node); diff --git a/packages/fdr-sdk/src/navigation/utils/pruneNavigationTree.ts b/packages/fdr-sdk/src/navigation/utils/pruneNavigationTree.ts index b7e0bac99d..3dd5736d0e 100644 --- a/packages/fdr-sdk/src/navigation/utils/pruneNavigationTree.ts +++ b/packages/fdr-sdk/src/navigation/utils/pruneNavigationTree.ts @@ -57,10 +57,24 @@ export class Pruner { const unauthedParents = new Set(); + console.log(`\nšŸ” [PRUNER_AUTHED] Starting authed pruning pass`); + let authedCount = 0; + let unauthedCount = 0; + // step 1. mark nodes as authed if the match the predicate FernNavigation.traverseBF(this.tree, (node, parents) => { if (FernNavigation.hasMetadata(node)) { - node.authed = predicate(node, parents) ? true : undefined; + const predicateResult = predicate(node, parents); + node.authed = predicateResult ? true : undefined; + + if (predicateResult) { + authedCount++; + // console.log(` šŸ”’ [PRUNER_AUTHED] Setting node.authed=true - title: "${node.title}", slug: ${node.slug}, id: ${node.id}`); + } else { + unauthedCount++; + // console.log(` āœ… [PRUNER_AUTHED] Setting node.authed=undefined - title: "${node.title}", slug: ${node.slug}, id: ${node.id}`); + } + if (!node.authed) { for (const parent of parents) { unauthedParents.add(parent); @@ -69,17 +83,24 @@ export class Pruner { } }); + console.log(`\nšŸ“Š [PRUNER_AUTHED] Step 1 complete - ${authedCount} nodes marked as authed, ${unauthedCount} nodes unmarked\n`); + // step 2. remove auth flag on edge nodes that contain children that are unauthed // this helps us improve the rendering of the navigation tree and avoid deleting nodes that contain // children that can be visited. // Note: sections with overview pages are skipped here because the pages need to be filtered out // by the mutableDeleteChild function, which relies on the authed flag in a separate pass. + let edgeNodeCount = 0; for (const parent of unauthedParents) { if (FernNavigation.hasMetadata(parent) && !FernNavigation.isPage(parent)) { + console.log(` šŸŒ³ [PRUNER_AUTHED] Clearing authed flag on edge node parent - title: "${parent.title}", slug: ${parent.slug}, id: ${parent.id}`); parent.authed = undefined; + edgeNodeCount++; } } + console.log(`\nšŸ“Š [PRUNER_AUTHED] Step 2 complete - ${edgeNodeCount} edge node parents cleared\n`); + return this; } diff --git a/packages/fern-docs/bundle/src/app/[host]/[domain]/api/fern-docs/llms.txt/route.ts b/packages/fern-docs/bundle/src/app/[host]/[domain]/api/fern-docs/llms.txt/route.ts index 877375868b..8f09c5d648 100644 --- a/packages/fern-docs/bundle/src/app/[host]/[domain]/api/fern-docs/llms.txt/route.ts +++ b/packages/fern-docs/bundle/src/app/[host]/[domain]/api/fern-docs/llms.txt/route.ts @@ -5,7 +5,6 @@ import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers"; import { isNonNullish, withDefaultProtocol } from "@fern-api/ui-core-utils"; import { getAuthEdgeConfig, getEdgeFlags } from "@fern-docs/edge-config"; -import { unstable_cacheTag } from "next/cache"; import { cookies } from "next/headers"; import { notFound } from "next/navigation"; import { type NextRequest, NextResponse } from "next/server"; @@ -65,7 +64,7 @@ export async function GET( let markdownProcessingMs = 0; try { - const { content: streamedContent, timingStats } = await getLlmsTxtStreaming( + const { timingStats } = await getLlmsTxtStreaming( host, domain, path, @@ -123,6 +122,17 @@ async function getLlmsTxtStreaming( ): Promise<{ content: string; timingStats: any }> { const loader = await createCachedDocsLoader(host, domain, fernToken); + // Debug: Check auth state + const authConfig = await loader.getAuthConfig(); + const authState = await loader.getAuthState?.(); + console.log(`\n====== [LLMS_TXT_REQUEST:${domain}] ======`); + console.log(`šŸ“ fernToken present: ${!!fernToken}`); + console.log(`āš™ļø authConfig exists: ${!!authConfig}`); + console.log(`šŸ‘¤ authState.authed: ${authState?.authed}`); + console.log(`šŸŽ­ authState.user.roles: ${authState?.authed ? JSON.stringify(authState.user?.roles || []) : 'N/A (not authed)'}`); + console.log(`šŸ”‘ authCacheKey would be: ${authState?.authed ? `authed:${authState.user?.roles?.sort().join(",") || "noroles"}` : "anon"}`); + console.log(`=======================================\n`); + // Time the root retrieval and section root const rootStartTime = performance.now(); const root = getSectionRoot(await loader.getRoot(), path); @@ -150,7 +160,20 @@ async function getLlmsTxtStreaming( }[] = []; const landingPage = getLandingPage(root); - const markdown = landingPage != null ? await getMarkdownForPath(landingPage, loader) : undefined; + + // Check if landing page is authed or hidden before including its markdown + const shouldIncludeLandingPage = + landingPage != null && + (!FernNavigation.hasMetadata(landingPage) || (!landingPage.authed && !landingPage.hidden)); + + const markdown = shouldIncludeLandingPage ? await getMarkdownForPath(landingPage, loader) : undefined; + + if (landingPage?.authed) { + console.log(`šŸ”’ [LLMS_TXT_SKIP:${domain}] AUTHED landing page - title: "${landingPage.title}", id: ${landingPage.id}`); + } + if (landingPage?.hidden) { + console.log(`šŸš« [LLMS_TXT_SKIP:${domain}] HIDDEN landing page - title: "${landingPage.title}", id: ${landingPage.id}`); + } const header = markdown?.content ?? `# ${root.title}`; onChunk(header + "\n\n"); @@ -166,7 +189,12 @@ async function getLlmsTxtStreaming( // if the node is hidden or authed, don't include it in the list // TODO: include "hidden" nodes in `llms-full.txt` if (FernNavigation.hasMetadata(node)) { - if (node.hidden || node.authed) { + if (node.hidden) { + console.log(`šŸš« [LLMS_TXT_SKIP:${domain}] HIDDEN node - title: "${node.title}", slug: ${node.slug}, id: ${node.id}`); + return SKIP; + } + if (node.authed) { + console.log(`šŸ”’ [LLMS_TXT_SKIP:${domain}] AUTHED node - title: "${node.title}", slug: ${node.slug}, id: ${node.id}, authed: ${node.authed}`); return SKIP; } } @@ -206,6 +234,11 @@ async function getLlmsTxtStreaming( return CONTINUE; }); + console.log(`\nšŸ“Š [LLMS_TXT_SUMMARY:${domain}] Traversal complete`); + console.log(` šŸ“„ Pages found: ${pageInfos.length}`); + console.log(` šŸ”Œ API endpoints found: ${endpointPageInfos.length}`); + console.log(` ā±ļø Root retrieval: ${(rootEndTime - rootStartTime).toFixed(2)}ms\n`); + const markdownStartTime = performance.now(); if (pageInfos.length > 0) { diff --git a/packages/fern-docs/edge-config/src/getAuthEdgeConfig.ts b/packages/fern-docs/edge-config/src/getAuthEdgeConfig.ts index 003252eaf7..594600795f 100644 --- a/packages/fern-docs/edge-config/src/getAuthEdgeConfig.ts +++ b/packages/fern-docs/edge-config/src/getAuthEdgeConfig.ts @@ -87,17 +87,45 @@ export async function getApiKeyInjectionDemoConfig(currentDomain: string): Promi } async function getRecord(currentDomain: string, key: string): Promise { + console.log(`\nšŸŒ [EDGE_CONFIG] Fetching record from edge config`); + console.log(` šŸ”‘ key: "${key}"`); + console.log(` šŸŒ currentDomain: "${currentDomain}"`); + console.log(` šŸŒ withoutStaging: "${withoutStaging(currentDomain)}"`); + const domainToTokenConfigMap = await getEdge>(key); + + if (!domainToTokenConfigMap) { + console.log(` āŒ [EDGE_CONFIG] No data found for key "${key}"`); + return; + } + + console.log(` āœ… [EDGE_CONFIG] Found edge config data with ${Object.keys(domainToTokenConfigMap).length} domains`); + console.log(` šŸ” Looking up: "${currentDomain}"`); + const toRet = domainToTokenConfigMap?.[currentDomain] ?? domainToTokenConfigMap?.[withoutStaging(currentDomain)]; + if (toRet != null) { + console.log(` āœ… [EDGE_CONFIG] Found config for domain`); + console.log(` šŸ” Raw config keys: ${Object.keys(toRet).join(", ")}`); + const config = AuthEdgeConfigSchema.safeParse(toRet); // if the config is present, it should be valid. // if it's malformed, custom auth for this domain will not work and may leak docs to the public. if (!config.success) { - console.error(`Could not parse AuthEdgeConfigSchema for ${currentDomain}`, config.error); + console.error(` āŒ [EDGE_CONFIG] Could not parse AuthEdgeConfigSchema for ${currentDomain}`, config.error); // TODO: sentry + return; } + + console.log(` āœ… [EDGE_CONFIG] Successfully parsed AuthEdgeConfig`); + console.log(` šŸ” type: ${config.data.type}`); + console.log(` šŸ“‹ allowlist: ${config.data.allowlist?.length || 0} paths`); + console.log(` šŸ“‹ denylist: ${config.data.denylist?.length || 0} paths`); + console.log(` šŸ“‹ anonymous: ${config.data.anonymous?.length || 0} paths`); + return config.data; } + + console.log(` āŒ [EDGE_CONFIG] No config found for domain "${currentDomain}" or "${withoutStaging(currentDomain)}"`); return; }