Merge pull request #11 from multivmlabs/fix/sitemap-recursive-public-dirs

fix: exclude build dirs from sitemap and llms-txt crawling

Author: rubenmarcus

src/core/llms-txt.ts

@@ -13,7 +13,8 @@ function collectMarkdownFiles(dir: string, base: string = dir): MarkdownFile[] {
       const fullPath = join(dir, entry);
       const stat = statSync(fullPath);
 
-      if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') {
+      const SKIP_DIRS = new Set(['node_modules', 'public', 'dist', '.next', '.nuxt', '.output', '.open-next', 'coverage', '__tests__', '__mocks__']);
+      if (stat.isDirectory() && !entry.startsWith('.') && !SKIP_DIRS.has(entry)) {
         files.push(...collectMarkdownFiles(fullPath, base));
       } else if (stat.isFile() && (extname(entry) === '.md' || extname(entry) === '.mdx')) {
         const content = readFileSync(fullPath, 'utf-8');

src/core/sitemap.ts

@@ -12,7 +12,8 @@ function collectUrls(dir: string, config: ResolvedAeoConfig, base: string = dir)
       const fullPath = join(dir, entry);
       const stat = statSync(fullPath);
 
-      if (stat.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') {
+      const SKIP_DIRS = new Set(['node_modules', 'public', 'dist', '.next', '.nuxt', '.output', '.open-next', 'coverage', '__tests__', '__mocks__']);
+      if (stat.isDirectory() && !entry.startsWith('.') && !SKIP_DIRS.has(entry)) {
         urls.push(...collectUrls(fullPath, config, base));
       } else if (stat.isFile() && (extname(entry) === '.md' || extname(entry) === '.mdx' || extname(entry) === '.html')) {
         const relativePath = relative(base, fullPath);

src/plugins/next.ts

@@ -99,9 +99,11 @@ export function withAeo(nextConfig: NextAeoConfig = {}): Record<string, any> {
           }
         }
 
-        // Resolve contentDir: prefer user-specified, then src/, then project root
+        // Resolve contentDir: prefer user-specified, then known content dirs, then src/
+        // Never fall back to project root — it causes recursive crawling of public/, .next/, etc.
         const contentDir = aeoOptions.contentDir
-          || (existsSync(join(projectRoot, 'src')) ? join(projectRoot, 'src') : projectRoot);
+          || [join(projectRoot, 'content'), join(projectRoot, 'docs'), join(projectRoot, 'src')].find(d => existsSync(d))
+          || join(projectRoot, 'content');
 
         const resolvedConfig = resolveConfig({
           ...aeoOptions,
@@ -247,7 +249,8 @@ export async function postBuild(config: AeoConfig = {}): Promise<void> {
   }
 
   const contentDir = config.contentDir
-    || (existsSync(join(projectRoot, 'src')) ? join(projectRoot, 'src') : projectRoot);
+    || [join(projectRoot, 'content'), join(projectRoot, 'docs'), join(projectRoot, 'src')].find(d => existsSync(d))
+    || join(projectRoot, 'content');
 
   const resolvedConfig = resolveConfig({
     ...config,