feat: implement sitemap generation in TypeScript and remove XML file (#6206)

Co-authored-by: daniel-lxs <[email protected]>
Co-authored-by: Daniel <[email protected]>

Author: abumalick

apps/web-roo-code/.env.example

@@ -6,3 +6,7 @@ NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com
 # Basin Form Endpoint for Static Form Submissions
 # Replace this with your actual Basin form endpoint (e.g., https://usebasin.com/f/your-form-id)
 NEXT_PUBLIC_BASIN_ENDPOINT=https://usebasin.com/f/your-form-id-here
+
+# Site URL Configuration
+# Used for generating absolute URLs in sitemap and other contexts
+NEXT_PUBLIC_SITE_URL=https://roocode.com

apps/web-roo-code/.gitignore

@@ -40,3 +40,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# generated files
+/public/sitemap*.xml
+/public/robots.txt

apps/web-roo-code/.npmrc

@@ -0,0 +1 @@
+enable-pre-post-scripts=true

apps/web-roo-code/next-sitemap.config.cjs

@@ -0,0 +1,57 @@
+/** @type {import('next-sitemap').IConfig} */
+module.exports = {
+  siteUrl: process.env.NEXT_PUBLIC_SITE_URL || 'https://roocode.com',
+  generateRobotsTxt: true,
+  generateIndexSitemap: false, // We don't need index sitemap for a small site
+  changefreq: 'monthly',
+  priority: 0.7,
+  sitemapSize: 5000,
+  exclude: [
+    '/api/*',
+    '/server-sitemap-index.xml',
+    '/404',
+    '/500',
+    '/_not-found',
+  ],
+  robotsTxtOptions: {
+    policies: [
+      {
+        userAgent: '*',
+        allow: '/',
+      },
+    ],
+    additionalSitemaps: [
+      // Add any additional sitemaps here if needed in the future
+    ],
+  },
+  // Custom transform function to set specific priorities and change frequencies
+  transform: async (config, path) => {
+    // Set custom priority for specific pages
+    let priority = config.priority;
+    let changefreq = config.changefreq;
+    
+    if (path === '/') {
+      priority = 1.0;
+      changefreq = 'yearly';
+    } else if (path === '/enterprise' || path === '/evals') {
+      priority = 0.8;
+      changefreq = 'monthly';
+    } else if (path === '/privacy' || path === '/terms') {
+      priority = 0.5;
+      changefreq = 'yearly';
+    }
+    
+    return {
+      loc: path,
+      changefreq,
+      priority,
+      lastmod: config.autoLastmod ? new Date().toISOString() : undefined,
+      alternateRefs: config.alternateRefs ?? [],
+    };
+  },
+  additionalPaths: async (config) => {
+    // Add any additional paths that might not be automatically discovered
+    // This is useful for dynamic routes or API-generated pages
+    return [];
+  },
+};

apps/web-roo-code/package.json

@@ -7,6 +7,7 @@
 		"check-types": "tsc --noEmit",
 		"dev": "next dev",
 		"build": "next build",
+		"postbuild": "next-sitemap --config next-sitemap.config.cjs",
 		"start": "next start",
 		"clean": "rimraf .next .turbo"
 	},
@@ -42,6 +43,7 @@
 		"@types/react": "^18.3.23",
 		"@types/react-dom": "^18.3.7",
 		"autoprefixer": "^10.4.21",
+		"next-sitemap": "^4.2.3",
 		"postcss": "^8.5.4",
 		"tailwindcss": "^3.4.17"
 	}

apps/web-roo-code/src/app/sitemap.xml

@@ -7,7 +7,8 @@
 		"src/activate/**",
 		"src/workers/countTokens.ts",
 		"src/extension.ts",
-		"scripts/**"
+		"scripts/**",
+		"apps/web-roo-code/next-sitemap.config.cjs"
 	],
 	"workspaces": {
 		"src": {