Chore: Remove sdk reference proxying (#270)

<!-- CURSOR_SUMMARY -->
> [!NOTE]
> **Medium Risk**
> Changes rewrite/sitemap behavior for `/docs/sdk-reference` paths,
which can affect production routing and SEO indexing if misconfigured.
Test coverage was added/updated, but incorrect domain/path config could
still cause broken docs links or missing sitemap entries.
> 
> **Overview**
> Stops treating the SDK reference as a separately proxied site: removes
`SDK_REFERENCE_DOMAIN` and its middleware rewrite rule, relying on the
main `DOCUMENTATION_DOMAIN` for `/docs/*` rewrites.
> 
> Updates sitemap generation to **not include** SDK reference URLs by
introducing `SITEMAP_EXCLUDE_CONFIG` and filtering out excluded path
prefixes when merging external sitemaps; adds a unit test asserting
`/docs/sdk-reference*` is omitted. Integration tests were updated to
assert middleware rewrites send docs traffic (including
`/docs/sdk-reference/*`) to the docs host.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
5181544. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

Author: ben-fornefeld

src/__test__/integration/proxy.test.ts

@@ -3,6 +3,7 @@ import { NextRequest, NextResponse } from 'next/server'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { AUTH_URLS, PROTECTED_URLS } from '@/configs/urls'
 import { proxy } from '@/proxy'
+import { DOCUMENTATION_DOMAIN } from '../../../next.config.mjs'
 
 // mock supabase auth
 vi.mock('@supabase/ssr', () => ({
@@ -46,7 +47,7 @@ vi.mock('next/server', async () => {
     return response
   })
 
-  const mockNext = vi.fn((init?: { request?: NextRequest }) => {
+  const mockNext = vi.fn((_init?: { request?: NextRequest }) => {
     const response = new actual.NextResponse()
 
     Object.defineProperty(response, 'cookies', {
@@ -61,7 +62,7 @@ vi.mock('next/server', async () => {
   })
 
   const mockRewrite = vi.fn(
-    (url: URL | string, init?: { request?: { headers?: Headers } }) => {
+    (_url: URL | string, _init?: { request?: { headers?: Headers } }) => {
       const response = new actual.NextResponse()
 
       Object.defineProperty(response, 'headers', {
@@ -140,7 +141,7 @@ describe('Proxy Integration Tests', () => {
         path: PROTECTED_URLS.DASHBOARD,
       })
 
-      const response = await proxy(request)
+      await proxy(request)
 
       expect(NextResponse.redirect).toHaveBeenCalled()
       const redirectUrl = vi
@@ -154,7 +155,7 @@ describe('Proxy Integration Tests', () => {
         path: AUTH_URLS.SIGN_IN,
       })
 
-      const response = await proxy(request)
+      await proxy(request)
 
       expect(NextResponse.redirect).toHaveBeenCalled()
       const redirectUrl = vi
@@ -168,7 +169,7 @@ describe('Proxy Integration Tests', () => {
         path: '/dashboard/team-123/sandboxes',
       })
 
-      const response = await proxy(request)
+      await proxy(request)
 
       expect(NextResponse.redirect).not.toHaveBeenCalled()
       expect(NextResponse.next).toHaveBeenCalled()
@@ -191,7 +192,7 @@ describe('Proxy Integration Tests', () => {
         path: '/',
       })
 
-      const response = await proxy(request)
+      await proxy(request)
 
       expect(NextResponse.redirect).not.toHaveBeenCalled()
       expect(NextResponse.next).toHaveBeenCalled()
@@ -215,10 +216,44 @@ describe('Proxy Integration Tests', () => {
         path: '/',
       })
 
-      const response = await proxy(request)
+      await proxy(request)
 
       // should not crash, should return next()
       expect(NextResponse.next).toHaveBeenCalled()
     })
   })
+
+  describe('Middleware Rewrites', () => {
+    it('rewrites supported docs paths to the docs host', async () => {
+      const request = createMockRequest({
+        path: '/docs/quickstart',
+      })
+
+      await proxy(request)
+
+      expect(NextResponse.rewrite).toHaveBeenCalledTimes(1)
+      expect(NextResponse.rewrite).toHaveBeenCalledWith(
+        expect.objectContaining({
+          href: `https://${DOCUMENTATION_DOMAIN}/docs/quickstart`,
+        }),
+        expect.anything()
+      )
+    })
+
+    it('rewrites the sdk reference subtree to the docs host', async () => {
+      const request = createMockRequest({
+        path: '/docs/sdk-reference/python',
+      })
+
+      await proxy(request)
+
+      expect(NextResponse.rewrite).toHaveBeenCalledTimes(1)
+      expect(NextResponse.rewrite).toHaveBeenCalledWith(
+        expect.objectContaining({
+          href: `https://${DOCUMENTATION_DOMAIN}/docs/sdk-reference/python`,
+        }),
+        expect.anything()
+      )
+    })
+  })
 })

src/__test__/unit/sitemap.test.ts

@@ -0,0 +1,80 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { LANDING_PAGE_DOMAIN } from '@/configs/rewrites'
+import { DOCUMENTATION_DOMAIN } from '../../../next.config.mjs'
+
+vi.mock('@/configs/flags', () => ({
+  ALLOW_SEO_INDEXING: true,
+}))
+
+import { constructSitemap } from '@/app/sitemap'
+
+describe('constructSitemap', () => {
+  const fetchMock = vi.fn<typeof fetch>()
+
+  beforeEach(() => {
+    vi.stubGlobal('fetch', fetchMock)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+    vi.resetAllMocks()
+  })
+
+  it('excludes sdk reference paths from the merged sitemap', async () => {
+    fetchMock.mockImplementation(async (input) => {
+      const url = input instanceof URL ? input.toString() : String(input)
+
+      if (url === `https://${LANDING_PAGE_DOMAIN}/sitemap.xml`) {
+        return new Response(
+          `
+            <urlset>
+              <url>
+                <loc>https://${LANDING_PAGE_DOMAIN}/pricing</loc>
+              </url>
+            </urlset>
+          `,
+          {
+            status: 200,
+            headers: {
+              'Content-Type': 'application/xml',
+            },
+          }
+        )
+      }
+
+      if (url === `https://${DOCUMENTATION_DOMAIN}/sitemap.xml`) {
+        return new Response(
+          `
+            <urlset>
+              <url>
+                <loc>https://${DOCUMENTATION_DOMAIN}/docs/quickstart</loc>
+              </url>
+              <url>
+                <loc>https://${DOCUMENTATION_DOMAIN}/docs/sdk-reference</loc>
+              </url>
+              <url>
+                <loc>https://${DOCUMENTATION_DOMAIN}/docs/sdk-reference/python</loc>
+              </url>
+            </urlset>
+          `,
+          {
+            status: 200,
+            headers: {
+              'Content-Type': 'application/xml',
+            },
+          }
+        )
+      }
+
+      throw new Error(`Unexpected sitemap fetch: ${url}`)
+    })
+
+    const sitemap = await constructSitemap()
+    const urls = sitemap.map((entry) => entry.url)
+
+    expect(urls).toContain('https://e2b.dev/pricing')
+    expect(urls).toContain('https://e2b.dev/docs/quickstart')
+    expect(urls).not.toContain('https://e2b.dev/docs/sdk-reference')
+    expect(urls).not.toContain('https://e2b.dev/docs/sdk-reference/python')
+  })
+})

src/app/(rewrites)/[[...slug]]/route.ts

@@ -43,7 +43,7 @@ export async function GET(request: NextRequest): Promise<Response> {
   const { config, rule } = getRewriteForPath(url.pathname, 'route')
 
   if (config) {
-    if (rule && rule.pathPreprocessor) {
+    if (rule?.pathPreprocessor) {
       url.pathname = rule.pathPreprocessor(url.pathname)
     }
     updateUrlHostname(config.domain)
@@ -131,7 +131,7 @@ export async function generateStaticParams() {
           if (isIndex && rule.path === '/') {
             return true
           }
-          if (pathname === rule.path || pathname.startsWith(rule.path + '/')) {
+          if (pathname === rule.path || pathname.startsWith(`${rule.path}/`)) {
             return true
           }
           return false

src/app/sitemap.ts

@@ -10,11 +10,8 @@
 import { XMLParser } from 'fast-xml-parser'
 import type { MetadataRoute } from 'next'
 import { ALLOW_SEO_INDEXING } from '@/configs/flags'
-import {
-  LANDING_PAGE_DOMAIN,
-  ROUTE_REWRITE_CONFIG,
-  SDK_REFERENCE_DOMAIN,
-} from '@/configs/rewrites'
+import { LANDING_PAGE_DOMAIN, ROUTE_REWRITE_CONFIG } from '@/configs/rewrites'
+import { SITEMAP_EXCLUDE_CONFIG } from '@/configs/sitemap'
 import type { DomainConfig } from '@/types/rewrites.types'
 import { DOCUMENTATION_DOMAIN } from '../../next.config.mjs'
 
@@ -56,12 +53,6 @@ const sites: Site[] = [
     changeFrequency: 'weekly',
     baseUrl: 'https://e2b.dev',
   },
-  {
-    sitemapUrl: `https://${SDK_REFERENCE_DOMAIN}/sitemap.xml`,
-    priority: 0.7,
-    changeFrequency: 'weekly',
-    baseUrl: 'https://e2b.dev',
-  },
   {
     sitemapUrl: `https://${DOCUMENTATION_DOMAIN}/sitemap.xml`,
     priority: 0.9,
@@ -119,6 +110,10 @@ async function getXmlData(url: string): Promise<Sitemap> {
   }
 }
 
+function matchesPathPrefix(path: string, prefix: string): boolean {
+  return path === prefix || path.startsWith(`${prefix}/`)
+}
+
 /**
  * Finds the corresponding rewrite configuration for a given site based on its sitemap URL domain.
  *
@@ -138,6 +133,22 @@ function findRewriteConfig(site: Site): DomainConfig | undefined {
   }
 }
 
+function findExcludedPathPrefixes(site: Site): string[] {
+  try {
+    const siteDomain = new URL(site.sitemapUrl).hostname
+    return (
+      SITEMAP_EXCLUDE_CONFIG.find((config) => config.domain === siteDomain)
+        ?.excludedPathPrefixes ?? []
+    )
+  } catch (error) {
+    console.error(
+      `Error parsing sitemapUrl ${site.sitemapUrl} to find sitemap exclusions:`,
+      error
+    )
+    return []
+  }
+}
+
 /**
  * Processes a site's sitemap and converts it to Next.js sitemap format
  * Applies path preprocessing based on ROUTE_REWRITE_CONFIG.
@@ -148,6 +159,7 @@ function findRewriteConfig(site: Site): DomainConfig | undefined {
 async function getSitemap(site: Site): Promise<MetadataRoute.Sitemap> {
   const data = await getXmlData(site.sitemapUrl)
   const rewriteConfig = findRewriteConfig(site) // Find the rewrite config for this site
+  const excludedPathPrefixes = findExcludedPathPrefixes(site)
 
   if (!data || !site.baseUrl) {
     // Ensure baseUrl is defined, as it's crucial for constructing final URLs
@@ -170,6 +182,14 @@ async function getSitemap(site: Site): Promise<MetadataRoute.Sitemap> {
       const rewrittenPathname = originalUrl.pathname
       let finalPathname = rewrittenPathname // Default to the path from the sitemap
 
+      if (
+        excludedPathPrefixes.some((prefix) =>
+          matchesPathPrefix(rewrittenPathname, prefix)
+        )
+      ) {
+        return null
+      }
+
       // Find the corresponding original path if a preprocessor was involved
       if (rewriteConfig) {
         for (const rule of rewriteConfig.rules) {

src/configs/rewrites.ts

@@ -1,9 +1,7 @@
 import type { DomainConfig } from '@/types/rewrites.types'
 
 export const LANDING_PAGE_DOMAIN = 'www.e2b-landing-page.com'
-export const SDK_REFERENCE_DOMAIN = 'e2b-docs.vercel.app'
 
-// NOTE: DOCUMENTATION_DOMAIN has to be defined in next.config.mjs, such that we are able to use it there
 import { DOCUMENTATION_DOMAIN } from '../../next.config.mjs'
 
 // Currently we have two locations for rewrites to happen.
@@ -56,10 +54,6 @@ export const ROUTE_REWRITE_CONFIG: DomainConfig[] = [
  * control over the request/response cycle.
  */
 export const MIDDLEWARE_REWRITE_CONFIG: DomainConfig[] = [
-  {
-    domain: SDK_REFERENCE_DOMAIN,
-    rules: [{ path: '/docs/sdk-reference' }],
-  },
   {
     domain: DOCUMENTATION_DOMAIN,
     rules: [

src/configs/sitemap.ts

@@ -0,0 +1,11 @@
+import { DOCUMENTATION_DOMAIN } from '../../next.config.mjs'
+
+export const SITEMAP_EXCLUDE_CONFIG: Array<{
+  domain: string
+  excludedPathPrefixes: string[]
+}> = [
+  {
+    domain: DOCUMENTATION_DOMAIN,
+    excludedPathPrefixes: ['/docs/sdk-reference'],
+  },
+]

src/lib/utils/rewrites.ts

@@ -20,15 +20,17 @@ function getRewriteForPath(
 
     const matchingRule = domainConfig.rules.find((rule) => {
       if (isIndex && rule.path === '/') {
-        return rule
+        return true
       }
 
       if (
         rule.path !== '/' &&
-        (path === rule.path || path.startsWith(rule.path + '/'))
+        (path === rule.path || path.startsWith(`${rule.path}/`))
       ) {
-        return rule
+        return true
       }
+
+      return false
     })
 
     if (matchingRule) {