fix(sitemap): add lastmod tags & remove duplicate API ref URLs

Author: khadni

astro.config.ts

@@ -11,6 +11,20 @@ import { RehypePlugins } from "@astrojs/markdown-remark"
 import yaml from "@rollup/plugin-yaml"
 import { ccipRedirects } from "./src/config/redirects/ccip"
 import trailingSlashMiddleware from "./src/integrations/trailing-slash-middleware"
+import { getLastModifiedDate } from "./src/utils/lastModified"
+import redirectsJson from "./src/features/redirects/redirects.json"
+
+// Prepare set of redirect source URLs to exclude from sitemap
+// This prevents duplicate entries and ensures only canonical URLs are indexed
+const redirectSources = new Set(
+  redirectsJson.redirects
+    .map((r) => r.source)
+    .filter((source) => source)
+    .map((source) => {
+      const normalized = source.startsWith("/") ? source : `/${source}`
+      return normalized.endsWith("/") ? normalized.slice(0, -1) : normalized
+    })
+)
 
 // https://astro.build/config
 export default defineConfig({
@@ -30,7 +44,41 @@ export default defineConfig({
     react({
       include: ["**/react/*"],
     }),
-    sitemap({ changefreq: "daily" }),
+    sitemap({
+      changefreq: "daily",
+      filter: (page) => {
+        // Exclude redirect source URLs from sitemap to prevent duplicates
+        const pathname = new URL(page).pathname
+        const cleanPath = pathname.endsWith("/") && pathname !== "/" ? pathname.slice(0, -1) : pathname
+
+        // Exclude short format API reference URLs (e.g., /api-reference/v150, /ccip/api-reference/evm/v150)
+        // These are aliases for versioned content - we keep only the canonical long format URLs
+        const shortVersionPattern = /\/api-reference\/(?:.*\/)?v\d{3,4}(?:\/|$)/
+        if (shortVersionPattern.test(cleanPath)) {
+          return false
+        }
+
+        return !redirectSources.has(cleanPath)
+      },
+      serialize(item) {
+        // Remove trailing slash from URLs (except for root)
+        const url = new URL(item.url)
+        if (url.pathname.endsWith("/") && url.pathname !== "/") {
+          url.pathname = url.pathname.slice(0, -1)
+          item.url = url.toString()
+        }
+
+        // Add last modified date using git commit history
+        // Supports content files, API reference pages, and CCIP directory pages
+        const path = url.pathname
+        const lastModified = getLastModifiedDate(path)
+        if (lastModified) {
+          item.lastmod = lastModified.toISOString()
+        }
+
+        return item
+      },
+    }),
     mdx(),
   ],
   markdown: {

src/utils/lastModified.ts

@@ -0,0 +1,155 @@
+import { execSync } from "child_process"
+import fs from "fs"
+
+/**
+ * Get the last modified date for a given page path
+ * Uses two strategies in order of preference:
+ * 1. Git commit date of the source file
+ * 2. Current date as fallback (for new/untracked/dynamic files)
+ */
+export function getLastModifiedDate(urlPath: string): Date | null {
+  try {
+    const filePath = convertUrlToFilePath(urlPath)
+
+    if (!filePath) {
+      return null
+    }
+
+    // Strategy 1: Git commit date
+    const gitDate = getGitLastModified(filePath)
+    if (gitDate) {
+      return gitDate
+    }
+
+    // Strategy 2: Fallback to current date for new/untracked files
+    return new Date()
+  } catch (error) {
+    console.warn(`Failed to get last modified date for ${urlPath}:`, error)
+    return null
+  }
+}
+
+function convertUrlToFilePath(urlPath: string): string | null {
+  // Handle index pages
+  if (urlPath === "/" || urlPath === "") {
+    return "src/pages/index.astro"
+  }
+
+  // Remove leading slash
+  const cleanPath = urlPath.replace(/^\//, "")
+
+  // Handle API reference pages - map URL paths to content file versions
+  const apiReferenceMatch = cleanPath.match(/^([^/]+)\/api-reference\/(.+)$/)
+  if (apiReferenceMatch) {
+    const [, product, apiPath] = apiReferenceMatch
+
+    // Convert version shorthand to full version directory names
+    const versionMap: Record<string, string> = {
+      v150: "v1.5.0",
+      v151: "v1.5.1",
+      v160: "v1.6.0",
+      v023: "v0.2.3",
+      // Add more version mappings as needed
+    }
+
+    // Split the path and check if it's a version or has subpaths
+    const pathParts = apiPath.split("/")
+
+    if (pathParts.length === 1) {
+      // Simple version page like "evm/v150" -> "evm/v1.5.0/index.mdx"
+      const versionKey = pathParts[0]
+      const fullVersion = versionMap[versionKey] || versionKey
+      const contentPath = `src/content/${product}/api-reference/${fullVersion}/index.mdx`
+      if (fs.existsSync(contentPath)) {
+        return contentPath
+      }
+    } else if (pathParts.length >= 2) {
+      // Versioned subpath like "evm/v150/client" -> "evm/v1.5.0/client.mdx"
+      const versionKey = pathParts[0]
+      const fullVersion = versionMap[versionKey] || versionKey
+      const subPath = pathParts.slice(1).join("/")
+
+      // Try with the subpath
+      const contentPath = `src/content/${product}/api-reference/${fullVersion}/${subPath}.mdx`
+      if (fs.existsSync(contentPath)) {
+        return contentPath
+      }
+
+      // Try with index in subdirectory
+      const indexPath = `src/content/${product}/api-reference/${fullVersion}/${subPath}/index.mdx`
+      if (fs.existsSync(indexPath)) {
+        return indexPath
+      }
+    }
+  }
+
+  // Handle CCIP directory pages - map URLs to configuration files
+  const ccipDirectoryChainMatch = cleanPath.match(/^ccip\/directory\/(mainnet|testnet)\/chain\/(.+)$/)
+  if (ccipDirectoryChainMatch) {
+    const [, environment] = ccipDirectoryChainMatch
+
+    // Map CCIP directory chain URLs to their source configuration files
+    // These pages are generated from the chains.json configuration files
+    const configPath = `src/config/data/ccip/v1_2_0/${environment}/chains.json`
+    if (fs.existsSync(configPath)) {
+      return configPath
+    }
+  }
+
+  // Handle CCIP directory token pages - map URLs to token configuration files
+  const ccipDirectoryTokenMatch = cleanPath.match(/^ccip\/directory\/(mainnet|testnet)\/token\/(.+)$/)
+  if (ccipDirectoryTokenMatch) {
+    const [, environment] = ccipDirectoryTokenMatch
+
+    // Map CCIP directory token URLs to their source configuration files
+    // These pages are generated from the tokens.json configuration files
+    const configPath = `src/config/data/ccip/v1_2_0/${environment}/tokens.json`
+    if (fs.existsSync(configPath)) {
+      return configPath
+    }
+  }
+
+  // Try different file extensions and locations
+  const possiblePaths = [
+    `src/content/${cleanPath}.md`,
+    `src/content/${cleanPath}.mdx`,
+    `src/content/${cleanPath}/index.md`,
+    `src/content/${cleanPath}/index.mdx`,
+    `src/pages/${cleanPath}.astro`,
+    `src/pages/${cleanPath}.md`,
+    `src/pages/${cleanPath}.mdx`,
+    `src/pages/${cleanPath}/index.astro`,
+    `src/pages/${cleanPath}/index.md`,
+    `src/pages/${cleanPath}/index.mdx`,
+  ]
+
+  for (const possiblePath of possiblePaths) {
+    if (fs.existsSync(possiblePath)) {
+      return possiblePath
+    }
+  }
+
+  return null
+}
+
+function getGitLastModified(filePath: string): Date | null {
+  try {
+    // Get the last commit date for this file
+    const gitCommand = `git log -1 --format="%ci" --follow "${filePath}"`
+    const result = execSync(gitCommand, {
+      encoding: "utf8",
+      stdio: "pipe",
+      cwd: process.cwd(),
+      timeout: 5000, // 5 second timeout to prevent hanging
+    })
+
+    if (result.trim()) {
+      return new Date(result.trim())
+    }
+  } catch (error) {
+    // Git command failed - file might not be tracked or git not available
+    console.debug(`Git command failed for ${filePath}:`, error)
+  }
+
+  return null
+}