CRE (#3128)

* cre init

* format

* update

---------

Co-authored-by: aelmanaa <[email protected]>

Author: khadni

.github/workflows/test.yml

@@ -161,6 +161,8 @@ jobs:
       
       - name: Check Internal Links
         run: npm run linkcheck-internal
+        env:
+          NODE_OPTIONS: "--max-old-space-size=8192"
 
   # Job 4: Check Types
   typecheck:

astro.config.ts

@@ -13,8 +13,10 @@ import yaml from "@rollup/plugin-yaml"
 import { ccipRedirects } from "./src/config/redirects/ccip"
 import trailingSlashMiddleware from "./src/integrations/trailing-slash-middleware"
 import redirectsJson from "./src/features/redirects/redirects.json"
+import { extractCanonicalUrlsWithLanguageVariants } from "./src/utils/sidebar"
 
 config() // Load .env file
+
 // Prepare set of redirect source URLs to exclude from sitemap
 // This prevents duplicate entries and ensures only canonical URLs are indexed
 const redirectSources = new Set(
@@ -27,6 +29,11 @@ const redirectSources = new Set(
     })
 )
 
+// Extract canonical URLs that have language-specific variants from sidebar config
+// These redirect pages should NOT be in the sitemap
+// Only the actual content pages (-go, -ts) are indexed
+const canonicalUrlsWithLanguageVariants = extractCanonicalUrlsWithLanguageVariants()
+
 // https://astro.build/config
 export default defineConfig({
   site: "https://docs.chain.link",
@@ -50,10 +57,14 @@ export default defineConfig({
       changefreq: "daily",
       customPages: [
         "https://docs.chain.link/llms.txt",
+        "https://docs.chain.link/cre/llms-full-go.txt",
+        "https://docs.chain.link/cre/llms-full-ts.txt",
         "https://docs.chain.link/vrf/llms-full.txt",
         "https://docs.chain.link/ccip/llms-full.txt",
         "https://docs.chain.link/data-feeds/llms-full.txt",
         "https://docs.chain.link/data-streams/llms-full.txt",
+        "https://docs.chain.link/dta-technical-standard/llms-full.txt",
+        "https://docs.chain.link/datalink/llms-full.txt",
         "https://docs.chain.link/chainlink-functions/llms-full.txt",
         "https://docs.chain.link/chainlink-automation/llms-full.txt",
         "https://docs.chain.link/resources/llms-full.txt",
@@ -74,6 +85,11 @@ export default defineConfig({
           return false
         }
 
+        // Exclude canonical URLs that have language-specific variants (from sidebar config)
+        if (canonicalUrlsWithLanguageVariants.has(cleanPath)) {
+          return false
+        }
+
         return !redirectSources.has(cleanPath)
       },
       serialize(item) {