SItemap generation

Author: dannyroosevelt

docs-v2/next-sitemap.config.js

@@ -0,0 +1,26 @@
+/** @type {import('next-sitemap').IConfig} */
+module.exports = {
+  siteUrl: "https://pipedream.com/docs",
+  generateRobotsTxt: true,
+  outDir: "./public",
+  generateIndexSitemap: false,
+  // Add a trailing slash to all URLs
+  trailingSlash: true,
+  // Exclude URLs matching these patterns
+  exclude: [
+    "/hidden/*",
+    "/deprecated/*",
+    "/api-docs-server",
+  ],
+  robotsTxtOptions: {
+    additionalSitemaps: [
+      "https://pipedream.com/sitemap.xml", // If you have a main site sitemap, include it here
+    ],
+    policies: [
+      {
+        userAgent: "*",
+        allow: "/",
+      },
+    ],
+  },
+};

docs-v2/package-lock.json

@@ -6,6 +6,7 @@
   "scripts": {
     "dev": "next dev",
     "build": "next build",
+    "postbuild": "next-sitemap",
     "start": "next start"
   },
   "packageManager": "[email protected]",
@@ -33,6 +34,7 @@
   "devDependencies": {
     "@types/node": "18.11.10",
     "autoprefixer": "^10.4.19",
+    "next-sitemap": "^4.2.3",
     "postcss": "^8.4.40",
     "prettier": "^3.3.3",
     "tailwindcss": "^3.4.7",

docs-v2/package.json

@@ -0,0 +1,10 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://pipedream.com/docs
+
+# Sitemaps
+Sitemap: https://pipedream.com/docs/sitemap.xml
+Sitemap: https://pipedream.com/sitemap.xml