Add robots.txt (#1060)

Author: egeakman

.github/workflows/preview.yml

@@ -33,7 +33,7 @@ jobs:
         run: make install
 
       - name: Build the website
-        run: make build
+        run: make build PREVIEW=true
 
       - name: Set up SSH key
         uses: webfactory/[email protected]

src/pages/robots.txt.ts

@@ -0,0 +1,19 @@
+import type { APIRoute } from "astro";
+
+const previewRobots = `
+User-agent: *
+Disallow: /
+`;
+
+const prodRobots = `
+User-agent: *
+Disallow: /_astro/
+Disallow: /*?
+Allow: /
+
+Sitemap: https://ep2025.europython.eu/sitemap-index.xml
+`;
+
+const isPreview = String(import.meta.env.PREVIEW).toLowerCase() === "true";
+export const GET: APIRoute = () =>
+  new Response(isPreview ? previewRobots : prodRobots);