chore: hopefully allow algolia web crawler

Author: mlaursen

apps/docs/src/app/robots.ts

@@ -4,12 +4,18 @@ import { BASE_URL } from "@/constants/env.js";
 
 export default function robots(): MetadataRoute.Robots {
   return {
-    rules: {
-      userAgent: "*",
-      // I don't want them to crawl this preview website
-      disallow: "/",
-      // allow: "/",
-    },
+    rules: [
+      {
+        userAgent: "*",
+        // I don't want them to crawl this preview website
+        disallow: "/",
+        // allow: "/",
+      },
+      {
+        userAgent: "Algolia Crawler/*",
+        allow: "/",
+      },
+    ],
     sitemap: `${BASE_URL}/sitemap.xml`,
   };
 }