remove robots txt and add noindex to meta

Author: bhavabhuthi

app/layout.tsx

@@ -3,7 +3,6 @@ import { Metadata } from 'next';
 
 import '../styles/globals.css';
 
-// A second layer of protection at meta tag level for crawlers
 export const metadata: Metadata = {
   robots: {
     index: false,

app/robots.ts

@@ -2,11 +2,11 @@ import { MetadataRoute } from 'next'
 
 // Function to generate the robots txt file to block crawlers. 
 // Please edit the metadata function in root layout along with changes here
-export default function robots(): MetadataRoute.Robots {
-  return {
-    rules: {
-      userAgent: '*',
-      disallow: '/',
-    }
-  }
-}
+// export default function robots(): MetadataRoute.Robots {
+//   return {
+//     rules: {
+//       userAgent: '*',
+//       disallow: '/',
+//     }
+//   }
+// }