Disallow crawling by web-robots of search/ask URLs (#3035)

Author: SamyPesse

.changeset/late-teachers-jump.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Disallow crawling by web-robots of search/ask URLs

packages/gitbook/src/routes/robots.ts

@@ -10,7 +10,10 @@ export async function serveRobotsTxt(context: GitBookSiteContext) {
     const isRoot = checkIsRootSiteContext(context);
     const lines = [
         'User-agent: *',
+        // Disallow image resizing
         'Disallow: /~gitbook/',
+        // Disallow dynamic routes / search queries
+        'Disallow: /*?',
         ...((await isSiteIndexable(context))
             ? [
                   'Allow: /',