Fix: Disambiguate Mediapartners-Google user agent (#82536)

`Mediapartners-Google` used to match both `HEADLESS_BROWSER_BOT_UA_RE`
and `HTML_LIMITED_BOT_UA_RE`, causing inconsistent state values for
`getBotType`, `isDomBotUA`, and `isHtmlLimitedBotUA`.

Author: gaojude

packages/next/src/shared/lib/router/utils/html-bots.ts

@@ -1,4 +1,6 @@
 // This regex contains the bots that we need to do a blocking render for and can't safely stream the response
 // due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent.
+// Note: The pattern [\w-]+-Google captures all Google crawlers with "-Google" suffix (e.g., Mediapartners-Google, AdsBot-Google, Storebot-Google)
+// as well as crawlers starting with "Google-" (e.g., Google-PageRenderer, Google-InspectionTool)
 export const HTML_LIMITED_BOT_UA_RE =
-  /Mediapartners-Google|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti/i
+  /[\w-]+-Google|Google-[\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight/i

packages/next/src/shared/lib/router/utils/is-bot.ts

@@ -1,9 +1,10 @@
 import { HTML_LIMITED_BOT_UA_RE } from './html-bots'
 
 // Bot crawler that will spin up a headless browser and execute JS.
-// By default, only googlebots are considered as DOM bots. Blow is where the regex is computed from:
+// Only the main Googlebot search crawler executes JavaScript, not other Google crawlers.
 // x-ref: https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
-const HEADLESS_BROWSER_BOT_UA_RE = /google/i
+// This regex specifically matches "Googlebot" but NOT "Mediapartners-Google", "AdsBot-Google", etc.
+const HEADLESS_BROWSER_BOT_UA_RE = /Googlebot(?!-)|Googlebot$/i
 
 export const HTML_LIMITED_BOT_UA_RE_STRING = HTML_LIMITED_BOT_UA_RE.source
 

test/production/app-dir/metadata-streaming-config/metadata-streaming-config.test.ts

@@ -11,7 +11,7 @@ describe('app-dir - metadata-streaming-config', () => {
     )
 
     expect(requiredServerFiles.config.htmlLimitedBots).toMatchInlineSnapshot(
-      `"Mediapartners-Google|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti"`
+      `"[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight"`
     )
 
     const prerenderManifest = JSON.parse(
@@ -38,7 +38,7 @@ describe('app-dir - metadata-streaming-config', () => {
        "/ppr": {
          "key": "user-agent",
          "type": "header",
-         "value": "Mediapartners-Google|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti",
+         "value": "[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight",
        },
      }
     `)