Update bot detection list and add support for oxygen-do-not-stream-response header (#2514)

* Update bot detection list and add support for oxygen-do-not-stream-response header

* Fix tests

Author: blittle

.changeset/happy-snails-pay.md

@@ -0,0 +1,5 @@
+---
+'@shopify/hydrogen': patch
+---
+
+Update bot detection list and add support for oxygen-do-not-stream-response header

packages/hydrogen/src/entry-server.tsx

@@ -323,7 +323,10 @@ async function processRequest(
     });
   }
 
-  if (isBotUA(url, request.headers.get('user-agent'))) {
+  if (
+    request.headers.get('oxygen-do-not-stream-response') === 'true' ||
+    isBotUA(url, request.headers.get('user-agent'))
+  ) {
     response.doNotStream();
   }
 

packages/hydrogen/src/foundation/Analytics/tests/Analytics.client.utm.test.tsx

@@ -9,7 +9,7 @@ import {ClientAnalytics} from '../ClientAnalytics.js';
 // window.location more than once
 describe('Analytics.client utm', () => {
   it('should process utm search parameters', async (done) => {
-    global.window = Object.create(window);
+    // global.window = Object.create(window);
     Object.defineProperty(window, 'location', {
       value: {
         search:

packages/hydrogen/src/utilities/bot-ua.ts

@@ -4,55 +4,56 @@
  * https://github.com/GoogleChrome/rendertron/blob/6f681688737846b28754fbfdf5db173846a826df/middleware/src/middleware.ts#L24-L41
  */
 const botUserAgents = [
-  'AdsBot-Google',
-  'AdsBot-Google-Mobile',
+  '360spider',
+  'adsbot-google',
+  'adsbot-google-mobile',
   'applebot',
-  'Baiduspider',
   'baiduspider',
-  'Bytespider',
-  '360Spider',
-  'PetalBot',
-  'Yisouspider',
+  'baiduspider',
+  'bingbot',
   'bingbot',
-  'Bingbot',
-  'BingPreview',
+  'bingpreview',
   'bitlybot',
-  'Discordbot',
-  'DuckDuckBot',
-  'Embedly',
+  'bytespider',
+  'discordbot',
+  'duckduckbot',
+  'embedly',
   'facebookcatalog',
   'facebookexternalhit',
-  'Google-PageRenderer',
-  'Googlebot',
-  'Googlebot-Image',
-  'Googlebot-News',
-  'Googlebot-Video',
+  'google-inspectiontool',
+  'google-pagerenderer',
+  'googlebot',
+  'googlebot-image',
+  'googlebot-news',
+  'googlebot-video',
+  'googleother',
   'googleweblight',
   'ia_archive',
-  'LinkedInBot',
-  'Mediapartners-Google',
+  'ia_archiver',
+  'linkedinbot',
+  'mediapartners-google',
   'outbrain',
+  'petalbot',
   'pinterest',
   'quora link preview',
   'redditbot',
   'rogerbot',
+  'seoradar',
   'showyoubot',
-  'SkypeUriPreview',
-  'Slackbot',
-  'Slurp',
+  'skypeuripreview',
+  'slackbot',
+  'slurp',
   'sogou',
-  'Storebot-Google',
-  'TelegramBot',
+  'storebot-google',
+  'telegrambot',
   'tumblr',
-  'Twitterbot',
-  'vkShare',
-  'W3C_Validator',
-  'WhatsApp',
+  'twitterbot',
+  'vkshare',
+  'w3c html2txt',
+  'w3c_validator',
+  'whatsapp',
   'yandex',
-
-  // SEO Tools
-  'Seoradar',
-  'W3C html2txt',
+  'yisouspider',
 ];
 
 /**