[testing] Improve running browser tests locally. (#25013)

This changes how the browser test runner works by default. It will now
automatically detect the browser type and:
 - Use the same options used in CircleCI (for chrome and firefox)
 - Create and delete a temporary profile directory
 - Terminate the browser
 
Adds two new options:
- `EMTEST_BROWSER_AUTO_CONFIG` or `--browser-auto-config`: disables the
automatic configuration,
 - `EMTEST_HEADLESS` or `--headless`: run the test in headless mode

Last, two flags were removed from chrome:
- `--disk-cache-dir` this was causing errors (on CI and locally)
- `--incognito` not allowed in corporate environments and not needed
since the profile directory is automatically deleted.

These changes make it much easier to reproduce the circle ci environment
locally and make running a test multiple times more reliable.

---------

Co-authored-by: Alon Zakai <[email protected]>

Author: brendandahl

.circleci/config.yml

@@ -314,18 +314,9 @@ commands:
           name: run tests (<< parameters.title >>)
           environment:
             EMTEST_DETECT_TEMPFILE_LEAKS: "0"
-            # --no-sandbox because we are running as root and chrome requires
-            # this flag for now: https://crbug.com/638180
-            CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader  --user-data-dir=/tmp/chrome-emscripten-profile --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports"
-            # Increase the window size to avoid flaky sdl tests see #24236.
-            CHROME_FLAGS_HEADLESS: "--headless=new --window-size=1024,768 --remote-debugging-port=1234"
-            CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features --js-flags=\"--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration\""
-            CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito"
-            # The runners lack sound hardware so fallback to a dummy device (and
-            # bypass the user gesture so audio tests work without interaction)
-            CHROME_FLAGS_AUDIO: " --use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required"
+            EMTEST_HEADLESS: "1"
+            EMTEST_BROWSER: "/usr/bin/google-chrome"
           command: |
-            export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE $CHROME_FLAGS_AUDIO"
             # There are tests in the browser test suite that using libraries
             # that are not included by "./embuilder build ALL".  For example the
             # PIC version of libSDL which is used by test_sdl2_misc_main_module
@@ -373,18 +364,6 @@ commands:
             apt-get update -y
             apt-get install -q -y pulseaudio
             pulseaudio --start
-      - run:
-          name: configure firefox
-          command: |
-            # Note: the autoplay pref allows playback without user interaction
-            mkdir ~/tmp-firefox-profile/
-            cat > ~/tmp-firefox-profile/user.js \<<EOF
-            user_pref("gfx.offscreencanvas.enabled", true);
-            user_pref("javascript.options.shared_memory", true);
-            user_pref("javascript.options.wasm_memory64", true);
-            user_pref("dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", true);
-            user_pref("media.autoplay.default", 0);
-            EOF
       - run:
           name: run tests (<< parameters.title >>)
           environment:
@@ -398,9 +377,10 @@ commands:
             # OffscreenCanvas support is not yet done in Firefox.
             EMTEST_LACKS_OFFSCREEN_CANVAS: "1"
             EMTEST_DETECT_TEMPFILE_LEAKS: "0"
+            EMTEST_HEADLESS: "1"
             DISPLAY: ":0"
           command: |
-            export EMTEST_BROWSER="$HOME/firefox/firefox -headless -profile $HOME/tmp-firefox-profile/"
+            export EMTEST_BROWSER="$HOME/firefox/firefox"
             # There are tests in the browser test suite that using libraries
             # that are not included by "./embuilder build ALL".  For example the
             # PIC version of libSDL which is used by test_sdl2_misc_main_module
@@ -423,14 +403,9 @@ commands:
           environment:
             EMTEST_LACKS_SOUND_HARDWARE: "1"
             EMTEST_DETECT_TEMPFILE_LEAKS: "0"
-            # --no-sandbox becasue we are running as root and chrome requires
-            # this flag for now: https://crbug.com/638180
-            CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile"
-            CHROME_FLAGS_HEADLESS: "--headless=new --remote-debugging-port=1234"
-            CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features"
-            CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito"
+            EMTEST_HEADLESS: "1"
+            EMTEST_BROWSER: "/usr/bin/google-chrome"
           command: |
-            export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE"
             test/runner sockets
       - upload-test-results
 

test/common.py

@@ -45,15 +45,21 @@
 
 # User can specify an environment variable EMTEST_BROWSER to force the browser
 # test suite to run using another browser command line than the default system
-# browser.
-# There are two special value that can be used here if running in an actual
+# browser. If only the path to the browser executable is given, the tests
+# will run in headless mode with a temporary profile with the same options
+# used in CI. To use a custom start command specify the executable and command
+# line flags.
+#
+# There are two special values that can be used here if running in an actual
 # browser is not desired:
 #  EMTEST_BROWSER=0 : This will disable the actual running of the test and simply
 #                     verify that it compiles and links.
 #  EMTEST_BROWSER=node : This will attempt to run the browser test under node.
 #                        For most browser tests this does not work, but it can
 #                        be useful for running pthread tests under node.
 EMTEST_BROWSER = None
+EMTEST_BROWSER_AUTO_CONFIG = None
+EMTEST_HEADLESS = None
 EMTEST_DETECT_TEMPFILE_LEAKS = None
 EMTEST_SAVE_DIR = None
 # generally js engines are equivalent, testing 1 is enough. set this
@@ -76,13 +82,47 @@
 if 'EM_BUILD_VERBOSE' in os.environ:
   exit_with_error('EM_BUILD_VERBOSE has been renamed to EMTEST_BUILD_VERBOSE')
 
+
+# Default flags used to run browsers in CI testing:
+class ChromeConfig:
+  data_dir_flag = '--user-data-dir='
+  default_flags = (
+    # --no-sandbox because we are running as root and chrome requires
+    # this flag for now: https://crbug.com/638180
+    '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports',
+    '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"',
+    # The runners lack sound hardware so fallback to a dummy device (and
+    # bypass the user gesture so audio tests work without interaction)
+    '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required',
+    # Cache options.
+    '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache',
+  )
+  headless_flags = '--headless=new --window-size=1024,768 --remote-debugging-port=1234'
+
+  @staticmethod
+  def configure(data_dir):
+    """Chrome has no special configuration step."""
+
+
+class FirefoxConfig:
+  data_dir_flag = '-profile '
+  default_flags = ()
+  headless_flags = '-headless'
+
+  @staticmethod
+  def configure(data_dir):
+    shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js'))
+
+
 # Special value for passing to assert_returncode which means we expect that program
 # to fail with non-zero return code, but we don't care about specifically which one.
 NON_ZERO = -1
 
 TEST_ROOT = path_from_root('test')
 LAST_TEST = path_from_root('out/last_test.txt')
 
+DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile')
+
 WEBIDL_BINDER = shared.bat_suffix(path_from_root('tools/webidl_binder'))
 
 EMBUILDER = shared.bat_suffix(path_from_root('embuilder'))
@@ -114,6 +154,17 @@ def has_browser():
   return EMTEST_BROWSER != '0'
 
 
+CHROMIUM_BASED_BROWSERS = ['chrom', 'edge', 'opera']
+
+
+def is_chrome():
+  return EMTEST_BROWSER and any(pattern in EMTEST_BROWSER.lower() for pattern in CHROMIUM_BASED_BROWSERS)
+
+
+def is_firefox():
+  return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower()
+
+
 def compiler_for(filename, force_c=False):
   if shared.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c:
     return EMXX
@@ -2323,9 +2374,7 @@ def __init__(self, *args, **kwargs):
     super().__init__(*args, **kwargs)
 
   @classmethod
-  def browser_restart(cls):
-    # Kill existing browser
-    logger.info('Restarting browser process')
+  def browser_terminate(cls):
     cls.browser_proc.terminate()
     # If the browser doesn't shut down gracefully (in response to SIGTERM)
     # after 2 seconds kill it with force (SIGKILL).
@@ -2335,6 +2384,13 @@ def browser_restart(cls):
       logger.info('Browser did not respond to `terminate`.  Using `kill`')
       cls.browser_proc.kill()
       cls.browser_proc.wait()
+      cls.browser_data_dir = None
+
+  @classmethod
+  def browser_restart(cls):
+    # Kill existing browser
+    logger.info('Restarting browser process')
+    cls.browser_terminate()
     cls.browser_open(cls.HARNESS_URL)
 
   @classmethod
@@ -2343,6 +2399,24 @@ def browser_open(cls, url):
     if not EMTEST_BROWSER:
       logger.info('No EMTEST_BROWSER set. Defaulting to `google-chrome`')
       EMTEST_BROWSER = 'google-chrome'
+
+    if EMTEST_BROWSER_AUTO_CONFIG:
+      logger.info('Using default CI configuration.')
+      cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR
+      if os.path.exists(cls.browser_data_dir):
+        utils.delete_dir(cls.browser_data_dir)
+      os.mkdir(cls.browser_data_dir)
+      if is_chrome():
+        config = ChromeConfig()
+      elif is_firefox():
+        config = FirefoxConfig()
+      else:
+        exit_with_error("EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox or chrome.")
+      EMTEST_BROWSER += f" {config.data_dir_flag}{cls.browser_data_dir} {' '.join(config.default_flags)}"
+      if EMTEST_HEADLESS == 1:
+        EMTEST_BROWSER += f" {config.headless_flags}"
+      config.configure(cls.browser_data_dir)
+
     if WINDOWS:
       # On Windows env. vars canonically use backslashes as directory delimiters, e.g.
       # set EMTEST_BROWSER=C:\Program Files\Mozilla Firefox\firefox.exe
@@ -2373,6 +2447,7 @@ def tearDownClass(cls):
       return
     cls.harness_server.terminate()
     print('[Browser harness server terminated]')
+    cls.browser_terminate()
     if WINDOWS:
       # On Windows, shutil.rmtree() in tearDown() raises this exception if we do not wait a bit:
       # WindowsError: [Error 32] The process cannot access the file because it is being used by another process.

test/firefox_user.js

@@ -0,0 +1,5 @@
+user_pref("gfx.offscreencanvas.enabled", true);
+user_pref("javascript.options.shared_memory", true);
+user_pref("javascript.options.wasm_memory64", true);
+user_pref("dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", true);
+user_pref("media.autoplay.default", 0);

test/runner.py

@@ -392,6 +392,10 @@ def parse_args():
                       help='Automatically update test expectations for tests that support it.')
   parser.add_argument('--browser',
                       help='Command to launch web browser in which to run browser tests.')
+  parser.add_argument('--headless', action='store_true',
+                      help='Run browser tests in headless mode.', default=None)
+  parser.add_argument('--browser-auto-config', type=bool, default=True,
+                      help='Use the default CI browser configuration.')
   parser.add_argument('tests', nargs='*')
   parser.add_argument('--failfast', action='store_true')
   parser.add_argument('--start-at', metavar='NAME', help='Skip all tests up until <NAME>')
@@ -407,6 +411,8 @@ def parse_args():
 
 def configure():
   common.EMTEST_BROWSER = os.getenv('EMTEST_BROWSER')
+  common.EMTEST_BROWSER_AUTO_CONFIG = os.getenv('EMTEST_BROWSER_AUTO_CONFIG')
+  common.EMTEST_HEADLESS = int(os.getenv('EMTEST_HEADLESS', '0'))
   common.EMTEST_DETECT_TEMPFILE_LEAKS = int(os.getenv('EMTEST_DETECT_TEMPFILE_LEAKS', '0'))
   common.EMTEST_ALL_ENGINES = int(os.getenv('EMTEST_ALL_ENGINES', '0'))
   common.EMTEST_SKIP_SLOW = int(os.getenv('EMTEST_SKIP_SLOW', '0'))
@@ -451,6 +457,8 @@ def set_env(name, option_value):
     os.environ[name] = value
 
   set_env('EMTEST_BROWSER', options.browser)
+  set_env('EMTEST_BROWSER_AUTO_CONFIG', options.browser_auto_config)
+  set_env('EMTEST_HEADLESS', options.headless)
   set_env('EMTEST_DETECT_TEMPFILE_LEAKS', options.detect_leaks)
   if options.save_dir:
     common.EMTEST_SAVE_DIR = 1

test/test_browser.py

@@ -20,7 +20,7 @@
 from urllib.request import urlopen
 
 import common
-from common import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting
+from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, CHROMIUM_BASED_BROWSERS
 from common import create_file, parameterized, ensure_dir, disabled, test_file, WEBIDL_BINDER
 from common import read_file, EMRUN, no_wasm64, no_2gb, no_4gb, copytree
 from common import requires_wasm2js, parameterize, find_browser_test_file, with_all_sjlj
@@ -130,23 +130,12 @@ def shell_with_script(shell_file, output_file, replacement):
   create_file(output_file, shell.replace('{{{ SCRIPT }}}', replacement))
 
 
-CHROMIUM_BASED_BROWSERS = ['chrom', 'edge', 'opera']
-
-
-def is_chrome():
-  return EMTEST_BROWSER and any(pattern in EMTEST_BROWSER.lower() for pattern in CHROMIUM_BASED_BROWSERS)
-
-
 def no_chrome(note='chrome is not supported'):
   if is_chrome():
     return unittest.skip(note)
   return lambda f: f
 
 
-def is_firefox():
-  return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower()
-
-
 def no_firefox(note='firefox is not supported'):
   if is_firefox():
     return unittest.skip(note)
@@ -162,7 +151,7 @@ def no_swiftshader(f):
 
   @wraps(f)
   def decorated(self, *args, **kwargs):
-    if is_chrome() and '--use-gl=swiftshader' in EMTEST_BROWSER:
+    if is_chrome() and '--use-gl=swiftshader' in common.EMTEST_BROWSER:
       self.skipTest('not compatible with swiftshader')
     return f(self, *args, **kwargs)
 
@@ -210,7 +199,7 @@ class browser(BrowserCore):
   def setUpClass(cls):
     super().setUpClass()
     cls.browser_timeout = 60
-    if EMTEST_BROWSER != 'node':
+    if common.EMTEST_BROWSER != 'node':
       print()
       print('Running the browser tests. Make sure the browser allows popups from localhost.')
       print()
@@ -220,7 +209,7 @@ def proxy_to_worker(self):
 
   def require_jspi(self):
     if not is_chrome():
-      self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.')
+      self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.')
     super().require_jspi()
 
   def post_manual_reftest(self):
@@ -3305,7 +3294,7 @@ def test_cocos2d_hello(self):
   })
   def test_async(self, args):
     if is_jspi(args) and not is_chrome():
-      self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.')
+      self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.')
 
     for opts in (0, 1, 2, 3):
       print(opts)
@@ -4977,7 +4966,7 @@ def test_embind_with_pthreads(self):
   })
   def test_embind(self, args):
     if is_jspi(args) and not is_chrome():
-      self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.')
+      self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.')
     if is_jspi(args) and self.is_wasm64():
       self.skipTest('_emval_await fails')
 
@@ -5597,9 +5586,9 @@ def test_no_browser(self):
     self.run_process([EMCC, test_file('test_emrun.c'), '--emrun', '-o', 'hello_world.html'])
     proc = subprocess.Popen([EMRUN, '--no-browser', '.', '--port=3333'], stdout=PIPE)
     try:
-      if EMTEST_BROWSER:
+      if common.EMTEST_BROWSER:
         print('Starting browser')
-        browser_cmd = shlex.split(EMTEST_BROWSER)
+        browser_cmd = shlex.split(common.EMTEST_BROWSER)
         browser = subprocess.Popen(browser_cmd + ['http://localhost:3333/hello_world.html'])
         try:
           while True:
@@ -5638,11 +5627,11 @@ def test_emrun(self):
                  '--log-stdout', self.in_dir('stdout.txt'),
                  '--log-stderr', self.in_dir('stderr.txt')]
 
-    if EMTEST_BROWSER is not None:
+    if common.EMTEST_BROWSER is not None:
       # If EMTEST_BROWSER carried command line arguments to pass to the browser,
       # (e.g. "firefox -profile /path/to/foo") those can't be passed via emrun,
       # so strip them out.
-      browser_cmd = shlex.split(EMTEST_BROWSER)
+      browser_cmd = shlex.split(common.EMTEST_BROWSER)
       browser_path = browser_cmd[0]
       args_base += ['--browser', browser_path]
       if len(browser_cmd) > 1: