From c477e94fa24b65e850e6c3ab03383163236ce8ad Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Thu, 21 Aug 2025 18:49:01 +0000 Subject: [PATCH 01/12] [testing] Improve running browser tests locally. This changes how the browser test runner works by default when only a browser executable is passed into EMTEST_BROWSER (e.g. EMTEST_BROWSER=chrome). The test runner will now automatically: - Use the same options used in CircleCI (for chrome and firefox) - Create and delete a temporary profile directory - Terminate the browser These changes make it much easier to reproduce the circle ci environment locally and make running a test multiple times more reliable. To make it easier to also run in "headed" mode, I added a `--headed` option. Also, if the user specifies EMTEST_BROWSER with options the old behavior will be used and a custom start command can be used. Last, two flags were removed from chrome: - `--disk-cache-dir` this was causing errors (on CI and locally) - `--incognito` not allowed in corporate environments and not needed since the profile directory is automatically deleted. --- .circleci/config.yml | 12 ++----- test/common.py | 78 +++++++++++++++++++++++++++++++++++++++++--- test/runner.py | 4 +++ 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 125f04f94435a..173f4322053c2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -325,7 +325,7 @@ commands: # bypass the user gesture so audio tests work without interaction) CHROME_FLAGS_AUDIO: " --use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required" command: | - export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE $CHROME_FLAGS_AUDIO" + export EMTEST_BROWSER="/usr/bin/google-chrome" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module @@ -400,7 +400,7 @@ commands: EMTEST_DETECT_TEMPFILE_LEAKS: "0" DISPLAY: ":0" command: | - export EMTEST_BROWSER="$HOME/firefox/firefox -headless -profile $HOME/tmp-firefox-profile/" + export EMTEST_BROWSER="$HOME/firefox/firefox" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module @@ -423,14 +423,8 @@ commands: environment: EMTEST_LACKS_SOUND_HARDWARE: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" - # --no-sandbox becasue we are running as root and chrome requires - # this flag for now: https://crbug.com/638180 - CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile" - CHROME_FLAGS_HEADLESS: "--headless=new --remote-debugging-port=1234" - CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features" - CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito" command: | - export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE" + export EMTEST_BROWSER="/usr/bin/google-chrome" test/runner sockets - upload-test-results diff --git a/test/common.py b/test/common.py index 090b19f0cb223..6412d7b2b14b2 100644 --- a/test/common.py +++ b/test/common.py @@ -45,8 +45,12 @@ # User can specify an environment variable EMTEST_BROWSER to force the browser # test suite to run using another browser command line than the default system -# browser. -# There are two special value that can be used here if running in an actual +# browser. If only the path to the browser executable is given, the tests +# will run in headless mode with a temporary profile with the same options +# used in CircleCI. To use a custom start command specify the executable and +# command line flags. +# +# There are two special values that can be used here if running in an actual # browser is not desired: # EMTEST_BROWSER=0 : This will disable the actual running of the test and simply # verify that it compiles and links. @@ -54,6 +58,7 @@ # For most browser tests this does not work, but it can # be useful for running pthread tests under node. EMTEST_BROWSER = None +EMTEST_HEADLESS = None EMTEST_DETECT_TEMPFILE_LEAKS = None EMTEST_SAVE_DIR = None # generally js engines are equivalent, testing 1 is enough. set this @@ -76,6 +81,41 @@ if 'EM_BUILD_VERBOSE' in os.environ: exit_with_error('EM_BUILD_VERBOSE has been renamed to EMTEST_BUILD_VERBOSE') +# Default flags used to run browsers in CI testing: +BROWSER_CONFIG = { + 'chrome': { + 'data_dir_flag': '--user-data-dir', + 'default': ( + # --no-sandbox because we are running as root and chrome requires + # this flag for now: https://crbug.com/638180 + '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports', + '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"', + # The runners lack sound hardware so fallback to a dummy device (and + # bypass the user gesture so audio tests work without interaction) + '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required', + # Cache options. + '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', + ), + 'headless': ( + # Increase the window size to avoid flaky sdl tests see #24236. + '--headless=new --window-size=1024,768 --remote-debugging-port=1234', + ), + }, + 'firefox': { + 'data_dir_flag': '-profile', + 'default': {}, + 'headless': ( + '-headless' + ), + }, + 'other': { + 'data_dir_flag': '', + 'default': {}, + 'headless': {}, + }, +} +DEFAULT_BROWSER_DATA_DIR = '/tmp/emscripten-profile' + # Special value for passing to assert_returncode which means we expect that program # to fail with non-zero return code, but we don't care about specifically which one. NON_ZERO = -1 @@ -2323,9 +2363,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @classmethod - def browser_restart(cls): - # Kill existing browser - logger.info('Restarting browser process') + def browser_terminate(cls): cls.browser_proc.terminate() # If the browser doesn't shut down gracefully (in response to SIGTERM) # after 2 seconds kill it with force (SIGKILL). @@ -2335,6 +2373,15 @@ def browser_restart(cls): logger.info('Browser did not respond to `terminate`. Using `kill`') cls.browser_proc.kill() cls.browser_proc.wait() + if cls.browser_data_dir: + utils.delete_dir(cls.browser_data_dir) + cls.browser_data_dir = None + + @classmethod + def browser_restart(cls): + # Kill existing browser + logger.info('Restarting browser process') + cls.browser_terminate() cls.browser_open(cls.HARNESS_URL) @classmethod @@ -2343,6 +2390,26 @@ def browser_open(cls, url): if not EMTEST_BROWSER: logger.info('No EMTEST_BROWSER set. Defaulting to `google-chrome`') EMTEST_BROWSER = 'google-chrome' + + # If only the the browser is specified, use the default arguments used in circleci. + browser_args = shlex.split(EMTEST_BROWSER) + if len(browser_args) == 1: + logger.info('No EMTEST_BROWSER flags set. Defaulting to CI configuration.') + browser = browser_args[0] + cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR + if os.path.exists(cls.browser_data_dir): + utils.delete_dir(cls.browser_data_dir) + if 'chrome' in browser: + config = BROWSER_CONFIG['chrome'] + elif 'firefox' in browser: + config = BROWSER_CONFIG['firefox'] + else: + logger.warning("Unknown browser type, not using default flags.") + config = BROWSER_CONFIG['other'] + EMTEST_BROWSER += f" {config['data_dir_flag']}={cls.browser_data_dir} {' '.join(config['default'])}" + if EMTEST_HEADLESS == '1': + EMTEST_BROWSER += f" {' '.join(config['headless'])}" + if WINDOWS: # On Windows env. vars canonically use backslashes as directory delimiters, e.g. # set EMTEST_BROWSER=C:\Program Files\Mozilla Firefox\firefox.exe @@ -2373,6 +2440,7 @@ def tearDownClass(cls): return cls.harness_server.terminate() print('[Browser harness server terminated]') + cls.browser_terminate() if WINDOWS: # On Windows, shutil.rmtree() in tearDown() raises this exception if we do not wait a bit: # WindowsError: [Error 32] The process cannot access the file because it is being used by another process. diff --git a/test/runner.py b/test/runner.py index 74e9bb4f64c9d..a916cc6a2c1d6 100755 --- a/test/runner.py +++ b/test/runner.py @@ -392,6 +392,8 @@ def parse_args(): help='Automatically update test expectations for tests that support it.') parser.add_argument('--browser', help='Command to launch web browser in which to run browser tests.') + parser.add_argument('--headed', action='store_true', + help='Run browser tests in regular (non-headless) mode.') parser.add_argument('tests', nargs='*') parser.add_argument('--failfast', action='store_true') parser.add_argument('--start-at', metavar='NAME', help='Skip all tests up until ') @@ -407,6 +409,7 @@ def parse_args(): def configure(): common.EMTEST_BROWSER = os.getenv('EMTEST_BROWSER') + common.EMTEST_HEADLESS = os.getenv('EMTEST_HEADLESS') common.EMTEST_DETECT_TEMPFILE_LEAKS = int(os.getenv('EMTEST_DETECT_TEMPFILE_LEAKS', '0')) common.EMTEST_ALL_ENGINES = int(os.getenv('EMTEST_ALL_ENGINES', '0')) common.EMTEST_SKIP_SLOW = int(os.getenv('EMTEST_SKIP_SLOW', '0')) @@ -451,6 +454,7 @@ def set_env(name, option_value): os.environ[name] = value set_env('EMTEST_BROWSER', options.browser) + set_env('EMTEST_HEADLESS', not options.headed) set_env('EMTEST_DETECT_TEMPFILE_LEAKS', options.detect_leaks) if options.save_dir: common.EMTEST_SAVE_DIR = 1 From 83de3cc49f7ea6e978cdb131e5fc12efc40fe509 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Thu, 21 Aug 2025 19:05:33 +0000 Subject: [PATCH 02/12] Use string. --- test/common.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/common.py b/test/common.py index 6412d7b2b14b2..4fe55bcf6dca8 100644 --- a/test/common.py +++ b/test/common.py @@ -96,22 +96,19 @@ # Cache options. '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', ), - 'headless': ( + 'headless': # Increase the window size to avoid flaky sdl tests see #24236. '--headless=new --window-size=1024,768 --remote-debugging-port=1234', - ), }, 'firefox': { 'data_dir_flag': '-profile', 'default': {}, - 'headless': ( - '-headless' - ), + 'headless': '-headless', }, 'other': { 'data_dir_flag': '', 'default': {}, - 'headless': {}, + 'headless': '', }, } DEFAULT_BROWSER_DATA_DIR = '/tmp/emscripten-profile' @@ -2408,7 +2405,7 @@ def browser_open(cls, url): config = BROWSER_CONFIG['other'] EMTEST_BROWSER += f" {config['data_dir_flag']}={cls.browser_data_dir} {' '.join(config['default'])}" if EMTEST_HEADLESS == '1': - EMTEST_BROWSER += f" {' '.join(config['headless'])}" + EMTEST_BROWSER += f" {config['headless']}" if WINDOWS: # On Windows env. vars canonically use backslashes as directory delimiters, e.g. From 0cf3f0bc2f4469901c65ff063a7976d907f53d1c Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Thu, 21 Aug 2025 20:20:09 +0000 Subject: [PATCH 03/12] comment and try to fix firefox --- test/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/common.py b/test/common.py index 4fe55bcf6dca8..943eeb2e593d2 100644 --- a/test/common.py +++ b/test/common.py @@ -47,8 +47,8 @@ # test suite to run using another browser command line than the default system # browser. If only the path to the browser executable is given, the tests # will run in headless mode with a temporary profile with the same options -# used in CircleCI. To use a custom start command specify the executable and -# command line flags. +# used in CI. To use a custom start command specify the executable and command +# line flags. # # There are two special values that can be used here if running in an actual # browser is not desired: @@ -84,7 +84,7 @@ # Default flags used to run browsers in CI testing: BROWSER_CONFIG = { 'chrome': { - 'data_dir_flag': '--user-data-dir', + 'data_dir_flag': '--user-data-dir=', 'default': ( # --no-sandbox because we are running as root and chrome requires # this flag for now: https://crbug.com/638180 @@ -101,7 +101,7 @@ '--headless=new --window-size=1024,768 --remote-debugging-port=1234', }, 'firefox': { - 'data_dir_flag': '-profile', + 'data_dir_flag': '-profile ', 'default': {}, 'headless': '-headless', }, @@ -2403,7 +2403,7 @@ def browser_open(cls, url): else: logger.warning("Unknown browser type, not using default flags.") config = BROWSER_CONFIG['other'] - EMTEST_BROWSER += f" {config['data_dir_flag']}={cls.browser_data_dir} {' '.join(config['default'])}" + EMTEST_BROWSER += f" {config['data_dir_flag']}{cls.browser_data_dir} {' '.join(config['default'])}" if EMTEST_HEADLESS == '1': EMTEST_BROWSER += f" {config['headless']}" From 169afad203534c64d27d5b25d4fd2de45a8dfa6d Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Thu, 21 Aug 2025 22:57:41 +0000 Subject: [PATCH 04/12] - use current is_chrome/is_firefox - add option to enable/disable auto browser config - switch to --headless flag - fix checking common.EMTEST_BROWSER since it's updated later --- test/common.py | 23 ++++++++++++++++------- test/runner.py | 10 +++++++--- test/test_browser.py | 31 ++++++++++--------------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/test/common.py b/test/common.py index 943eeb2e593d2..4538f57ac2273 100644 --- a/test/common.py +++ b/test/common.py @@ -58,6 +58,7 @@ # For most browser tests this does not work, but it can # be useful for running pthread tests under node. EMTEST_BROWSER = None +EMTEST_BROWSER_AUTO_CONFIG = None EMTEST_HEADLESS = None EMTEST_DETECT_TEMPFILE_LEAKS = None EMTEST_SAVE_DIR = None @@ -151,6 +152,17 @@ def has_browser(): return EMTEST_BROWSER != '0' +CHROMIUM_BASED_BROWSERS = ['chrom', 'edge', 'opera'] + + +def is_chrome(): + return EMTEST_BROWSER and any(pattern in EMTEST_BROWSER.lower() for pattern in CHROMIUM_BASED_BROWSERS) + + +def is_firefox(): + return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower() + + def compiler_for(filename, force_c=False): if shared.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c: return EMXX @@ -2388,17 +2400,14 @@ def browser_open(cls, url): logger.info('No EMTEST_BROWSER set. Defaulting to `google-chrome`') EMTEST_BROWSER = 'google-chrome' - # If only the the browser is specified, use the default arguments used in circleci. - browser_args = shlex.split(EMTEST_BROWSER) - if len(browser_args) == 1: - logger.info('No EMTEST_BROWSER flags set. Defaulting to CI configuration.') - browser = browser_args[0] + if EMTEST_BROWSER_AUTO_CONFIG: + logger.info('Using default to CI configuration.') cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR if os.path.exists(cls.browser_data_dir): utils.delete_dir(cls.browser_data_dir) - if 'chrome' in browser: + if is_chrome(): config = BROWSER_CONFIG['chrome'] - elif 'firefox' in browser: + elif is_firefox(): config = BROWSER_CONFIG['firefox'] else: logger.warning("Unknown browser type, not using default flags.") diff --git a/test/runner.py b/test/runner.py index a916cc6a2c1d6..9f1c616497b9c 100755 --- a/test/runner.py +++ b/test/runner.py @@ -392,8 +392,10 @@ def parse_args(): help='Automatically update test expectations for tests that support it.') parser.add_argument('--browser', help='Command to launch web browser in which to run browser tests.') - parser.add_argument('--headed', action='store_true', - help='Run browser tests in regular (non-headless) mode.') + parser.add_argument('--headless', default='1', + help='Run browser in headless (default) or non-headless (--headless=0) mode.') + parser.add_argument('--browser-auto-config', type=bool, default=True, + help='Use the default CI browser configuration.') parser.add_argument('tests', nargs='*') parser.add_argument('--failfast', action='store_true') parser.add_argument('--start-at', metavar='NAME', help='Skip all tests up until ') @@ -409,6 +411,7 @@ def parse_args(): def configure(): common.EMTEST_BROWSER = os.getenv('EMTEST_BROWSER') + common.EMTEST_BROWSER_AUTO_CONFIG = os.getenv('EMTEST_BROWSER_AUTO_CONFIG') common.EMTEST_HEADLESS = os.getenv('EMTEST_HEADLESS') common.EMTEST_DETECT_TEMPFILE_LEAKS = int(os.getenv('EMTEST_DETECT_TEMPFILE_LEAKS', '0')) common.EMTEST_ALL_ENGINES = int(os.getenv('EMTEST_ALL_ENGINES', '0')) @@ -454,7 +457,8 @@ def set_env(name, option_value): os.environ[name] = value set_env('EMTEST_BROWSER', options.browser) - set_env('EMTEST_HEADLESS', not options.headed) + set_env('EMTEST_BROWSER_AUTO_CONFIG', options.browser_auto_config) + set_env('EMTEST_HEADLESS', options.headless) set_env('EMTEST_DETECT_TEMPFILE_LEAKS', options.detect_leaks) if options.save_dir: common.EMTEST_SAVE_DIR = 1 diff --git a/test/test_browser.py b/test/test_browser.py index e99af393e1051..b0cd0ced14166 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -20,7 +20,7 @@ from urllib.request import urlopen import common -from common import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting +from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, CHROMIUM_BASED_BROWSERS from common import create_file, parameterized, ensure_dir, disabled, test_file, WEBIDL_BINDER from common import read_file, EMRUN, no_wasm64, no_2gb, no_4gb, copytree from common import requires_wasm2js, parameterize, find_browser_test_file, with_all_sjlj @@ -130,23 +130,12 @@ def shell_with_script(shell_file, output_file, replacement): create_file(output_file, shell.replace('{{{ SCRIPT }}}', replacement)) -CHROMIUM_BASED_BROWSERS = ['chrom', 'edge', 'opera'] - - -def is_chrome(): - return EMTEST_BROWSER and any(pattern in EMTEST_BROWSER.lower() for pattern in CHROMIUM_BASED_BROWSERS) - - def no_chrome(note='chrome is not supported'): if is_chrome(): return unittest.skip(note) return lambda f: f -def is_firefox(): - return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower() - - def no_firefox(note='firefox is not supported'): if is_firefox(): return unittest.skip(note) @@ -162,7 +151,7 @@ def no_swiftshader(f): @wraps(f) def decorated(self, *args, **kwargs): - if is_chrome() and '--use-gl=swiftshader' in EMTEST_BROWSER: + if is_chrome() and '--use-gl=swiftshader' in common.EMTEST_BROWSER: self.skipTest('not compatible with swiftshader') return f(self, *args, **kwargs) @@ -210,7 +199,7 @@ class browser(BrowserCore): def setUpClass(cls): super().setUpClass() cls.browser_timeout = 60 - if EMTEST_BROWSER != 'node': + if common.EMTEST_BROWSER != 'node': print() print('Running the browser tests. Make sure the browser allows popups from localhost.') print() @@ -220,7 +209,7 @@ def proxy_to_worker(self): def require_jspi(self): if not is_chrome(): - self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') + self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') super().require_jspi() def post_manual_reftest(self): @@ -3305,7 +3294,7 @@ def test_cocos2d_hello(self): }) def test_async(self, args): if is_jspi(args) and not is_chrome(): - self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') + self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') for opts in (0, 1, 2, 3): print(opts) @@ -4977,7 +4966,7 @@ def test_embind_with_pthreads(self): }) def test_embind(self, args): if is_jspi(args) and not is_chrome(): - self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') + self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') if is_jspi(args) and self.is_wasm64(): self.skipTest('_emval_await fails') @@ -5597,9 +5586,9 @@ def test_no_browser(self): self.run_process([EMCC, test_file('test_emrun.c'), '--emrun', '-o', 'hello_world.html']) proc = subprocess.Popen([EMRUN, '--no-browser', '.', '--port=3333'], stdout=PIPE) try: - if EMTEST_BROWSER: + if common.EMTEST_BROWSER: print('Starting browser') - browser_cmd = shlex.split(EMTEST_BROWSER) + browser_cmd = shlex.split(common.EMTEST_BROWSER) browser = subprocess.Popen(browser_cmd + ['http://localhost:3333/hello_world.html']) try: while True: @@ -5638,11 +5627,11 @@ def test_emrun(self): '--log-stdout', self.in_dir('stdout.txt'), '--log-stderr', self.in_dir('stderr.txt')] - if EMTEST_BROWSER is not None: + if common.EMTEST_BROWSER is not None: # If EMTEST_BROWSER carried command line arguments to pass to the browser, # (e.g. "firefox -profile /path/to/foo") those can't be passed via emrun, # so strip them out. - browser_cmd = shlex.split(EMTEST_BROWSER) + browser_cmd = shlex.split(common.EMTEST_BROWSER) browser_path = browser_cmd[0] args_base += ['--browser', browser_path] if len(browser_cmd) > 1: From e9a8116b24b5ac47cad1cd5c94e02d7ece5bbfdf Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Fri, 22 Aug 2025 18:58:40 +0000 Subject: [PATCH 05/12] -put browser profile in out dir -copy firefox user.js file to profile --- test/common.py | 87 ++++++++++++++++++++++++++------------------ test/firefox_user.js | 5 +++ 2 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 test/firefox_user.js diff --git a/test/common.py b/test/common.py index 4538f57ac2273..90796ecfa1b7f 100644 --- a/test/common.py +++ b/test/common.py @@ -82,37 +82,52 @@ if 'EM_BUILD_VERBOSE' in os.environ: exit_with_error('EM_BUILD_VERBOSE has been renamed to EMTEST_BUILD_VERBOSE') + # Default flags used to run browsers in CI testing: -BROWSER_CONFIG = { - 'chrome': { - 'data_dir_flag': '--user-data-dir=', - 'default': ( - # --no-sandbox because we are running as root and chrome requires - # this flag for now: https://crbug.com/638180 - '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports', - '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"', - # The runners lack sound hardware so fallback to a dummy device (and - # bypass the user gesture so audio tests work without interaction) - '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required', - # Cache options. - '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', - ), - 'headless': - # Increase the window size to avoid flaky sdl tests see #24236. - '--headless=new --window-size=1024,768 --remote-debugging-port=1234', - }, - 'firefox': { - 'data_dir_flag': '-profile ', - 'default': {}, - 'headless': '-headless', - }, - 'other': { - 'data_dir_flag': '', - 'default': {}, - 'headless': '', - }, -} -DEFAULT_BROWSER_DATA_DIR = '/tmp/emscripten-profile' +class BrowserConfig: + def __init__(self, data_dir_flag, default_flags, headless_flags): + self.data_dir_flag = data_dir_flag + self.default_flags = default_flags + self.headless_flags = headless_flags + + def configure(self, data_dir): + pass + + +class ChromeConfig(BrowserConfig): + def __init__(self): + super().__init__( + data_dir_flag = '--user-data-dir=', + default_flags = ( + # --no-sandbox because we are running as root and chrome requires + # this flag for now: https://crbug.com/638180 + '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports', + '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"', + # The runners lack sound hardware so fallback to a dummy device (and + # bypass the user gesture so audio tests work without interaction) + '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required', + # Cache options. + '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', + ), + headless_flags = + # Increase the window size to avoid flaky sdl tests see #24236. + '--headless=new --window-size=1024,768 --remote-debugging-port=1234', + ) + + +class FirefoxConfig(BrowserConfig): + def __init__(self): + super().__init__( + data_dir_flag = '-profile ', + default_flags = {}, + headless_flags = '-headless', + ) + + def configure(self, data_dir): + shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js')) + + +DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile') # Special value for passing to assert_returncode which means we expect that program # to fail with non-zero return code, but we don't care about specifically which one. @@ -2405,16 +2420,18 @@ def browser_open(cls, url): cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR if os.path.exists(cls.browser_data_dir): utils.delete_dir(cls.browser_data_dir) + os.mkdir(cls.browser_data_dir) if is_chrome(): - config = BROWSER_CONFIG['chrome'] + config = ChromeConfig() elif is_firefox(): - config = BROWSER_CONFIG['firefox'] + config = FirefoxConfig() else: logger.warning("Unknown browser type, not using default flags.") - config = BROWSER_CONFIG['other'] - EMTEST_BROWSER += f" {config['data_dir_flag']}{cls.browser_data_dir} {' '.join(config['default'])}" + config = BrowserConfig() + EMTEST_BROWSER += f" {config.data_dir_flag}{cls.browser_data_dir} {' '.join(config.default_flags)}" if EMTEST_HEADLESS == '1': - EMTEST_BROWSER += f" {config['headless']}" + EMTEST_BROWSER += f" {config.headless_flags}" + config.configure(cls.browser_data_dir) if WINDOWS: # On Windows env. vars canonically use backslashes as directory delimiters, e.g. diff --git a/test/firefox_user.js b/test/firefox_user.js new file mode 100644 index 0000000000000..43fe2a14da8c7 --- /dev/null +++ b/test/firefox_user.js @@ -0,0 +1,5 @@ +user_pref("gfx.offscreencanvas.enabled", true); +user_pref("javascript.options.shared_memory", true); +user_pref("javascript.options.wasm_memory64", true); +user_pref("dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", true); +user_pref("media.autoplay.default", 0); From 1c554bce5bf12365fd93b1c64dc88760eee8144c Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Fri, 22 Aug 2025 22:57:19 +0000 Subject: [PATCH 06/12] default to headed mode --- .circleci/config.yml | 25 +++---------------------- test/runner.py | 4 ++-- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 173f4322053c2..3341135b42294 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -314,16 +314,7 @@ commands: name: run tests (<< parameters.title >>) environment: EMTEST_DETECT_TEMPFILE_LEAKS: "0" - # --no-sandbox because we are running as root and chrome requires - # this flag for now: https://crbug.com/638180 - CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports" - # Increase the window size to avoid flaky sdl tests see #24236. - CHROME_FLAGS_HEADLESS: "--headless=new --window-size=1024,768 --remote-debugging-port=1234" - CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features --js-flags=\"--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration\"" - CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito" - # The runners lack sound hardware so fallback to a dummy device (and - # bypass the user gesture so audio tests work without interaction) - CHROME_FLAGS_AUDIO: " --use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required" + EMTEST_HEADLESS: "1" command: | export EMTEST_BROWSER="/usr/bin/google-chrome" # There are tests in the browser test suite that using libraries @@ -373,18 +364,6 @@ commands: apt-get update -y apt-get install -q -y pulseaudio pulseaudio --start - - run: - name: configure firefox - command: | - # Note: the autoplay pref allows playback without user interaction - mkdir ~/tmp-firefox-profile/ - cat > ~/tmp-firefox-profile/user.js \<>) environment: @@ -398,6 +377,7 @@ commands: # OffscreenCanvas support is not yet done in Firefox. EMTEST_LACKS_OFFSCREEN_CANVAS: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" + EMTEST_HEADLESS: "1" DISPLAY: ":0" command: | export EMTEST_BROWSER="$HOME/firefox/firefox" @@ -423,6 +403,7 @@ commands: environment: EMTEST_LACKS_SOUND_HARDWARE: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" + EMTEST_HEADLESS: "1" command: | export EMTEST_BROWSER="/usr/bin/google-chrome" test/runner sockets diff --git a/test/runner.py b/test/runner.py index 9f1c616497b9c..df217ea7c3b4c 100755 --- a/test/runner.py +++ b/test/runner.py @@ -392,8 +392,8 @@ def parse_args(): help='Automatically update test expectations for tests that support it.') parser.add_argument('--browser', help='Command to launch web browser in which to run browser tests.') - parser.add_argument('--headless', default='1', - help='Run browser in headless (default) or non-headless (--headless=0) mode.') + parser.add_argument('--headless', action='store_true', + help='Run browser tests in headless mode.') parser.add_argument('--browser-auto-config', type=bool, default=True, help='Use the default CI browser configuration.') parser.add_argument('tests', nargs='*') From 8258ccb9d9772f967002f03ab76ce24cac5f6ce0 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Fri, 22 Aug 2025 23:31:04 +0000 Subject: [PATCH 07/12] fix env var --- test/common.py | 2 +- test/runner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/common.py b/test/common.py index 90796ecfa1b7f..2f1b28e7d3c4f 100644 --- a/test/common.py +++ b/test/common.py @@ -2429,7 +2429,7 @@ def browser_open(cls, url): logger.warning("Unknown browser type, not using default flags.") config = BrowserConfig() EMTEST_BROWSER += f" {config.data_dir_flag}{cls.browser_data_dir} {' '.join(config.default_flags)}" - if EMTEST_HEADLESS == '1': + if EMTEST_HEADLESS == 1: EMTEST_BROWSER += f" {config.headless_flags}" config.configure(cls.browser_data_dir) diff --git a/test/runner.py b/test/runner.py index df217ea7c3b4c..ee496d3478f89 100755 --- a/test/runner.py +++ b/test/runner.py @@ -393,7 +393,7 @@ def parse_args(): parser.add_argument('--browser', help='Command to launch web browser in which to run browser tests.') parser.add_argument('--headless', action='store_true', - help='Run browser tests in headless mode.') + help='Run browser tests in headless mode.', default=None) parser.add_argument('--browser-auto-config', type=bool, default=True, help='Use the default CI browser configuration.') parser.add_argument('tests', nargs='*') @@ -412,7 +412,7 @@ def parse_args(): def configure(): common.EMTEST_BROWSER = os.getenv('EMTEST_BROWSER') common.EMTEST_BROWSER_AUTO_CONFIG = os.getenv('EMTEST_BROWSER_AUTO_CONFIG') - common.EMTEST_HEADLESS = os.getenv('EMTEST_HEADLESS') + common.EMTEST_HEADLESS = int(os.getenv('EMTEST_HEADLESS', '0')) common.EMTEST_DETECT_TEMPFILE_LEAKS = int(os.getenv('EMTEST_DETECT_TEMPFILE_LEAKS', '0')) common.EMTEST_ALL_ENGINES = int(os.getenv('EMTEST_ALL_ENGINES', '0')) common.EMTEST_SKIP_SLOW = int(os.getenv('EMTEST_SKIP_SLOW', '0')) From da4239b4272540404d7245f7224fb2be92219783 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Tue, 26 Aug 2025 11:47:23 -0700 Subject: [PATCH 08/12] Update test/common.py Co-authored-by: Alon Zakai --- test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common.py b/test/common.py index 2f1b28e7d3c4f..7ae969ce668ab 100644 --- a/test/common.py +++ b/test/common.py @@ -2416,7 +2416,7 @@ def browser_open(cls, url): EMTEST_BROWSER = 'google-chrome' if EMTEST_BROWSER_AUTO_CONFIG: - logger.info('Using default to CI configuration.') + logger.info('Using default CI configuration.') cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR if os.path.exists(cls.browser_data_dir): utils.delete_dir(cls.browser_data_dir) From ec8109ab48c82d9044188c07393dfa1d77ecf3dc Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Tue, 26 Aug 2025 21:14:42 +0000 Subject: [PATCH 09/12] comments --- .circleci/config.yml | 6 ++-- test/common.py | 72 +++++++++++++++++--------------------------- 2 files changed, 31 insertions(+), 47 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3341135b42294..30215c53a80b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -315,8 +315,8 @@ commands: environment: EMTEST_DETECT_TEMPFILE_LEAKS: "0" EMTEST_HEADLESS: "1" + EMTEST_BROWSER: "/usr/bin/google-chrome" command: | - export EMTEST_BROWSER="/usr/bin/google-chrome" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module @@ -378,9 +378,9 @@ commands: EMTEST_LACKS_OFFSCREEN_CANVAS: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" EMTEST_HEADLESS: "1" + EMTEST_BROWSER: "$HOME/firefox/firefox" DISPLAY: ":0" command: | - export EMTEST_BROWSER="$HOME/firefox/firefox" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module @@ -404,8 +404,8 @@ commands: EMTEST_LACKS_SOUND_HARDWARE: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" EMTEST_HEADLESS: "1" + EMTEST_BROWSER: "/usr/bin/google-chrome" command: | - export EMTEST_BROWSER="/usr/bin/google-chrome" test/runner sockets - upload-test-results diff --git a/test/common.py b/test/common.py index 7ae969ce668ab..2feeb74f84ee9 100644 --- a/test/common.py +++ b/test/common.py @@ -84,46 +84,33 @@ # Default flags used to run browsers in CI testing: -class BrowserConfig: - def __init__(self, data_dir_flag, default_flags, headless_flags): - self.data_dir_flag = data_dir_flag - self.default_flags = default_flags - self.headless_flags = headless_flags - - def configure(self, data_dir): - pass - - -class ChromeConfig(BrowserConfig): - def __init__(self): - super().__init__( - data_dir_flag = '--user-data-dir=', - default_flags = ( - # --no-sandbox because we are running as root and chrome requires - # this flag for now: https://crbug.com/638180 - '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports', - '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"', - # The runners lack sound hardware so fallback to a dummy device (and - # bypass the user gesture so audio tests work without interaction) - '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required', - # Cache options. - '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', - ), - headless_flags = - # Increase the window size to avoid flaky sdl tests see #24236. - '--headless=new --window-size=1024,768 --remote-debugging-port=1234', - ) - - -class FirefoxConfig(BrowserConfig): - def __init__(self): - super().__init__( - data_dir_flag = '-profile ', - default_flags = {}, - headless_flags = '-headless', - ) - - def configure(self, data_dir): +class ChromeConfig: + data_dir_flag = '--user-data-dir=' + default_flags = ( + # --no-sandbox because we are running as root and chrome requires + # this flag for now: https://crbug.com/638180 + '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports', + '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"', + # The runners lack sound hardware so fallback to a dummy device (and + # bypass the user gesture so audio tests work without interaction) + '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required', + # Cache options. + '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', + ) + headless_flags = '--headless=new --window-size=1024,768 --remote-debugging-port=1234' + + @staticmethod + def configure(data_dir): + """Chrome has no special configuration step.""" + + +class FirefoxConfig: + data_dir_flag = '-profile ' + default_flags = () + headless_flags = '-headless' + + @staticmethod + def configure(data_dir): shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js')) @@ -2415,7 +2402,7 @@ def browser_open(cls, url): logger.info('No EMTEST_BROWSER set. Defaulting to `google-chrome`') EMTEST_BROWSER = 'google-chrome' - if EMTEST_BROWSER_AUTO_CONFIG: + if EMTEST_BROWSER_AUTO_CONFIG and (is_chrome() or is_firefox()): logger.info('Using default CI configuration.') cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR if os.path.exists(cls.browser_data_dir): @@ -2425,9 +2412,6 @@ def browser_open(cls, url): config = ChromeConfig() elif is_firefox(): config = FirefoxConfig() - else: - logger.warning("Unknown browser type, not using default flags.") - config = BrowserConfig() EMTEST_BROWSER += f" {config.data_dir_flag}{cls.browser_data_dir} {' '.join(config.default_flags)}" if EMTEST_HEADLESS == 1: EMTEST_BROWSER += f" {config.headless_flags}" From 8aaa1c1a1818f7f1c6facdefbe7492e3376963c4 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Tue, 26 Aug 2025 21:36:07 +0000 Subject: [PATCH 10/12] fix firefox --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30215c53a80b0..b7d3cb8628824 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -378,9 +378,9 @@ commands: EMTEST_LACKS_OFFSCREEN_CANVAS: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" EMTEST_HEADLESS: "1" - EMTEST_BROWSER: "$HOME/firefox/firefox" DISPLAY: ":0" command: | + export EMTEST_BROWSER="$HOME/firefox/firefox" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module From bed291f2bde803db5f869b5b1e3f33127b64f3a9 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Tue, 26 Aug 2025 21:49:24 +0000 Subject: [PATCH 11/12] review --- test/common.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/common.py b/test/common.py index 2feeb74f84ee9..bded42d383d3b 100644 --- a/test/common.py +++ b/test/common.py @@ -114,8 +114,6 @@ def configure(data_dir): shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js')) -DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile') - # Special value for passing to assert_returncode which means we expect that program # to fail with non-zero return code, but we don't care about specifically which one. NON_ZERO = -1 @@ -123,6 +121,8 @@ def configure(data_dir): TEST_ROOT = path_from_root('test') LAST_TEST = path_from_root('out/last_test.txt') +DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile') + WEBIDL_BINDER = shared.bat_suffix(path_from_root('tools/webidl_binder')) EMBUILDER = shared.bat_suffix(path_from_root('embuilder')) @@ -2402,7 +2402,7 @@ def browser_open(cls, url): logger.info('No EMTEST_BROWSER set. Defaulting to `google-chrome`') EMTEST_BROWSER = 'google-chrome' - if EMTEST_BROWSER_AUTO_CONFIG and (is_chrome() or is_firefox()): + if EMTEST_BROWSER_AUTO_CONFIG: logger.info('Using default CI configuration.') cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR if os.path.exists(cls.browser_data_dir): @@ -2412,6 +2412,8 @@ def browser_open(cls, url): config = ChromeConfig() elif is_firefox(): config = FirefoxConfig() + else: + exit_with_error("EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox or chrome.") EMTEST_BROWSER += f" {config.data_dir_flag}{cls.browser_data_dir} {' '.join(config.default_flags)}" if EMTEST_HEADLESS == 1: EMTEST_BROWSER += f" {config.headless_flags}" From 586f88a6e406909e3045c1d38d9587647291e234 Mon Sep 17 00:00:00 2001 From: Brendan Dahl Date: Tue, 26 Aug 2025 21:50:37 +0000 Subject: [PATCH 12/12] no auto delete --- test/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/common.py b/test/common.py index bded42d383d3b..f5dcd97f9058a 100644 --- a/test/common.py +++ b/test/common.py @@ -2384,8 +2384,6 @@ def browser_terminate(cls): logger.info('Browser did not respond to `terminate`. Using `kill`') cls.browser_proc.kill() cls.browser_proc.wait() - if cls.browser_data_dir: - utils.delete_dir(cls.browser_data_dir) cls.browser_data_dir = None @classmethod