diff --git a/.circleci/config.yml b/.circleci/config.yml index 125f04f94435a..b7d3cb8628824 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -314,18 +314,9 @@ commands: name: run tests (<< parameters.title >>) environment: EMTEST_DETECT_TEMPFILE_LEAKS: "0" - # --no-sandbox because we are running as root and chrome requires - # this flag for now: https://crbug.com/638180 - CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports" - # Increase the window size to avoid flaky sdl tests see #24236. - CHROME_FLAGS_HEADLESS: "--headless=new --window-size=1024,768 --remote-debugging-port=1234" - CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features --js-flags=\"--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration\"" - CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito" - # The runners lack sound hardware so fallback to a dummy device (and - # bypass the user gesture so audio tests work without interaction) - CHROME_FLAGS_AUDIO: " --use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required" + EMTEST_HEADLESS: "1" + EMTEST_BROWSER: "/usr/bin/google-chrome" command: | - export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE $CHROME_FLAGS_AUDIO" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module @@ -373,18 +364,6 @@ commands: apt-get update -y apt-get install -q -y pulseaudio pulseaudio --start - - run: - name: configure firefox - command: | - # Note: the autoplay pref allows playback without user interaction - mkdir ~/tmp-firefox-profile/ - cat > ~/tmp-firefox-profile/user.js \<>) environment: @@ -398,9 +377,10 @@ commands: # OffscreenCanvas support is not yet done in Firefox. EMTEST_LACKS_OFFSCREEN_CANVAS: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" + EMTEST_HEADLESS: "1" DISPLAY: ":0" command: | - export EMTEST_BROWSER="$HOME/firefox/firefox -headless -profile $HOME/tmp-firefox-profile/" + export EMTEST_BROWSER="$HOME/firefox/firefox" # There are tests in the browser test suite that using libraries # that are not included by "./embuilder build ALL". For example the # PIC version of libSDL which is used by test_sdl2_misc_main_module @@ -423,14 +403,9 @@ commands: environment: EMTEST_LACKS_SOUND_HARDWARE: "1" EMTEST_DETECT_TEMPFILE_LEAKS: "0" - # --no-sandbox becasue we are running as root and chrome requires - # this flag for now: https://crbug.com/638180 - CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile" - CHROME_FLAGS_HEADLESS: "--headless=new --remote-debugging-port=1234" - CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features" - CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito" + EMTEST_HEADLESS: "1" + EMTEST_BROWSER: "/usr/bin/google-chrome" command: | - export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE" test/runner sockets - upload-test-results diff --git a/test/common.py b/test/common.py index 090b19f0cb223..f5dcd97f9058a 100644 --- a/test/common.py +++ b/test/common.py @@ -45,8 +45,12 @@ # User can specify an environment variable EMTEST_BROWSER to force the browser # test suite to run using another browser command line than the default system -# browser. -# There are two special value that can be used here if running in an actual +# browser. If only the path to the browser executable is given, the tests +# will run in headless mode with a temporary profile with the same options +# used in CI. To use a custom start command specify the executable and command +# line flags. +# +# There are two special values that can be used here if running in an actual # browser is not desired: # EMTEST_BROWSER=0 : This will disable the actual running of the test and simply # verify that it compiles and links. @@ -54,6 +58,8 @@ # For most browser tests this does not work, but it can # be useful for running pthread tests under node. EMTEST_BROWSER = None +EMTEST_BROWSER_AUTO_CONFIG = None +EMTEST_HEADLESS = None EMTEST_DETECT_TEMPFILE_LEAKS = None EMTEST_SAVE_DIR = None # generally js engines are equivalent, testing 1 is enough. set this @@ -76,6 +82,38 @@ if 'EM_BUILD_VERBOSE' in os.environ: exit_with_error('EM_BUILD_VERBOSE has been renamed to EMTEST_BUILD_VERBOSE') + +# Default flags used to run browsers in CI testing: +class ChromeConfig: + data_dir_flag = '--user-data-dir=' + default_flags = ( + # --no-sandbox because we are running as root and chrome requires + # this flag for now: https://crbug.com/638180 + '--no-first-run -start-maximized --no-sandbox --enable-unsafe-swiftshader --use-gl=swiftshader --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports', + '--enable-experimental-webassembly-features --js-flags="--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration"', + # The runners lack sound hardware so fallback to a dummy device (and + # bypass the user gesture so audio tests work without interaction) + '--use-fake-device-for-media-stream --autoplay-policy=no-user-gesture-required', + # Cache options. + '--disk-cache-size=1 --media-cache-size=1 --disable-application-cache', + ) + headless_flags = '--headless=new --window-size=1024,768 --remote-debugging-port=1234' + + @staticmethod + def configure(data_dir): + """Chrome has no special configuration step.""" + + +class FirefoxConfig: + data_dir_flag = '-profile ' + default_flags = () + headless_flags = '-headless' + + @staticmethod + def configure(data_dir): + shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js')) + + # Special value for passing to assert_returncode which means we expect that program # to fail with non-zero return code, but we don't care about specifically which one. NON_ZERO = -1 @@ -83,6 +121,8 @@ TEST_ROOT = path_from_root('test') LAST_TEST = path_from_root('out/last_test.txt') +DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile') + WEBIDL_BINDER = shared.bat_suffix(path_from_root('tools/webidl_binder')) EMBUILDER = shared.bat_suffix(path_from_root('embuilder')) @@ -114,6 +154,17 @@ def has_browser(): return EMTEST_BROWSER != '0' +CHROMIUM_BASED_BROWSERS = ['chrom', 'edge', 'opera'] + + +def is_chrome(): + return EMTEST_BROWSER and any(pattern in EMTEST_BROWSER.lower() for pattern in CHROMIUM_BASED_BROWSERS) + + +def is_firefox(): + return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower() + + def compiler_for(filename, force_c=False): if shared.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c: return EMXX @@ -2323,9 +2374,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @classmethod - def browser_restart(cls): - # Kill existing browser - logger.info('Restarting browser process') + def browser_terminate(cls): cls.browser_proc.terminate() # If the browser doesn't shut down gracefully (in response to SIGTERM) # after 2 seconds kill it with force (SIGKILL). @@ -2335,6 +2384,13 @@ def browser_restart(cls): logger.info('Browser did not respond to `terminate`. Using `kill`') cls.browser_proc.kill() cls.browser_proc.wait() + cls.browser_data_dir = None + + @classmethod + def browser_restart(cls): + # Kill existing browser + logger.info('Restarting browser process') + cls.browser_terminate() cls.browser_open(cls.HARNESS_URL) @classmethod @@ -2343,6 +2399,24 @@ def browser_open(cls, url): if not EMTEST_BROWSER: logger.info('No EMTEST_BROWSER set. Defaulting to `google-chrome`') EMTEST_BROWSER = 'google-chrome' + + if EMTEST_BROWSER_AUTO_CONFIG: + logger.info('Using default CI configuration.') + cls.browser_data_dir = DEFAULT_BROWSER_DATA_DIR + if os.path.exists(cls.browser_data_dir): + utils.delete_dir(cls.browser_data_dir) + os.mkdir(cls.browser_data_dir) + if is_chrome(): + config = ChromeConfig() + elif is_firefox(): + config = FirefoxConfig() + else: + exit_with_error("EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox or chrome.") + EMTEST_BROWSER += f" {config.data_dir_flag}{cls.browser_data_dir} {' '.join(config.default_flags)}" + if EMTEST_HEADLESS == 1: + EMTEST_BROWSER += f" {config.headless_flags}" + config.configure(cls.browser_data_dir) + if WINDOWS: # On Windows env. vars canonically use backslashes as directory delimiters, e.g. # set EMTEST_BROWSER=C:\Program Files\Mozilla Firefox\firefox.exe @@ -2373,6 +2447,7 @@ def tearDownClass(cls): return cls.harness_server.terminate() print('[Browser harness server terminated]') + cls.browser_terminate() if WINDOWS: # On Windows, shutil.rmtree() in tearDown() raises this exception if we do not wait a bit: # WindowsError: [Error 32] The process cannot access the file because it is being used by another process. diff --git a/test/firefox_user.js b/test/firefox_user.js new file mode 100644 index 0000000000000..43fe2a14da8c7 --- /dev/null +++ b/test/firefox_user.js @@ -0,0 +1,5 @@ +user_pref("gfx.offscreencanvas.enabled", true); +user_pref("javascript.options.shared_memory", true); +user_pref("javascript.options.wasm_memory64", true); +user_pref("dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", true); +user_pref("media.autoplay.default", 0); diff --git a/test/runner.py b/test/runner.py index 74e9bb4f64c9d..ee496d3478f89 100755 --- a/test/runner.py +++ b/test/runner.py @@ -392,6 +392,10 @@ def parse_args(): help='Automatically update test expectations for tests that support it.') parser.add_argument('--browser', help='Command to launch web browser in which to run browser tests.') + parser.add_argument('--headless', action='store_true', + help='Run browser tests in headless mode.', default=None) + parser.add_argument('--browser-auto-config', type=bool, default=True, + help='Use the default CI browser configuration.') parser.add_argument('tests', nargs='*') parser.add_argument('--failfast', action='store_true') parser.add_argument('--start-at', metavar='NAME', help='Skip all tests up until ') @@ -407,6 +411,8 @@ def parse_args(): def configure(): common.EMTEST_BROWSER = os.getenv('EMTEST_BROWSER') + common.EMTEST_BROWSER_AUTO_CONFIG = os.getenv('EMTEST_BROWSER_AUTO_CONFIG') + common.EMTEST_HEADLESS = int(os.getenv('EMTEST_HEADLESS', '0')) common.EMTEST_DETECT_TEMPFILE_LEAKS = int(os.getenv('EMTEST_DETECT_TEMPFILE_LEAKS', '0')) common.EMTEST_ALL_ENGINES = int(os.getenv('EMTEST_ALL_ENGINES', '0')) common.EMTEST_SKIP_SLOW = int(os.getenv('EMTEST_SKIP_SLOW', '0')) @@ -451,6 +457,8 @@ def set_env(name, option_value): os.environ[name] = value set_env('EMTEST_BROWSER', options.browser) + set_env('EMTEST_BROWSER_AUTO_CONFIG', options.browser_auto_config) + set_env('EMTEST_HEADLESS', options.headless) set_env('EMTEST_DETECT_TEMPFILE_LEAKS', options.detect_leaks) if options.save_dir: common.EMTEST_SAVE_DIR = 1 diff --git a/test/test_browser.py b/test/test_browser.py index e99af393e1051..b0cd0ced14166 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -20,7 +20,7 @@ from urllib.request import urlopen import common -from common import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting +from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, CHROMIUM_BASED_BROWSERS from common import create_file, parameterized, ensure_dir, disabled, test_file, WEBIDL_BINDER from common import read_file, EMRUN, no_wasm64, no_2gb, no_4gb, copytree from common import requires_wasm2js, parameterize, find_browser_test_file, with_all_sjlj @@ -130,23 +130,12 @@ def shell_with_script(shell_file, output_file, replacement): create_file(output_file, shell.replace('{{{ SCRIPT }}}', replacement)) -CHROMIUM_BASED_BROWSERS = ['chrom', 'edge', 'opera'] - - -def is_chrome(): - return EMTEST_BROWSER and any(pattern in EMTEST_BROWSER.lower() for pattern in CHROMIUM_BASED_BROWSERS) - - def no_chrome(note='chrome is not supported'): if is_chrome(): return unittest.skip(note) return lambda f: f -def is_firefox(): - return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower() - - def no_firefox(note='firefox is not supported'): if is_firefox(): return unittest.skip(note) @@ -162,7 +151,7 @@ def no_swiftshader(f): @wraps(f) def decorated(self, *args, **kwargs): - if is_chrome() and '--use-gl=swiftshader' in EMTEST_BROWSER: + if is_chrome() and '--use-gl=swiftshader' in common.EMTEST_BROWSER: self.skipTest('not compatible with swiftshader') return f(self, *args, **kwargs) @@ -210,7 +199,7 @@ class browser(BrowserCore): def setUpClass(cls): super().setUpClass() cls.browser_timeout = 60 - if EMTEST_BROWSER != 'node': + if common.EMTEST_BROWSER != 'node': print() print('Running the browser tests. Make sure the browser allows popups from localhost.') print() @@ -220,7 +209,7 @@ def proxy_to_worker(self): def require_jspi(self): if not is_chrome(): - self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') + self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') super().require_jspi() def post_manual_reftest(self): @@ -3305,7 +3294,7 @@ def test_cocos2d_hello(self): }) def test_async(self, args): if is_jspi(args) and not is_chrome(): - self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') + self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') for opts in (0, 1, 2, 3): print(opts) @@ -4977,7 +4966,7 @@ def test_embind_with_pthreads(self): }) def test_embind(self, args): if is_jspi(args) and not is_chrome(): - self.skipTest(f'Current browser ({EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') + self.skipTest(f'Current browser ({common.EMTEST_BROWSER}) does not support JSPI. Only chromium-based browsers ({CHROMIUM_BASED_BROWSERS}) support JSPI today.') if is_jspi(args) and self.is_wasm64(): self.skipTest('_emval_await fails') @@ -5597,9 +5586,9 @@ def test_no_browser(self): self.run_process([EMCC, test_file('test_emrun.c'), '--emrun', '-o', 'hello_world.html']) proc = subprocess.Popen([EMRUN, '--no-browser', '.', '--port=3333'], stdout=PIPE) try: - if EMTEST_BROWSER: + if common.EMTEST_BROWSER: print('Starting browser') - browser_cmd = shlex.split(EMTEST_BROWSER) + browser_cmd = shlex.split(common.EMTEST_BROWSER) browser = subprocess.Popen(browser_cmd + ['http://localhost:3333/hello_world.html']) try: while True: @@ -5638,11 +5627,11 @@ def test_emrun(self): '--log-stdout', self.in_dir('stdout.txt'), '--log-stderr', self.in_dir('stderr.txt')] - if EMTEST_BROWSER is not None: + if common.EMTEST_BROWSER is not None: # If EMTEST_BROWSER carried command line arguments to pass to the browser, # (e.g. "firefox -profile /path/to/foo") those can't be passed via emrun, # so strip them out. - browser_cmd = shlex.split(EMTEST_BROWSER) + browser_cmd = shlex.split(common.EMTEST_BROWSER) browser_path = browser_cmd[0] args_base += ['--browser', browser_path] if len(browser_cmd) > 1: