diff --git a/test/common.py b/test/common.py index d5f6b70456d26..cbe5c875c41e6 100644 --- a/test/common.py +++ b/test/common.py @@ -18,6 +18,7 @@ import json import logging import os +import plistlib import psutil import re import shlex @@ -52,6 +53,11 @@ # used in CI. To use a custom start command specify the executable and command # line flags. # +# Note that when specifying EMTEST_BROWSER to run tests on a Safari browser: +# the command line must point to the root of the app bundle, and not to the +# Safari executable inside the bundle. I.e. pass EMTEST_BROWSER=/Applications/Safari.app +# instead of EMTEST_BROWSER=/Applications/Safari.app/Contents/MacOS/Safari +# # There are two special values that can be used here if running in an actual # browser is not desired: # EMTEST_BROWSER=0 : This will disable the actual running of the test and simply @@ -130,6 +136,21 @@ def configure(data_dir): shutil.copy(test_file('firefox_user.js'), os.path.join(data_dir, 'user.js')) +class SafariConfig: + default_flags = ('', ) + executable_name = 'Safari' + # For the macOS 'open' command, pass + # --new: to make a new Safari app be launched, rather than add a tab to an existing Safari process/window + # --fresh: do not restore old tabs (e.g. if user had old navigated windows open) + # --background: Open the new Safari window behind the current Terminal window, to make following the test run more pleasing (this is for convenience only) + # -a : The path to the executable to open, in this case Safari + launch_prefix = ('open', '--new', '--fresh', '--background', '-a') + + @staticmethod + def configure(data_dir): + """ Safari has no special configuration step.""" + + # Special value for passing to assert_returncode which means we expect that program # to fail with non-zero return code, but we don't care about specifically which one. NON_ZERO = -1 @@ -202,6 +223,21 @@ def is_firefox(): return EMTEST_BROWSER and 'firefox' in EMTEST_BROWSER.lower() +def is_safari(): + return EMTEST_BROWSER and 'safari' in EMTEST_BROWSER.lower() + + +def get_safari_version(): + plist_path = os.path.join(EMTEST_BROWSER.strip(), 'Contents', 'version.plist') + version_str = plistlib.load(open(plist_path, 'rb')).get('CFBundleShortVersionString') + # Split into parts (major.minor.patch) + parts = (version_str.split('.') + ['0', '0', '0'])[:3] + # Convert each part into integers, discarding any trailing string, e.g. '13a' -> 13. + parts = [int(re.match(r"\d+", s).group()) if re.match(r"\d+", s) else 0 for s in parts] + # Return version as XXYYZZ + return parts[0] * 10000 + parts[1] * 100 + parts[2] + + def compiler_for(filename, force_c=False): if shared.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c: return EMXX @@ -2206,7 +2242,12 @@ def end_headers(self): self.send_header('Cross-Origin-Opener-Policy', 'same-origin') self.send_header('Cross-Origin-Embedder-Policy', 'require-corp') self.send_header('Cross-Origin-Resource-Policy', 'cross-origin') - self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate') + + self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate, private, max-age=0') + self.send_header('Expires', '0') + self.send_header('Pragma', 'no-cache') + self.send_header('Vary', '*') # Safari insists on caching if this header is not present in addition to the above + return SimpleHTTPRequestHandler.end_headers(self) def do_POST(self): # noqa: DC04 @@ -2293,9 +2334,13 @@ def do_GET(self): raise Exception('browser harness error, excessive response to server - test must be fixed! "%s"' % self.path) self.send_response(200) self.send_header('Content-type', 'text/plain') - self.send_header('Cache-Control', 'no-cache, must-revalidate') + + self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate, private, max-age=0') + self.send_header('Expires', '0') + self.send_header('Pragma', 'no-cache') + self.send_header('Vary', '*') # Safari insists on caching if this header is not present in addition to the above + self.send_header('Connection', 'close') - self.send_header('Expires', '-1') self.end_headers() self.wfile.write(b'OK') @@ -2418,6 +2463,8 @@ def configure_test_browser(): config = ChromeConfig() elif is_firefox(): config = FirefoxConfig() + elif is_safari(): + config = SafariConfig() if config: EMTEST_BROWSER += ' ' + ' '.join(config.default_flags) if EMTEST_HEADLESS == 1: @@ -2548,11 +2595,23 @@ def browser_restart(cls): def browser_open(cls, url): assert has_browser() browser_args = EMTEST_BROWSER + parallel_harness = worker_id is not None - if EMTEST_BROWSER_AUTO_CONFIG: + config = None + if is_chrome(): + config = ChromeConfig() + elif is_firefox(): + config = FirefoxConfig() + elif is_safari(): + config = SafariConfig() + elif EMTEST_BROWSER_AUTO_CONFIG: + exit_with_error(f'EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox, chrome and safari. EMTEST_BROWSER was "{EMTEST_BROWSER}"') + + # Prepare the browser data directory, if it uses one. + if EMTEST_BROWSER_AUTO_CONFIG and config and hasattr(config, 'data_dir_flag'): logger.info('Using default CI configuration.') browser_data_dir = DEFAULT_BROWSER_DATA_DIR - if worker_id is not None: + if parallel_harness: # Running in parallel mode, give each browser its own profile dir. browser_data_dir += '-' + str(worker_id) @@ -2567,12 +2626,6 @@ def browser_open(cls, url): # Recreate the new data directory. os.mkdir(browser_data_dir) - if is_chrome(): - config = ChromeConfig() - elif is_firefox(): - config = FirefoxConfig() - else: - exit_with_error(f'EMTEST_BROWSER_AUTO_CONFIG only currently works with firefox or chrome. EMTEST_BROWSER was "{EMTEST_BROWSER}"') if WINDOWS: # Escape directory delimiter backslashes for shlex.split. browser_data_dir = browser_data_dir.replace('\\', '\\\\') @@ -2580,39 +2633,51 @@ def browser_open(cls, url): browser_args += f' {config.data_dir_flag}"{browser_data_dir}"' browser_args = shlex.split(browser_args) + if hasattr(config, 'launch_prefix'): + browser_args = list(config.launch_prefix) + browser_args + logger.info('Launching browser: %s', str(browser_args)) - if WINDOWS and is_firefox(): - cls.launch_browser_harness_windows_firefox(worker_id, config, browser_args, url) + if (WINDOWS and is_firefox()) or is_safari(): + cls.launch_browser_harness_with_proc_snapshot_workaround(parallel_harness, config, browser_args, url) else: cls.browser_procs = [subprocess.Popen(browser_args + [url])] @classmethod - def launch_browser_harness_windows_firefox(cls, worker_id, config, browser_args, url): - ''' Dedicated function for launching browser harness on Firefox on Windows, - which requires extra care for window positioning and process tracking.''' + def launch_browser_harness_with_proc_snapshot_workaround(cls, parallel_harness, config, browser_args, url): + ''' Dedicated function for launching browser harness in scenarios where + we need to identify the launched browser processes via a before-after + subprocess snapshotting delta workaround.''' + # In order for this to work, each browser needs to be launched one at a time + # so that we know which process belongs to which browser. with FileLock(browser_spawn_lock_filename) as count: - # Firefox is a multiprocess browser. On Windows, killing the spawned - # process will not bring down the whole browser, but only one browser tab. - # So take a delta snapshot before->after spawning the browser to find - # which subprocesses we launched. - if worker_id is not None: + # Take a snapshot before spawning the browser to find which processes + # existed before launching the browser. + if parallel_harness or is_safari(): procs_before = list_processes_by_name(config.executable_name) + + # Browser launch cls.browser_procs = [subprocess.Popen(browser_args + [url])] - # Give Firefox time to spawn its subprocesses. Use an increasing timeout - # as a crude way to account for system load. - if worker_id is not None: + + # Give the browser time to spawn its subprocesses. Use an increasing + # timeout as a crude way to account for system load. + if parallel_harness or is_safari(): time.sleep(2 + count * 0.3) procs_after = list_processes_by_name(config.executable_name) + + # Take a snapshot again to find which processes exist after launching + # the browser. Then the newly launched browser processes are determined + # by the delta before->after. + cls.browser_procs = list(set(procs_after).difference(set(procs_before))) + if len(cls.browser_procs) == 0: + exit_with_error('Could not detect the launched browser subprocesses. The test harness will not be able to close the browser after testing is done, so aborting the test run here.') + + # Firefox on Windows quirk: # Make sure that each browser window is visible on the desktop. Otherwise # browser might decide that the tab is backgrounded, and not load a test, # or it might not tick rAF()s forward, causing tests to hang. - if worker_id is not None and not EMTEST_HEADLESS: - # On Firefox on Windows we needs to track subprocesses that got created - # by Firefox. Other setups can use 'browser_proc' directly to terminate - # the browser. - cls.browser_procs = list(set(procs_after).difference(set(procs_before))) + if WINDOWS and parallel_harness and not EMTEST_HEADLESS: # Wrap window positions on a Full HD desktop area modulo primes. for proc in cls.browser_procs: move_browser_window(proc.pid, (300 + count * 47) % 1901, (10 + count * 37) % 997) @@ -2686,6 +2751,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr if DEBUG: print('[browser launch:', html_file, ']') assert not (message and expected), 'run_browser expects `expected` or `message`, but not both' + if expected is not None: try: self.harness_in_queue.put(( diff --git a/test/parallel_testsuite.py b/test/parallel_testsuite.py index 769803b299bae..1ae052ef2ddba 100644 --- a/test/parallel_testsuite.py +++ b/test/parallel_testsuite.py @@ -35,7 +35,7 @@ def python_multiprocessing_structures_are_buggy(): def cap_max_workers_in_pool(max_workers, is_browser): if is_browser and 'EMTEST_CORES' not in os.environ and 'EMCC_CORES' not in os.environ: # TODO experiment with this number. In browser tests we'll be creating - # a chrome instance per worker which is expensive. + # a browser instance per worker which is expensive. max_workers = max_workers // 2 # Python has an issue that it can only use max 61 cores on Windows: https://github.com/python/cpython/issues/89240 if WINDOWS: diff --git a/test/test_browser.py b/test/test_browser.py index 5de846338809d..0ef1bc5689bc7 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -18,7 +18,7 @@ from urllib.request import urlopen import common -from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, CHROMIUM_BASED_BROWSERS +from common import BrowserCore, RunnerCore, path_from_root, has_browser, Reporting, is_chrome, is_firefox, is_safari, CHROMIUM_BASED_BROWSERS from common import create_file, parameterized, ensure_dir, disabled, flaky, test_file, WEBIDL_BINDER from common import read_file, EMRUN, no_wasm64, no_2gb, no_4gb, copytree, skip_if, skip_if_simple from common import requires_wasm2js, parameterize, find_browser_test_file, with_all_sjlj @@ -40,7 +40,12 @@ def sendheaders(s, extra=None, length=None): s.send_header("Content-Length", str(length)) s.send_header("Access-Control-Allow-Origin", "http://localhost:%s" % port) s.send_header('Cross-Origin-Resource-Policy', 'cross-origin') - s.send_header('Cache-Control', 'no-cache, no-store, must-revalidate') + + s.send_header('Cache-Control', 'no-cache, no-store, must-revalidate, private, max-age=0') + s.send_header('Expires', '0') + s.send_header('Pragma', 'no-cache') + s.send_header('Vary', '*') # Safari insists on caching if this header is not present in addition to the above + s.send_header("Access-Control-Expose-Headers", "Content-Length, Accept-Ranges") s.send_header("Content-type", "application/octet-stream") if support_byte_ranges: @@ -142,6 +147,8 @@ def is_swiftshader(_): no_firefox = skip_if('no_firefox', lambda _: is_firefox(), 'firefox is not supported') +no_safari = skip_if('no_safari', lambda _: is_safari(), 'safari is not supported') + def is_jspi(args): return '-sASYNCIFY=2' in args @@ -2797,6 +2804,7 @@ def test_webgl2_pbo(self): self.btest_exit('webgl2_pbo.c', cflags=['-sMAX_WEBGL_VERSION=2', '-lGL']) @no_firefox('fails on CI likely due to GPU drivers there') + @no_safari('TODO: Fails with report_result?5') # Safari 17.6 (17618.3.11.11.7, 17618) @requires_graphics_hardware def test_webgl2_sokol_mipmap(self): self.reftest('third_party/sokol/mipmap-emsc.c', 'third_party/sokol/mipmap-emsc.png', @@ -2955,6 +2963,7 @@ def test_sdl2_image_jpeg(self): @also_with_wasmfs @requires_graphics_hardware @with_all_sjlj + @no_safari('Test enables Wasm exceptions, so will not run in Safari 17.6 (17618.3.11.11.7, 17618)') # Safari 17.6 (17618.3.11.11.7, 17618) def test_sdl2_image_formats(self): shutil.copy(test_file('screenshot.png'), '.') shutil.copy(test_file('screenshot.jpg'), '.') @@ -3356,6 +3365,7 @@ def test_async_returnvalue(self, args): create_file('filey.txt', 'sync_tunnel\nsync_tunnel_bool\n') self.btest('test_async_returnvalue.c', '0', cflags=['-sASSERTIONS', '-sASYNCIFY', '-sASYNCIFY_IGNORE_INDIRECT', '--js-library', test_file('browser/test_async_returnvalue.js')] + args) + @no_safari('TODO: Never reports a result, so times out') # Safari 17.6 (17618.3.11.11.7, 17618) def test_async_bad_list(self): self.btest('test_async_bad_list.c', '0', cflags=['-sASYNCIFY', '-sASYNCIFY_ONLY=waka', '--profiling']) @@ -3408,6 +3418,7 @@ def test_modularize(self, args, code, opts): self.run_browser('a.html', '/report_result?0') @no_firefox('source phase imports not implemented yet in firefox') + @no_safari('croaks on line "import source wasmModule from \'./out.wasm\';"') # Safari 17.6 (17618.3.11.11.7, 17618) def test_source_phase_imports(self): self.compile_btest('browser_test_hello_world.c', ['-sEXPORT_ES6', '-sSOURCE_PHASE_IMPORTS', '-Wno-experimental', '-o', 'out.mjs']) create_file('a.html', ''' @@ -3794,6 +3805,7 @@ def test_pthread_gcc_64bit_atomic_fetch_and_op(self): # Test the old GCC atomic __sync_op_and_fetch builtin operations. @also_with_wasm2js + @no_safari('TODO: browser.test_pthread_gcc_atomic_op_and_fetch_wasm2js fails with "abort:Assertion failed: nand_and_fetch_data == -1"') # Safari 17.6 (17618.3.11.11.7, 17618) def test_pthread_gcc_atomic_op_and_fetch(self): self.cflags += ['-Wno-sync-fetch-and-nand-semantics-changed'] self.btest_exit('pthread/test_pthread_gcc_atomic_op_and_fetch.c', cflags=['-O3', '-pthread', '-sPTHREAD_POOL_SIZE=8']) @@ -3891,6 +3903,7 @@ def test_pthread_cleanup(self): '': ([],), 'spinlock': (['-DSPINLOCK_TEST'],), }) + @no_safari('TODO: browser.test_pthread_mutex and browser.test_pthread_mutex_spinlock both hang Safari') # Safari 17.6 (17618.3.11.11.7, 17618) def test_pthread_mutex(self, args): self.btest_exit('pthread/test_pthread_mutex.c', cflags=['-O3', '-pthread', '-sPTHREAD_POOL_SIZE=8'] + args) @@ -4078,6 +4091,7 @@ def test_pthread_safe_stack(self): 'no_leak': ['test_pthread_lsan_no_leak', []], }) @no_firefox('https://github.com/emscripten-core/emscripten/issues/15978') + @no_safari('TODO: browser.test_pthread_lsan_leak fails with /report_result?0') # Safari 17.6 (17618.3.11.11.7, 17618) def test_pthread_lsan(self, name, args): self.btest(Path('pthread', name + '.cpp'), expected='1', cflags=['-fsanitize=leak', '-pthread', '-sPROXY_TO_PTHREAD', '--pre-js', test_file('pthread', name + '.js')] + args) @@ -4089,6 +4103,7 @@ def test_pthread_lsan(self, name, args): 'leak': ['test_pthread_lsan_leak', ['-gsource-map']], 'no_leak': ['test_pthread_lsan_no_leak', []], }) + @no_safari('TODO: browser.test_pthread_asan_leak fails with /report_result?0') # Safari 17.6 (17618.3.11.11.7, 17618) def test_pthread_asan(self, name, args): self.btest(Path('pthread', name + '.cpp'), expected='1', cflags=['-fsanitize=address', '-pthread', '-sPROXY_TO_PTHREAD', '--pre-js', test_file('pthread', name + '.js')] + args) @@ -4102,6 +4117,7 @@ def test_pthread_asan_use_after_free(self): @no_2gb('ASAN + GLOBAL_BASE') @no_4gb('ASAN + GLOBAL_BASE') @no_firefox('https://github.com/emscripten-core/emscripten/issues/20006') + @no_safari('TODO: Hangs') # Safari Version 18.5 (20621.2.5.11.8) @also_with_wasmfs def test_pthread_asan_use_after_free_2(self): # similiar to test_pthread_asan_use_after_free, but using a pool instead @@ -4120,6 +4136,7 @@ def test_pthread_exit_process(self): args += ['--pre-js', test_file('core/pthread/test_pthread_exit_runtime.pre.js')] self.btest('core/pthread/test_pthread_exit_runtime.c', expected='onExit status: 42', cflags=args) + @no_safari('TODO: Fails with report_result?unexpected: [object ErrorEvent]') # Safari 17.6 (17618.3.11.11.7, 17618) def test_pthread_trap(self): create_file('pre.js', ''' if (typeof window === 'object' && window) { @@ -5367,6 +5384,7 @@ def test_wasmfs_fetch_backend_threaded(self, args): 'jspi': (['-Wno-experimental', '-sASYNCIFY=2'],), 'jspi_wasm_bigint': (['-Wno-experimental', '-sASYNCIFY=2', '-sWASM_BIGINT'],), }) + @no_safari('TODO: Fails with abort:Assertion failed: err == 0') # Safari 17.6 (17618.3.11.11.7, 17618) def test_wasmfs_opfs(self, args): if '-sASYNCIFY=2' in args: self.require_jspi() @@ -5376,6 +5394,7 @@ def test_wasmfs_opfs(self, args): self.btest_exit(test, cflags=args + ['-DWASMFS_RESUME']) @no_firefox('no OPFS support yet') + @no_safari('TODO: Fails with exception:Did not get expected EIO when unlinking file') # Safari 17.6 (17618.3.11.11.7, 17618) def test_wasmfs_opfs_errors(self): test = test_file('wasmfs/wasmfs_opfs_errors.c') postjs = test_file('wasmfs/wasmfs_opfs_errors_post.js') @@ -5450,6 +5469,7 @@ def test_manual_pthread_proxy_hammer(self, args): def test_assert_failure(self): self.btest('test_assert_failure.c', 'abort:Assertion failed: false && "this is a test"') + @no_safari('TODO: Fails with report_result?exception:rejected! / undefined') # Safari 17.6 (17618.3.11.11.7, 17618) def test_pthread_unhandledrejection(self): # Check that an unhandled promise rejection is propagated to the main thread # as an error. This test is failing if it hangs!