Fix parallel browser test harness to also work with Firefox on Windows. (#25275)

This is a cleaned up version 2 of #25237.

-------

Two problems that I run into with the parallel browser harness on
Windows with Firefox:

1. On Windows, all the browser windows open up in the exact same X,Y
coordinate. Firefox has "is browser in the background" detection, which
causes all but one test foreground harness to hang when running any
`requestAnimationFrame()` based test.

2. Killing a Firefox browser process with

```py
proc = subprocess.Popen(['firefox'])
proc.kill()
```

does not work. This is because Firefox is a multiprocess browser, and
does not have a process hierarchy where the first launched process would
be some kind of a master process. (this is an old known issue, that can
be seen also in existing emrun.py code) There seems to be about 12
processes that come alive when running `subprocess.Popen(['firefox'])`.

--------------

To fix these issues:

1. ask the Windows Win32 API to move each spawned process to its own X,Y
location. Use a file-based multiprocessing lock + counter file to get
unique counter to each process.

2. Snapshot alive processes before spawning Firefox, and after. Then
acquire the delta of these two to find which process IDs correspond to
the launched Firefox.

Author: juj

.circleci/config.yml

@@ -395,6 +395,68 @@ commands:
             #  "Firefox is already running, but is not responding."
             # TODO: find out a way to shut down and restart firefox
       - upload-test-results
+  run-tests-firefox-windows:
+    description: "Runs emscripten tests under firefox on Windows"
+    parameters:
+      test_targets:
+        description: "Test suites to run"
+        type: string
+      title:
+        description: "Name of given test suite"
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: download firefox
+          shell: powershell.exe -ExecutionPolicy Bypass
+          command: |
+            # To download Firefox, we must first figure out what the latest Firefox version name is.
+            # This is because there does not exist a stable/static URL to download latest Firefox from.
+            $html = Invoke-WebRequest "https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/"
+            $zipLink = $html.Links |
+              Where-Object { $_.href -match "firefox-.*\.en-US\.win64\.zip$" } |
+              Select-Object -Last 1 -ExpandProperty href
+
+            # Download Win64 Firefox.
+            $nightlyUrl = "https://archive.mozilla.org$zipLink"
+            $outZip = "$env:TEMP\firefox_nightly.zip"
+
+            Write-Host "Downloading latest Firefox Nightly: $nightlyUrl"
+            Invoke-WebRequest -Uri $nightlyUrl -OutFile $outZip
+
+            # Extract to user home directory
+            $installDir = Join-Path $env:USERPROFILE "firefox"
+            if (Test-Path $installDir) { Remove-Item -Recurse -Force $installDir }
+            Expand-Archive -LiteralPath $outZip -DestinationPath $installDir
+
+            # Set environment variable for tests
+            $ffExe = Join-Path $installDir "firefox.exe"
+            [System.Environment]::SetEnvironmentVariable("EMTEST_BROWSER", $ffExe, "User")
+            $env:EMTEST_BROWSER = $ffExe
+            Write-Host "EMTEST_BROWSER set to $env:EMTEST_BROWSER"
+      - run:
+          name: run tests (<< parameters.title >>)
+          environment:
+            EMTEST_LACKS_GRAPHICS_HARDWARE: "1"
+            # TODO(https://github.com/emscripten-core/emscripten/issues/24205)
+            EMTEST_LACKS_SOUND_HARDWARE: "1"
+            EMTEST_LACKS_WEBGPU: "1"
+            # OffscreenCanvas support is not yet done in Firefox.
+            EMTEST_LACKS_OFFSCREEN_CANVAS: "1"
+            EMTEST_DETECT_TEMPFILE_LEAKS: "0"
+            EMTEST_HEADLESS: "1"
+            EMTEST_CORES: "2"
+            DISPLAY: ":0"
+          command: |
+            # There are tests in the browser test suite that using libraries
+            # that are not included by "./embuilder build ALL".  For example the
+            # PIC version of libSDL which is used by test_sdl2_misc_main_module
+            set EM_FROZEN_CACHE=
+            echo "-----"
+            echo "Running browser tests"
+            echo "-----"
+            test/runner << parameters.test_targets >>
+      - upload-test-results
   test-sockets-chrome:
     description: "Runs emscripten sockets tests under chrome"
     steps:
@@ -1065,6 +1127,19 @@ jobs:
           title: "sockets.test_nodejs_sockets_echo*"
           test_targets: "sockets.test_nodejs_sockets_echo*"
       - upload-test-results
+      # Run browser tests as well.
+      - run-tests-firefox-windows:
+          title: "browser on firefox on windows"
+          # skip browser.test_glbook, as it requires mingw32-make, which is not
+          # installed on CircleCI.
+          # skip browser.test_sdl2_mixer_wav_dash_l, fails to build on Windows
+          # on CircleCI (works locally)
+          test_targets: "
+            browser
+            skip:browser.test_glbook
+            skip:browser.test_sdl2_mixer_wav_dash_l
+            "
+      - upload-test-results
 
   test-mac-arm64:
     executor: mac-arm64

pyproject.toml

@@ -104,3 +104,7 @@ module = [
   "ply.*",
 ]
 ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = ["psutil", "win32gui", "win32process"]
+ignore_missing_imports = true

requirements-dev.txt

@@ -6,6 +6,7 @@
 
 coverage[toml]==6.5
 mypy==1.14
+psutil==7.0.0
 ruff==0.11.7
 types-requests==2.32.0.20241016
 unittest-xml-reporting==3.2.0

test/common.py

@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import psutil
 import re
 import shlex
 import shutil
@@ -36,7 +37,7 @@
 import clang_native
 import jsrun
 import line_endings
-from tools.shared import EMCC, EMXX, DEBUG
+from tools.shared import EMCC, EMXX, DEBUG, exe_suffix
 from tools.shared import get_canonical_temp_dir, path_from_root
 from tools.utils import MACOS, WINDOWS, read_file, read_binary, write_binary, exit_with_error
 from tools.settings import COMPILE_TIME_SETTINGS
@@ -87,6 +88,7 @@
 # file to track which tests were flaky so they can be graphed in orange color to
 # visually stand out.
 flaky_tests_log_filename = os.path.join(path_from_root('out/flaky_tests.txt'))
+browser_spawn_lock_filename = os.path.join(path_from_root('out/browser_spawn_lock'))
 
 
 # Default flags used to run browsers in CI testing:
@@ -116,6 +118,7 @@ class FirefoxConfig:
   data_dir_flag = '-profile '
   default_flags = ()
   headless_flags = '-headless'
+  executable_name = exe_suffix('firefox')
 
   @staticmethod
   def configure(data_dir):
@@ -945,8 +948,25 @@ def make_dir_writeable(dirname):
 
 
 def force_delete_dir(dirname):
-  make_dir_writeable(dirname)
-  utils.delete_dir(dirname)
+  """Deletes a directory. Returns whether deletion succeeded."""
+  if not os.path.exists(dirname):
+    return True
+  assert not os.path.isfile(dirname)
+
+  try:
+    make_dir_writeable(dirname)
+    utils.delete_dir(dirname)
+  except PermissionError as e:
+    # This issue currently occurs on Windows when running browser tests e.g.
+    # on Firefox browser. Killing Firefox browser is not 100% watertight, and
+    # occassionally a Firefox browser process can be left behind, holding on
+    # to a file handle, preventing the deletion from succeeding.
+    # We expect this issue to only occur on Windows.
+    if not WINDOWS:
+      raise e
+    print(f'Warning: Failed to delete directory "{dirname}"\n{e}')
+    return False
+  return True
 
 
 def force_delete_contents(dirname):
@@ -2508,6 +2528,81 @@ def configure_test_browser():
         EMTEST_BROWSER += f" {config.headless_flags}"
 
 
+def list_processes_by_name(exe_name):
+  pids = []
+  if exe_name:
+    for proc in psutil.process_iter():
+      try:
+        pinfo = proc.as_dict(attrs=['pid', 'name', 'exe'])
+        if pinfo['exe'] and exe_name in pinfo['exe'].replace('\\', '/').split('/'):
+          pids.append(psutil.Process(pinfo['pid']))
+      except psutil.NoSuchProcess: # E.g. "process no longer exists (pid=13132)" (code raced to acquire the iterator and process it)
+        pass
+
+  return pids
+
+
+class FileLock:
+  """Implements a filesystem-based mutex, with an additional feature that it
+  returns an integer counter denoting how many times the lock has been locked
+  before (during the current python test run instance)"""
+  def __init__(self, path):
+    self.path = path
+    self.counter = 0
+
+  def __enter__(self):
+    # Acquire the lock
+    while True:
+      try:
+        self.fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        break
+      except FileExistsError:
+        time.sleep(0.1)
+    # Return the locking count number
+    try:
+      self.counter = int(open(f'{self.path}_counter').read())
+    except Exception:
+      pass
+    return self.counter
+
+  def __exit__(self, *a):
+    # Increment locking count number before releasing the lock
+    with open(f'{self.path}_counter', 'w') as f:
+      f.write(str(self.counter + 1))
+    # And release the lock
+    os.close(self.fd)
+    try:
+      os.remove(self.path)
+    except Exception:
+      pass # Another process has raced to acquire the lock, and will delete it.
+
+
+def move_browser_window(pid, x, y):
+  """Utility function to move the top-level window owned by given process to
+  (x,y) coordinate. Used to ensure each browser window has some visible area."""
+  import win32gui
+  import win32process
+
+  def enum_windows_callback(hwnd, _unused):
+    _, win_pid = win32process.GetWindowThreadProcessId(hwnd)
+    if win_pid == pid and win32gui.IsWindowVisible(hwnd):
+      rect = win32gui.GetWindowRect(hwnd)
+      win32gui.MoveWindow(hwnd, x, y, rect[2] - rect[0], rect[3] - rect[1], True)
+    return True
+
+  win32gui.EnumWindows(enum_windows_callback, None)
+
+
+def increment_suffix_number(str_with_maybe_suffix):
+  match = re.match(r"^(.*?)(?:_(\d+))?$", str_with_maybe_suffix)
+  if match:
+    base, number = match.groups()
+    if number:
+      return f'{base}_{int(number) + 1}'
+
+  return f'{str_with_maybe_suffix}_1'
+
+
 class BrowserCore(RunnerCore):
   # note how many tests hang / do not send an output. if many of these
   # happen, likely something is broken and it is best to abort the test
@@ -2524,15 +2619,19 @@ def __init__(self, *args, **kwargs):
 
   @classmethod
   def browser_terminate(cls):
-    cls.browser_proc.terminate()
-    # If the browser doesn't shut down gracefully (in response to SIGTERM)
-    # after 2 seconds kill it with force (SIGKILL).
-    try:
-      cls.browser_proc.wait(2)
-    except subprocess.TimeoutExpired:
-      logger.info('Browser did not respond to `terminate`.  Using `kill`')
-      cls.browser_proc.kill()
-      cls.browser_proc.wait()
+    for proc in cls.browser_procs:
+      try:
+        proc.terminate()
+        # If the browser doesn't shut down gracefully (in response to SIGTERM)
+        # after 2 seconds kill it with force (SIGKILL).
+        try:
+          proc.wait(2)
+        except (subprocess.TimeoutExpired, psutil.TimeoutExpired):
+          logger.info('Browser did not respond to `terminate`.  Using `kill`')
+          proc.kill()
+          proc.wait()
+      except (psutil.NoSuchProcess, ProcessLookupError):
+        pass
 
   @classmethod
   def browser_restart(cls):
@@ -2551,9 +2650,18 @@ def browser_open(cls, url):
       if worker_id is not None:
         # Running in parallel mode, give each browser its own profile dir.
         browser_data_dir += '-' + str(worker_id)
-      if os.path.exists(browser_data_dir):
-        utils.delete_dir(browser_data_dir)
+
+      # Delete old browser data directory.
+      if WINDOWS:
+        # If we cannot (the data dir is in use on Windows), switch to another dir.
+        while not force_delete_dir(browser_data_dir):
+          browser_data_dir = increment_suffix_number(browser_data_dir)
+      else:
+        force_delete_dir(browser_data_dir)
+
+      # Recreate the new data directory.
       os.mkdir(browser_data_dir)
+
       if is_chrome():
         config = ChromeConfig()
       elif is_firefox():
@@ -2568,7 +2676,41 @@ def browser_open(cls, url):
 
     browser_args = shlex.split(browser_args)
     logger.info('Launching browser: %s', str(browser_args))
-    cls.browser_proc = subprocess.Popen(browser_args + [url])
+
+    if WINDOWS and is_firefox():
+      cls.launch_browser_harness_windows_firefox(worker_id, config, browser_args, url)
+    else:
+      cls.browser_procs = [subprocess.Popen(browser_args + [url])]
+
+  @classmethod
+  def launch_browser_harness_windows_firefox(cls, worker_id, config, browser_args, url):
+    ''' Dedicated function for launching browser harness on Firefox on Windows,
+    which requires extra care for window positioning and process tracking.'''
+
+    with FileLock(browser_spawn_lock_filename) as count:
+      # Firefox is a multiprocess browser. On Windows, killing the spawned
+      # process will not bring down the whole browser, but only one browser tab.
+      # So take a delta snapshot before->after spawning the browser to find
+      # which subprocesses we launched.
+      if worker_id is not None:
+        procs_before = list_processes_by_name(config.executable_name)
+      cls.browser_procs = [subprocess.Popen(browser_args + [url])]
+      # Give Firefox time to spawn its subprocesses. Use an increasing timeout
+      # as a crude way to account for system load.
+      if worker_id is not None:
+        time.sleep(2 + count * 0.3)
+        procs_after = list_processes_by_name(config.executable_name)
+      # Make sure that each browser window is visible on the desktop. Otherwise
+      # browser might decide that the tab is backgrounded, and not load a test,
+      # or it might not tick rAF()s forward, causing tests to hang.
+      if worker_id is not None and not EMTEST_HEADLESS:
+        # On Firefox on Windows we needs to track subprocesses that got created
+        # by Firefox. Other setups can use 'browser_proc' directly to terminate
+        # the browser.
+        cls.browser_procs = list(set(procs_after).difference(set(procs_before)))
+        # Wrap window positions on a Full HD desktop area modulo primes.
+        for proc in cls.browser_procs:
+          move_browser_window(proc.pid, (300 + count * 47) % 1901, (10 + count * 37) % 997)
 
   @classmethod
   def setUpClass(cls):

test/runner.py

@@ -564,6 +564,8 @@ def set_env(name, option_value):
 
   # Remove any old test files before starting the run
   utils.delete_file(common.flaky_tests_log_filename)
+  utils.delete_file(common.browser_spawn_lock_filename)
+  utils.delete_file(f'{common.browser_spawn_lock_filename}_counter')
 
   def prepend_default(arg):
     if arg.startswith('test_'):