Fix parallel browser test harness to also work with Firefox on Windows.

.circleci/config.yml

@@ -395,6 +395,68 @@ commands:
             #  "Firefox is already running, but is not responding."
             # TODO: find out a way to shut down and restart firefox
       - upload-test-results
+  run-tests-firefox-windows:
+    description: "Runs emscripten tests under firefox on Windows"
+    parameters:
+      test_targets:
+        description: "Test suites to run"
+        type: string
+      title:
+        description: "Name of given test suite"
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: download firefox
+          shell: powershell.exe -ExecutionPolicy Bypass
+          command: |
+            # To download Firefox, we must first figure out what the latest Firefox version name is.
+            # This is because there does not exist a stable/static URL to download latest Firefox from.
+            $html = Invoke-WebRequest "https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central/"
+            $zipLink = $html.Links |
+              Where-Object { $_.href -match "firefox-.*\.en-US\.win64\.zip$" } |
+              Select-Object -Last 1 -ExpandProperty href
+
+            # Download Win64 Firefox.
+            $nightlyUrl = "https://archive.mozilla.org$zipLink"
+            $outZip = "$env:TEMP\firefox_nightly.zip"
+
+            Write-Host "Downloading latest Firefox Nightly: $nightlyUrl"
+            Invoke-WebRequest -Uri $nightlyUrl -OutFile $outZip
+
+            # Extract to user home directory
+            $installDir = Join-Path $env:USERPROFILE "firefox"
+            if (Test-Path $installDir) { Remove-Item -Recurse -Force $installDir }
+            Expand-Archive -LiteralPath $outZip -DestinationPath $installDir
+
+            # Set environment variable for tests
+            $ffExe = Join-Path $installDir "firefox.exe"
+            [System.Environment]::SetEnvironmentVariable("EMTEST_BROWSER", $ffExe, "User")
+            $env:EMTEST_BROWSER = $ffExe
+            Write-Host "EMTEST_BROWSER set to $env:EMTEST_BROWSER"
+      - run:
+          name: run tests (<< parameters.title >>)
+          environment:
+            EMTEST_LACKS_GRAPHICS_HARDWARE: "1"
+            # TODO(https://github.com/emscripten-core/emscripten/issues/24205)
+            EMTEST_LACKS_SOUND_HARDWARE: "1"
+            EMTEST_LACKS_WEBGPU: "1"
+            # OffscreenCanvas support is not yet done in Firefox.
+            EMTEST_LACKS_OFFSCREEN_CANVAS: "1"
+            EMTEST_DETECT_TEMPFILE_LEAKS: "0"
+            EMTEST_HEADLESS: "1"
+            EMTEST_CORES: "2"
+            DISPLAY: ":0"
+          command: |
+            # There are tests in the browser test suite that using libraries
+            # that are not included by "./embuilder build ALL".  For example the
+            # PIC version of libSDL which is used by test_sdl2_misc_main_module
+            set EM_FROZEN_CACHE=
+            echo "-----"
+            echo "Running browser tests"
+            echo "-----"
+            test/runner << parameters.test_targets >>
+      - upload-test-results
   test-sockets-chrome:
     description: "Runs emscripten sockets tests under chrome"
     steps:
@@ -1065,6 +1127,19 @@ jobs:
           title: "sockets.test_nodejs_sockets_echo*"
           test_targets: "sockets.test_nodejs_sockets_echo*"
       - upload-test-results
+      # Run browser tests as well.
+      - run-tests-firefox-windows:
+          title: "browser on firefox on windows"
+          # skip browser.test_glbook, as it requires mingw32-make, which is not
+          # installed on CircleCI.
+          # skip browser.test_sdl2_mixer_wav_dash_l, fails to build on Windows
+          # on CircleCI (works locally)
+          test_targets: "
+            browser
+            skip:browser.test_glbook
+            skip:browser.test_sdl2_mixer_wav_dash_l
+            "
+      - upload-test-results
 
   test-mac-arm64:
     executor: mac-arm64

pyproject.toml

@@ -104,3 +104,7 @@ module = [
   "ply.*",
 ]
 ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = ["psutil", "win32gui", "win32process"]
+ignore_missing_imports = true

requirements-dev.txt

@@ -6,6 +6,7 @@
 
 coverage[toml]==6.5
 mypy==1.14
+psutil==7.0.0
 ruff==0.11.7
 types-requests==2.32.0.20241016
 unittest-xml-reporting==3.2.0

test/common.py

@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import psutil
 import re
 import shlex
 import shutil
@@ -36,7 +37,7 @@
 import clang_native
 import jsrun
 import line_endings
-from tools.shared import EMCC, EMXX, DEBUG
+from tools.shared import EMCC, EMXX, DEBUG, exe_suffix
 from tools.shared import get_canonical_temp_dir, path_from_root
 from tools.utils import MACOS, WINDOWS, read_file, read_binary, write_binary, exit_with_error
 from tools.settings import COMPILE_TIME_SETTINGS
@@ -87,6 +88,7 @@
 # file to track which tests were flaky so they can be graphed in orange color to
 # visually stand out.
 flaky_tests_log_filename = os.path.join(path_from_root('out/flaky_tests.txt'))
+browser_spawn_lock_filename = os.path.join(path_from_root('out/browser_spawn_lock'))
 
 
 # Default flags used to run browsers in CI testing:
@@ -116,6 +118,7 @@ class FirefoxConfig:
   data_dir_flag = '-profile '
   default_flags = ()
   headless_flags = '-headless'
+  executable_name = exe_suffix('firefox')
 
   @staticmethod
   def configure(data_dir):
@@ -938,8 +941,25 @@ def make_dir_writeable(dirname):
 
 
 def force_delete_dir(dirname):
-  make_dir_writeable(dirname)
-  utils.delete_dir(dirname)
+  """Deletes a directory. Returns whether deletion succeeded."""
+  if not os.path.exists(dirname):
+    return True
+  assert not os.path.isfile(dirname)
+
+  try:
+    make_dir_writeable(dirname)
+    utils.delete_dir(dirname)
+  except PermissionError as e:
+    # This issue currently occurs on Windows when running browser tests e.g.
+    # on Firefox browser. Killing Firefox browser is not 100% watertight, and
+    # occassionally a Firefox browser process can be left behind, holding on
+    # to a file handle, preventing the deletion from succeeding.
+    # We expect this issue to only occur on Windows.
+    if not WINDOWS:
+      raise e
+    print(f'Warning: Failed to delete directory "{dirname}"\n{e}')
+    return False
+  return True
 
 
 def force_delete_contents(dirname):
@@ -2501,6 +2521,81 @@ def configure_test_browser():
         EMTEST_BROWSER += f" {config.headless_flags}"
 
 
+def list_processes_by_name(exe_name):
+  pids = []
+  if exe_name:
+    for proc in psutil.process_iter():
+      try:
+        pinfo = proc.as_dict(attrs=['pid', 'name', 'exe'])
+        if pinfo['exe'] and exe_name in pinfo['exe'].replace('\\', '/').split('/'):
+          pids.append(psutil.Process(pinfo['pid']))
+      except psutil.NoSuchProcess: # E.g. "process no longer exists (pid=13132)" (code raced to acquire the iterator and process it)
+        pass
+
+  return pids
+
+
+class FileLock:
+  """Implements a filesystem-based mutex, with an additional feature that it
+  returns an integer counter denoting how many times the lock has been locked
+  before (during the current python test run instance)"""
+  def __init__(self, path):
+    self.path = path
+    self.counter = 0
+
+  def __enter__(self):
+    # Acquire the lock
+    while True:
+      try:
+        self.fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        break
+      except FileExistsError:
+        time.sleep(0.1)
+    # Return the locking count number
+    try:
+      self.counter = int(open(f'{self.path}_counter').read())
+    except Exception:
+      pass
+    return self.counter
+
+  def __exit__(self, *a):
+    # Increment locking count number before releasing the lock
+    with open(f'{self.path}_counter', 'w') as f:
+      f.write(str(self.counter + 1))
+    # And release the lock
+    os.close(self.fd)
+    try:
+      os.remove(self.path)
+    except Exception:
+      pass # Another process has raced to acquire the lock, and will delete it.
+
+
+def move_browser_window(pid, x, y):
+  """Utility function to move the top-level window owned by given process to
+  (x,y) coordinate. Used to ensure each browser window has some visible area."""
+  import win32gui
+  import win32process
+
+  def enum_windows_callback(hwnd, _unused):
+    _, win_pid = win32process.GetWindowThreadProcessId(hwnd)
+    if win_pid == pid and win32gui.IsWindowVisible(hwnd):
+      rect = win32gui.GetWindowRect(hwnd)
+      win32gui.MoveWindow(hwnd, x, y, rect[2] - rect[0], rect[3] - rect[1], True)
+    return True
+
+  win32gui.EnumWindows(enum_windows_callback, None)
+
+
+def increment_suffix_number(str_with_maybe_suffix):
+  match = re.match(r"^(.*?)(?:_(\d+))?$", str_with_maybe_suffix)
+  if match:
+    base, number = match.groups()
+    if number:
+      return f'{base}_{int(number) + 1}'
+
+  return f'{str_with_maybe_suffix}_1'
+
+
 class BrowserCore(RunnerCore):
   # note how many tests hang / do not send an output. if many of these
   # happen, likely something is broken and it is best to abort the test
@@ -2517,15 +2612,19 @@ def __init__(self, *args, **kwargs):
 
   @classmethod
   def browser_terminate(cls):
-    cls.browser_proc.terminate()
-    # If the browser doesn't shut down gracefully (in response to SIGTERM)
-    # after 2 seconds kill it with force (SIGKILL).
-    try:
-      cls.browser_proc.wait(2)
-    except subprocess.TimeoutExpired:
-      logger.info('Browser did not respond to `terminate`.  Using `kill`')
-      cls.browser_proc.kill()
-      cls.browser_proc.wait()
+    for proc in cls.browser_procs:
+      try:
+        proc.terminate()
+        # If the browser doesn't shut down gracefully (in response to SIGTERM)
+        # after 2 seconds kill it with force (SIGKILL).
+        try:
+          proc.wait(2)
+        except (subprocess.TimeoutExpired, psutil.TimeoutExpired):
+          logger.info('Browser did not respond to `terminate`.  Using `kill`')
+          proc.kill()
+          proc.wait()
+      except (psutil.NoSuchProcess, ProcessLookupError):
+        pass
 
   @classmethod
   def browser_restart(cls):
@@ -2544,9 +2643,18 @@ def browser_open(cls, url):
       if worker_id is not None:
         # Running in parallel mode, give each browser its own profile dir.
         browser_data_dir += '-' + str(worker_id)
-      if os.path.exists(browser_data_dir):
-        utils.delete_dir(browser_data_dir)
+
+      # Delete old browser data directory.
+      if WINDOWS:
+        # If we cannot (the data dir is in use on Windows), switch to another dir.
+        while not force_delete_dir(browser_data_dir):
+          browser_data_dir = increment_suffix_number(browser_data_dir)
+      else:
+        force_delete_dir(browser_data_dir)
+
+      # Recreate the new data directory.
       os.mkdir(browser_data_dir)
+
       if is_chrome():
         config = ChromeConfig()
       elif is_firefox():
@@ -2561,7 +2669,41 @@ def browser_open(cls, url):
 
     browser_args = shlex.split(browser_args)
     logger.info('Launching browser: %s', str(browser_args))
-    cls.browser_proc = subprocess.Popen(browser_args + [url])
+
+    if WINDOWS and is_firefox():
+      cls.launch_browser_harness_windows_firefox(worker_id, config, browser_args, url)
+    else:
+      cls.browser_procs = [subprocess.Popen(browser_args + [url])]
+
+  @classmethod
+  def launch_browser_harness_windows_firefox(cls, worker_id, config, browser_args, url):
+    ''' Dedicated function for launching browser harness on Firefox on Windows,
+    which requires extra care for window positioning and process tracking.'''
+
+    with FileLock(browser_spawn_lock_filename) as count:
+      # Firefox is a multiprocess browser. On Windows, killing the spawned
+      # process will not bring down the whole browser, but only one browser tab.
+      # So take a delta snapshot before->after spawning the browser to find
+      # which subprocesses we launched.
+      if worker_id is not None:
+        procs_before = list_processes_by_name(config.executable_name)
+      cls.browser_procs = [subprocess.Popen(browser_args + [url])]
+      # Give Firefox time to spawn its subprocesses. Use an increasing timeout
+      # as a crude way to account for system load.
+      if worker_id is not None:
+        time.sleep(2 + count * 0.3)
+        procs_after = list_processes_by_name(config.executable_name)
+      # Make sure that each browser window is visible on the desktop. Otherwise
+      # browser might decide that the tab is backgrounded, and not load a test,
+      # or it might not tick rAF()s forward, causing tests to hang.
+      if worker_id is not None and not EMTEST_HEADLESS:
+        # On Firefox on Windows we needs to track subprocesses that got created
+        # by Firefox. Other setups can use 'browser_proc' directly to terminate
+        # the browser.
+        cls.browser_procs = list(set(procs_after).difference(set(procs_before)))
+        # Wrap window positions on a Full HD desktop area modulo primes.
+        for proc in cls.browser_procs:
+          move_browser_window(proc.pid, (300 + count * 47) % 1901, (10 + count * 37) % 997)
 
   @classmethod
   def setUpClass(cls):

test/runner.py

@@ -564,6 +564,8 @@ def set_env(name, option_value):
 
   # Remove any old test files before starting the run
   utils.delete_file(common.flaky_tests_log_filename)
+  utils.delete_file(common.browser_spawn_lock_filename)
+  utils.delete_file(f'{common.browser_spawn_lock_filename}_counter')
 
   def prepend_default(arg):
     if arg.startswith('test_'):