feat: detect sanitizer instrumentation in build

Author: tysmith

src/ffpuppet/core.py

@@ -25,7 +25,7 @@
 from .checks import CheckLogContents, CheckLogSize, CheckMemoryUsage
 from .display import DISPLAYS, DisplayMode
 from .exceptions import BrowserExecutionError, InvalidPrefs, LaunchError
-from .helpers import prepare_environment, wait_on_files
+from .helpers import detect_sanitizer, prepare_environment, wait_on_files
 from .minidump_parser import MDSW_URL, MinidumpParser
 from .process_tree import ProcessTree
 from .profile import Profile
@@ -292,7 +292,10 @@ def available_logs(self) -> frozenset[str]:
         return self._logs.available_logs()
 
     def build_launch_cmd(
-        self, bin_path: str, additional_args: Sequence[str] | None = None
+        self,
+        bin_path: str,
+        additional_args: Sequence[str] | None = None,
+        sanitizer: str | None = None,
     ) -> list[str]:
         """Build a command that can be used to launch the browser.
 
@@ -402,9 +405,9 @@ def build_launch_cmd(
                 "rr",
                 "record",
             ]
-            if getenv("RR_ASAN") == "1":
+            if sanitizer == "asan" or getenv("RR_ASAN") == "1":
                 rr_cmd.append("--asan")
-            if getenv("RR_TSAN") == "1":
+            if sanitizer == "tsan" or getenv("RR_TSAN") == "1":
                 rr_cmd.append("--tsan")
             if getenv("RR_CHAOS") == "1":
                 rr_cmd.append("--chaos")
@@ -730,6 +733,9 @@ def launch(
         bin_path = bin_path.resolve()
         if not bin_path.is_file() or not access(bin_path, X_OK):
             raise OSError(f"{bin_path} is not an executable")
+        detected_sanitizer = detect_sanitizer(bin_path)
+        LOG.debug("detected sanitizer: %s", detected_sanitizer)
+
         # need the path to help find symbols
         self._bin_path = bin_path.parent
 
@@ -805,7 +811,11 @@ def launch(
 
             self.profile.add_prefs(prefs)
 
-            cmd = self.build_launch_cmd(str(bin_path), additional_args=launch_args)
+            cmd = self.build_launch_cmd(
+                str(bin_path),
+                additional_args=launch_args,
+                sanitizer=detected_sanitizer,
+            )
 
             # open logs
             self._logs.add_log("stdout")
@@ -827,6 +837,7 @@ def launch(
                 env=prepare_environment(
                     self._logs.path / self._logs.PREFIX_SAN,
                     env_mod=env_mod,
+                    sanitizer=detected_sanitizer,
                 ),
                 shell=False,
                 stderr=stderr,

src/ffpuppet/helpers.py

@@ -8,6 +8,7 @@
 import sys
 from contextlib import suppress
 from logging import getLogger
+from mmap import ACCESS_READ, mmap
 from os import environ
 from pathlib import Path
 from subprocess import STDOUT, CalledProcessError, check_output
@@ -28,14 +29,16 @@
 else:
     IS_WINDOWS = False
 
+__author__ = "Tyson Smith"
+
 CERTUTIL = "certutil.exe" if IS_WINDOWS else "certutil"
 LOG = getLogger(__name__)
 
-__author__ = "Tyson Smith"
-
 
 def _configure_sanitizers(
-    orig_env: Mapping[str, str], log_path: Path
+    orig_env: Mapping[str, str],
+    log_path: Path,
+    symbolize: bool = False,
 ) -> dict[str, str]:
     """Copy environment and update default values in *SAN_OPTIONS entries.
     These values are only updated if they are not provided, with the exception of
@@ -44,13 +47,15 @@ def _configure_sanitizers(
     Args:
         orig_env: Current environment.
         log_path: Location to write sanitizer logs to.
+        symbolize: Enable automatic symbolizing. This should only used when required to
+            minimize memory usage.
 
     Returns:
         Environment with *SAN_OPTIONS defaults set.
     """
     env = dict(orig_env)
     # https://github.com/google/sanitizers/wiki/SanitizerCommonFlags
-    common_flags = [
+    common_flags = (
         ("abort_on_error", "false"),
         ("allocator_may_return_null", "true"),
         ("disable_coredump", "true"),
@@ -64,10 +69,7 @@ def _configure_sanitizers(
         ("handle_sigfpe", "true"),
         # set to be safe
         ("handle_sigill", "true"),
-        # do not automatically symbolize
-        # this should be done after to avoid hitting memory limitations
-        ("symbolize", "false"),
-    ]
+    )
 
     # setup Address Sanitizer options ONLY if not set manually in environment
     # https://github.com/google/sanitizers/wiki/AddressSanitizerFlags
@@ -99,6 +101,7 @@ def _configure_sanitizers(
     asan_config.add("strict_init_order", "true")
     # temporarily revert to default (false) until https://bugzil.la/1767068 is fixed
     # asan_config.add("strict_string_checks", "true")
+    asan_config.add("symbolize", "1" if symbolize else "0")
     env["ASAN_OPTIONS"] = str(asan_config)
 
     # setup Leak Sanitizer options ONLY if not set manually in environment
@@ -126,6 +129,7 @@ def _configure_sanitizers(
     tsan_config.add("log_path", f"'{log_path}'", overwrite=True)
     # This is an experimental feature added in Bug 1792757
     tsan_config.add("rss_limit_heap_profile", "true")
+    tsan_config.add("symbolize", "1" if symbolize else "0")
     env["TSAN_OPTIONS"] = str(tsan_config)
 
     # setup Undefined Behavior Sanitizer options ONLY if not set manually in environment
@@ -140,6 +144,7 @@ def _configure_sanitizers(
     ubsan_config.add("log_path", f"'{log_path}'", overwrite=True)
     ubsan_config.add("print_stacktrace", "1")
     ubsan_config.add("report_error_type", "1")
+    ubsan_config.add("symbolize", "1" if symbolize else "0")
     env["UBSAN_OPTIONS"] = str(ubsan_config)
 
     return env
@@ -186,6 +191,28 @@ def certutil_find(browser_bin: Path | None = None) -> str:
     return CERTUTIL
 
 
+def detect_sanitizer(binary: Path) -> str | None:
+    """Detect sanitizer instrumentation in browser build.
+
+    Args:
+        binary: Location of browser binary.
+
+    Returns:
+        Name of sanitizer in use or None.
+    """
+    with (
+        binary.open("rb") as bin_fp,
+        mmap(bin_fp.fileno(), 0, access=ACCESS_READ) as bmm,
+    ):
+        if bmm.find(b"__tsan_") != -1:
+            return "tsan"
+        if bmm.find(b"__asan_") != -1:
+            return "asan"
+        if bmm.find(b"__ubsan_") != -1:
+            return "ubsan"
+    return None
+
+
 def files_in_use(files: Iterable[Path]) -> Generator[tuple[Path, int, str]]:
     """Check if any of the given files are open.
     WARNING: This can be slow on Windows.
@@ -226,6 +253,7 @@ def files_in_use(files: Iterable[Path]) -> Generator[tuple[Path, int, str]]:
 def prepare_environment(
     sanitizer_log: Path,
     env_mod: Mapping[str, str | None] | None = None,
+    sanitizer: str | None = None,
 ) -> dict[str, str]:
     """Create environment that can be used when launching the browser.
 
@@ -235,6 +263,7 @@ def prepare_environment(
         env_mod: Environment modifier. Add, remove and update entries
                  in the prepared environment. Add/update by setting
                  value or remove entry by setting value to None.
+        sanitizer: Sanitizer in use.
 
     Returns:
         Environment to use when launching browser.
@@ -302,7 +331,9 @@ def prepare_environment(
         env.pop("MOZ_CRASHREPORTER_NO_REPORT", None)
         env.pop("MOZ_CRASHREPORTER_SHUTDOWN", None)
 
-    env = _configure_sanitizers(env, sanitizer_log)
+    # automatically symbolize traces when TSan is in use
+    # it is required for runtime TSan suppressions
+    env = _configure_sanitizers(env, sanitizer_log, symbolize=sanitizer == "tsan")
     # filter environment to avoid leaking sensitive information
     return {k: v for k, v in env.items() if "_SECRET" not in k}
 

src/ffpuppet/test_ffpuppet.py

@@ -1,7 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
-# pylint: disable=invalid-name,missing-docstring,protected-access
+# pylint: disable=missing-docstring,protected-access
 """ffpuppet tests"""
 
 import os
@@ -32,6 +32,7 @@
 
 
 class ReqHandler(BaseHTTPRequestHandler):
+    # pylint: disable=invalid-name
     def do_GET(self):
         self.send_response(200)
         self.end_headers()

src/ffpuppet/test_helpers.py

@@ -14,6 +14,7 @@
     _configure_sanitizers,
     certutil_available,
     certutil_find,
+    detect_sanitizer,
     files_in_use,
     prepare_environment,
     wait_on_files,
@@ -33,6 +34,13 @@ def test_helpers_01(tmp_path):
     assert opts.get("log_path") == f"'{tmp_path}'"
     assert "LSAN_OPTIONS" in env
     assert "UBSAN_OPTIONS" in env
+    # test symbolize
+    env = _configure_sanitizers({}, tmp_path, symbolize=False)
+    assert "symbolize=0" in env["ASAN_OPTIONS"]
+    assert "symbolize=0" in env["TSAN_OPTIONS"]
+    env = _configure_sanitizers({}, tmp_path, symbolize=True)
+    assert "symbolize=1" in env["ASAN_OPTIONS"]
+    assert "symbolize=1" in env["TSAN_OPTIONS"]
     # test with presets environment
     env = _configure_sanitizers(
         {
@@ -239,3 +247,19 @@ def test_certutil_find_01(tmp_path):
     certutil_bin.parent.mkdir()
     certutil_bin.touch()
     assert certutil_find(browser_bin) == str(certutil_bin)
+
+
+@mark.parametrize(
+    "bin_content, result",
+    [
+        (b"_foo", None),
+        (b"foo __asan_foo", "asan"),
+        (b"foo __tsan_foo", "tsan"),
+        (b"foo __ubsan_foo", "ubsan"),
+    ],
+)
+def test_detect_sanitizer_01(tmp_path, bin_content, result):
+    """test detect_sanitizer()"""
+    binary = tmp_path / "file.bin"
+    binary.write_bytes(bin_content)
+    assert detect_sanitizer(binary) == result