Load view hierarchies only once during processing (#98512)

This moves view hierarchy handling into its own file, and changes the
java language processor in such a way that it only loads the view
hierarchy once.

Previously, it would load them twice, once to collect all the window
class names, and then a second time to actually deobfuscate it.

Author: Swatinem

src/sentry/lang/dart/utils.py

@@ -109,46 +109,3 @@ def replace_symbol(match: re.Match[str]) -> str:
                 new_value = re.sub(INSTANCE_OF_VALUE_RE, replace_symbol, exception_value)
                 if new_value != exception_value:
                     exception["value"] = new_value
-
-
-# TODO(buenaflor): Add this back in when we decide to deobfuscate view hierarchies
-#
-# def _deobfuscate_view_hierarchy(event_data: dict[str, Any], project: Project, view_hierarchy):
-#     """
-#     Deobfuscates a view hierarchy in-place.
-
-#     If we're unable to fetch a dart symbols uuid, then the view hierarchy remains unmodified.
-#     """
-#     dart_symbols_uuids = get_debug_meta_image_ids(event_data)
-#     if len(dart_symbols_uuids) == 0:
-#         return
-
-#     with sentry_sdk.start_span(op="dartsymbolmap.deobfuscate_view_hierarchy_data"):
-#         for dart_symbols_uuid in dart_symbols_uuids:
-#             map = generate_dart_symbols_map(dart_symbols_uuid, project)
-#             if map is None:
-#                 return
-
-#             windows_to_deobfuscate = [*view_hierarchy.get("windows")]
-#             while windows_to_deobfuscate:
-#                 window = windows_to_deobfuscate.pop()
-
-#                 if window.get("type") is None:
-#                     # If there is no type, then skip this window
-#                     continue
-
-#                 matcher = re.match(VIEW_HIERARCHY_TYPE_REGEX, window.get("type"))
-#                 if not matcher:
-#                     continue
-#                 obfuscated_values = matcher.groups()
-#                 for obfuscated_value in obfuscated_values:
-#                     if obfuscated_value is not None and obfuscated_value in map:
-#                         window["type"] = window["type"].replace(
-#                             obfuscated_value, map[obfuscated_value]
-#                         )
-
-#                 if children := window.get("children"):
-#                     windows_to_deobfuscate.extend(children)
-
-# def deobfuscate_view_hierarchy(data):
-#     return deobfuscation_template(data, "dartsymbolmap", _deobfuscate_view_hierarchy)

src/sentry/lang/java/processing.py

@@ -3,19 +3,15 @@
 from collections.abc import Mapping
 from typing import Any
 
-import orjson
-
-from sentry.attachments import CachedAttachment, attachment_cache
-from sentry.ingest.consumer.processors import CACHE_TIMEOUT
 from sentry.lang.java.utils import JAVA_PLATFORMS, get_jvm_images, get_proguard_images
+from sentry.lang.java.view_hierarchies import ViewHierarchies
 from sentry.lang.native.error import SymbolicationFailed, write_error
 from sentry.lang.native.symbolicator import Symbolicator
 from sentry.models.eventerror import EventError
 from sentry.models.project import Project
 from sentry.models.release import Release
 from sentry.stacktraces.processing import find_stacktraces_in_data
 from sentry.utils import metrics
-from sentry.utils.cache import cache_key_for_event
 from sentry.utils.safe import get_path
 
 logger = logging.getLogger(__name__)
@@ -150,76 +146,6 @@ def _get_release_package(project: Project, release_name: str | None) -> str | No
     return release.package if release else None
 
 
-def _get_window_class_names(attachments: list[CachedAttachment]) -> list[str]:
-    """Returns the class names of all windows in all view hierarchies
-    contained in `attachments`."""
-
-    class_names = []
-    windows_to_deobfuscate = []
-
-    for attachment in attachments:
-        if attachment.type == "event.view_hierarchy":
-            view_hierarchy = orjson.loads(attachment_cache.get_data(attachment))
-            windows_to_deobfuscate.extend(view_hierarchy.get("windows"))
-
-    while windows_to_deobfuscate:
-        window = windows_to_deobfuscate.pop()
-        if window.get("type") is not None:
-            class_names.append(window["type"])
-        if children := window.get("children"):
-            windows_to_deobfuscate.extend(children)
-
-    return class_names
-
-
-def _deobfuscate_view_hierarchy(view_hierarchy: Any, class_names: dict[str, str]) -> None:
-    """Deobfuscates a view hierarchy in-place.
-
-    The `class_names` dict is used to resolve obfuscated to deobfuscated names. If
-    an obfuscated class name isn't present in `class_names`, it is left unchanged."""
-
-    windows_to_deobfuscate = [*view_hierarchy.get("windows")]
-
-    while windows_to_deobfuscate:
-        window = windows_to_deobfuscate.pop()
-        if (
-            window.get("type") is not None
-            and (mapped_type := class_names.get(window["type"])) is not None
-        ):
-            window["type"] = mapped_type
-        if children := window.get("children"):
-            windows_to_deobfuscate.extend(children)
-
-
-def _deobfuscate_view_hierarchies(
-    attachments: list[CachedAttachment], class_names: dict[str, str]
-) -> list[CachedAttachment]:
-    """Deobfuscates all view hierarchies contained in `attachments`, returning a new list of attachments.
-
-    Non-view-hierarchy attachments are unchanged.
-    """
-    new_attachments = []
-    for attachment in attachments:
-        if attachment.type == "event.view_hierarchy":
-            view_hierarchy = orjson.loads(attachment_cache.get_data(attachment))
-            _deobfuscate_view_hierarchy(view_hierarchy, class_names)
-            # Reupload to cache as a unchunked data
-            new_attachments.append(
-                CachedAttachment(
-                    type=attachment.type,
-                    id=attachment.id,
-                    name=attachment.name,
-                    content_type=attachment.content_type,
-                    data=orjson.dumps(view_hierarchy),
-                    chunks=None,
-                )
-            )
-        else:
-            new_attachments.append(attachment)
-
-    return new_attachments
-
-
 def map_symbolicator_process_jvm_errors(
     errors: list[dict[str, Any]] | None,
 ) -> list[dict[str, Any]]:
@@ -276,9 +202,8 @@ def process_jvm_stacktraces(symbolicator: Symbolicator, data: Any) -> Any:
     ]
 
     processable_exceptions = _get_exceptions_for_symbolication(data)
-    cache_key = cache_key_for_event(data)
-    attachments = [*attachment_cache.get(cache_key)]
-    window_class_names = _get_window_class_names(attachments)
+    view_hierarchies = ViewHierarchies(data)
+    window_class_names = view_hierarchies.get_window_class_names()
 
     metrics.incr("proguard.symbolicator.events")
 
@@ -341,7 +266,6 @@ def process_jvm_stacktraces(symbolicator: Symbolicator, data: Any) -> Any:
         raw_exc["type"] = exc["type"]
 
     classes = response.get("classes")
-    new_attachments = _deobfuscate_view_hierarchies(attachments, classes)
-    attachment_cache.set(cache_key, attachments=new_attachments, timeout=CACHE_TIMEOUT)
+    view_hierarchies.deobfuscate_and_save(classes)
 
     return data

src/sentry/lang/java/utils.py

@@ -2,14 +2,7 @@
 
 from typing import Any
 
-import orjson
-import sentry_sdk
-
-from sentry.attachments import CachedAttachment, attachment_cache
-from sentry.ingest.consumer.processors import CACHE_TIMEOUT
-from sentry.models.project import Project
 from sentry.stacktraces.processing import StacktraceInfo
-from sentry.utils.cache import cache_key_for_event
 from sentry.utils.safe import get_path
 
 # Platform values that should mark an event
@@ -53,44 +46,6 @@ def get_jvm_images(event: dict[str, Any]) -> set[str]:
     return images
 
 
-@sentry_sdk.trace
-def deobfuscation_template(data, map_type, deobfuscation_fn):
-    """
-    Template for operations involved in deobfuscating view hierarchies.
-
-    The provided deobfuscation function is expected to modify the view hierarchy dict in-place.
-    """
-    project = Project.objects.get_from_cache(id=data["project"])
-
-    cache_key = cache_key_for_event(data)
-    attachments = [*attachment_cache.get(cache_key)]
-
-    if not any(attachment.type == "event.view_hierarchy" for attachment in attachments):
-        return
-
-    new_attachments = []
-    for attachment in attachments:
-        if attachment.type == "event.view_hierarchy":
-            view_hierarchy = orjson.loads(attachment_cache.get_data(attachment))
-            deobfuscation_fn(data, project, view_hierarchy)
-
-            # Reupload to cache as a unchunked data
-            new_attachments.append(
-                CachedAttachment(
-                    type=attachment.type,
-                    id=attachment.id,
-                    name=attachment.name,
-                    content_type=attachment.content_type,
-                    data=orjson.dumps(view_hierarchy),
-                    chunks=None,
-                )
-            )
-        else:
-            new_attachments.append(attachment)
-
-    attachment_cache.set(cache_key, attachments=new_attachments, timeout=CACHE_TIMEOUT)
-
-
 def is_jvm_event(data: Any, stacktraces: list[StacktraceInfo]) -> bool:
     """Returns whether `data` is a JVM event, based on its platform,
     the supplied stacktraces, and its images."""

src/sentry/lang/java/view_hierarchies.py

@@ -0,0 +1,82 @@
+from typing import Any
+
+import orjson
+
+from sentry.attachments import CachedAttachment, attachment_cache
+from sentry.ingest.consumer.processors import CACHE_TIMEOUT
+from sentry.utils.cache import cache_key_for_event
+
+
+class ViewHierarchies:
+    def __init__(self, data: Any):
+        self._cache_key = cache_key_for_event(data)
+        self._view_hierarchies: list[tuple[CachedAttachment, Any]] = []
+        self._other_attachments: list[CachedAttachment] = []
+
+        for attachment in attachment_cache.get(self._cache_key):
+            if attachment.type == "event.view_hierarchy":
+                view_hierarchy = orjson.loads(attachment.data)
+                self._view_hierarchies.append((attachment, view_hierarchy))
+            else:
+                self._other_attachments.append(attachment)
+
+    def get_window_class_names(self) -> list[str]:
+        """
+        Returns the class names of all windows in all view hierarchies.
+        """
+        windows_to_deobfuscate = []
+        for _, view_hierarchy in self._view_hierarchies:
+            windows_to_deobfuscate.extend(view_hierarchy.get("windows"))
+
+        class_names = []
+        while windows_to_deobfuscate:
+            window = windows_to_deobfuscate.pop()
+            if window.get("type") is not None:
+                class_names.append(window["type"])
+            if children := window.get("children"):
+                windows_to_deobfuscate.extend(children)
+
+        return class_names
+
+    def deobfuscate_and_save(self, class_names: dict[str, str]):
+        """
+        Deobfuscates all view hierarchies in-place and persists any changes made.
+        """
+        if not self._view_hierarchies:
+            return
+
+        new_attachments: list[CachedAttachment] = []
+        for attachment, view_hierarchy in self._view_hierarchies:
+            _deobfuscate_view_hierarchy(view_hierarchy, class_names)
+            new_attachments.append(
+                CachedAttachment(
+                    type=attachment.type,
+                    id=attachment.id,
+                    name=attachment.name,
+                    content_type=attachment.content_type,
+                    data=orjson.dumps(view_hierarchy),
+                    chunks=None,
+                )
+            )
+
+        attachments = self._other_attachments + new_attachments
+        attachment_cache.set(self._cache_key, attachments, timeout=CACHE_TIMEOUT)
+
+
+def _deobfuscate_view_hierarchy(view_hierarchy: Any, class_names: dict[str, str]):
+    """
+    Deobfuscates a view hierarchy in-place.
+
+    The `class_names` dict is used to resolve obfuscated to deobfuscated names. If
+    an obfuscated class name isn't present in `class_names`, it is left unchanged.
+    """
+    windows_to_deobfuscate = list(view_hierarchy.get("windows"))
+    while windows_to_deobfuscate:
+        window = windows_to_deobfuscate.pop()
+        if (
+            window.get("type") is not None
+            and (mapped_type := class_names.get(window["type"])) is not None
+        ):
+            window["type"] = mapped_type
+        if children := window.get("children"):
+            windows_to_deobfuscate.extend(children)

tests/sentry/lang/dart/test_utils.py

@@ -539,45 +539,3 @@ def test_deobfuscate_exception_value_without_type() -> None:
         # Third: no pattern; unchanged
         assert data["exception"]["values"][2]["type"] is None
         assert data["exception"]["values"][2]["value"] == "No pattern here"
-
-
-# @mock.patch("sentry.lang.dart.utils.generate_dart_symbols_map", return_value=MOCK_DEBUG_MAP)
-# @mock.patch("sentry.lang.dart.utils.get_debug_meta_image_ids", return_value=["test-uuid"])
-# def test_view_hierarchy_deobfuscation(
-#     mock_images: mock.MagicMock, mock_map: mock.MagicMock
-# ) -> None:
-#     test_view_hierarchy = {
-#         "windows": [
-#             {
-#                 "type": "mD",
-#                 "children": [
-#                     {
-#                         "type": "er",
-#                         "children": [
-#                             {"type": "_YMa<er>", "children": [{"type": "_NativeInteger"}]}
-#                         ],
-#                     },
-#                 ],
-#             }
-#         ]
-#     }
-#     _deobfuscate_view_hierarchy(mock.Mock(), mock.Mock(), test_view_hierarchy)
-
-#     assert test_view_hierarchy == {
-#         "windows": [
-#             {
-#                 "type": "ButtonTheme",
-#                 "children": [
-#                     {
-#                         "type": "SemanticsAction",
-#                         "children": [
-#                             {
-#                                 "type": "_entry<SemanticsAction>",
-#                                 "children": [{"type": "_NativeInteger"}],
-#                             }
-#                         ],
-#                     }
-#                 ],
-#             }
-#         ]
-#     }