android_adb use proto parsers for meminfo

Use new generated proto parsers to parse the output of dumpsys meminfo
instead of using regex.

Also plumb through action timeout so that we can specify a longer
timeout on tests that have lots of processes in a package.

Add tests for timeouts on Linux.

Change-Id: Ie4ef7de9a2112e9113e207b7638a8535a477c994
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/6638795
Reviewed-by: Kameron Lutes <[email protected]>
Commit-Queue: Charles Dick <[email protected]>

Author: cwd-goog

crossbench/action_runner/default_action_runner.py

@@ -368,7 +368,7 @@ def dump_meminfo_impl(self, run: Run, action: i_action.MeminfoAction) -> None:
       logging.warning("No meminfo probe for dump on %s", repr(self.info_stack))
       return
     assert isinstance(ctx, MeminfoProbeContext)
-    ctx.dump_meminfo(action.target, action.package)
+    ctx.dump_meminfo(action.target, action.timeout, action.package)
 
   def wait_for_download(self, run: Run,
                         action: i_action.WaitForDownloadAction) -> None:

crossbench/browsers/browser.py

@@ -205,8 +205,8 @@ def is_running_process(self) -> Optional[bool]:
     # we don't get the status back.
     return False
 
-  def meminfo(self) -> dict[str, ProcessMeminfo]:
-    return self.platform.meminfo(str(self.path))
+  def meminfo(self, timeout: dt.timedelta) -> dict[str, ProcessMeminfo]:
+    return self.platform.meminfo(str(self.path), timeout)
 
   @property
   def is_running(self) -> bool:

crossbench/browsers/chromium/webdriver.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import atexit
+import datetime as dt
 import logging
 import re
 import subprocess
@@ -146,8 +147,8 @@ def force_quit(self) -> None:
       self._restore_chrome_flags()
 
   @override
-  def meminfo(self) -> dict[str, ProcessMeminfo]:
-    return self.platform.meminfo(self.android_package)
+  def meminfo(self, timeout: dt.timedelta) -> dict[str, ProcessMeminfo]:
+    return self.platform.meminfo(self.android_package, timeout)
 
   def _restore_chrome_flags(self) -> None:
     atexit.unregister(self._restore_chrome_flags)

crossbench/plt/android_adb.py

@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import datetime as dt
 import functools
 import logging
 import math
@@ -25,6 +26,8 @@
 from crossbench.plt.posix import RemotePosixPlatform
 from crossbench.plt.process_meminfo import ProcessMeminfo
 
+from android_protoc import activitymanagerservice_pb2
+
 # Defines the Android permissions to be granted.
 # TODO(381985595): make this configurable.
 ANDROID_PERMISSIONS = ["POST_NOTIFICATIONS", "CAMERA", "RECORD_AUDIO"]
@@ -755,9 +758,13 @@ def processes(self,
     return res
 
   @override
-  def meminfo(self, process_name: str) -> dict[str, ProcessMeminfo]:
-    dumpsys_output = self.sh_stdout("dumpsys", "meminfo", "--package",
-                                    process_name)
+  def meminfo(
+      self, process_name: str, timeout: dt.timedelta = dt.timedelta(seconds=10)
+  ) -> dict[str, ProcessMeminfo]:
+    timeout_ms = int(timeout / dt.timedelta(milliseconds=1))
+    dumpsys_output = self.sh_stdout_bytes("dumpsys", "-T", str(timeout_ms),
+                                          "meminfo", "--proto", "--package",
+                                          process_name)
     return self._parse_dumpsys_meminfo(dumpsys_output)
 
   @functools.lru_cache(maxsize=1)
@@ -837,37 +844,21 @@ def display_resolution(self) -> tuple[int, int]:
   def user_id(self) -> int:
     return NumberParser.any_int(self.sh_stdout("am", "get-current-user"))
 
-  def _parse_dumpsys_meminfo(self,
-                             meminfo_output: str) -> dict[str, ProcessMeminfo]:
-    pid_sections = re.split(r"\*\* MEMINFO in pid (\d+) \[(.*?)] \*\*",
-                            meminfo_output)[1:]
+  _DUMPSYS_TIMEOUT_RE = re.compile(
+      rb"\*\*\* SERVICE '[^']+' DUMP TIMEOUT \(\d+ms\) EXPIRED \*\*\*")
 
+  def _parse_dumpsys_meminfo(
+      self, meminfo_output: bytes) -> dict[str, ProcessMeminfo]:
+    if self._DUMPSYS_TIMEOUT_RE.search(meminfo_output):
+      raise TimeoutError("dumpsys meminfo timed out")
+    proto_dump = activitymanagerservice_pb2.MemInfoDumpProto()
+    proto_dump.ParseFromString(meminfo_output)
     meminfos = {}
-
-    for i in range(0, len(pid_sections), 3):
-      pid = pid_sections[i]
-      process_name = pid_sections[i + 1].strip()
-      raw_process_info = pid_sections[i + 2]
-
-      pss_rss_total_v1 = re.search(
-          r"TOTAL PSS:\s+(?P<pss_total>\d+)\s+TOTAL RSS:\s+(?P<rss_total>\d+)"
-          r"\s+TOTAL SWAP \(KB\):\s+(?P<swap_total>\d+)", raw_process_info)
-
-      # TOTAL PSS: 91273  TOTAL RSS: 259028  TOTAL SWAP PSS: 209
-      pss_rss_total_v2 = re.search(
-          r"TOTAL PSS:\s+(?P<pss_total>\d+)\s+TOTAL RSS:\s+(?P<rss_total>\d+)"
-          r"\s+TOTAL SWAP PSS:\s+(?P<swap_total>\d+)", raw_process_info)
-
-      if pss_rss_total_v1:
-        pss_rss_total = pss_rss_total_v1
-      elif pss_rss_total_v2:
-        pss_rss_total = pss_rss_total_v2
-      else:
-        raise ValueError("Failed to parse meminfo.")
-
-      meminfos[process_name] = ProcessMeminfo(
-          pid=int(pid),
-          pss_total=int(pss_rss_total["pss_total"]),
-          rss_total=int(pss_rss_total["rss_total"]),
-          swap_total=int(pss_rss_total["swap_total"]))
+    for app_process in proto_dump.app_processes:
+      mem_info = app_process.process_memory.total_heap.mem_info
+      meminfos[app_process.process_memory.process_name] = ProcessMeminfo(
+          pid=app_process.process_memory.pid,
+          pss_total=mem_info.total_pss_kb,
+          rss_total=mem_info.total_rss_kb,
+          swap_total=mem_info.dirty_swap_pss_kb or mem_info.dirty_swap_kb)
     return meminfos

crossbench/plt/base.py

@@ -563,7 +563,9 @@ def process_info(self, process: ProcessLike) -> Optional[dict[str, Any]]:
     except proc_helper.PROCESS_NOT_FOUND_EXCEPTIONS:
       return None
 
-  def meminfo(self, process_name: str) -> dict[str, ProcessMeminfo]:
+  def meminfo(self, process_name: str,
+              timeout: dt.timedelta) -> dict[str, ProcessMeminfo]:
+    del process_name, timeout
     raise NotImplementedError(f"meminfo not implemented for {self}.")
 
   def foreground_process(self) -> Optional[dict[str, Any]]:

crossbench/plt/linux.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import dataclasses
+import datetime as dt
 import functools
 import logging
 import os
@@ -174,12 +175,17 @@ def display_details(self) -> tuple[DisplayInfo, ...]:
     return tuple()
 
   @override
-  def meminfo(self, process_name: str) -> dict[str, ProcessMeminfo]:
+  def meminfo(
+      self, process_name: str, timeout: dt.timedelta = dt.timedelta(seconds=10)
+  ) -> dict[str, ProcessMeminfo]:
+    deadline = dt.datetime.now() + timeout
     matching_pids = self.sh_stdout("pgrep", "-f", process_name).splitlines()
 
     meminfos: dict[str, ProcessMeminfo] = {}
 
     for pid in matching_pids:
+      if dt.datetime.now() > deadline:
+        raise TimeoutError("meminfo timed out")
       try:
         proc_name = self.cat(f"/proc/{pid}/cmdline")
       except (SubprocessError, OSError):

crossbench/probes/meminfo.py

@@ -6,6 +6,7 @@
 
 import csv
 import dataclasses
+import datetime as dt
 from typing import TYPE_CHECKING, Optional, Self, Type
 
 from typing_extensions import override
@@ -59,15 +60,16 @@ def start(self) -> None:
   def stop(self) -> None:
     pass
 
-  def dump_meminfo(self, target: MeminfoTarget, package: Optional[str]) -> None:
+  def dump_meminfo(self, target: MeminfoTarget, timeout: dt.timedelta,
+                   package: Optional[str]) -> None:
     timestamp = self.browser_platform.sh_stdout("date",
                                                 "+%Y-%m-%d %H:%M:%S").rstrip()
 
     if target is MeminfoTarget.BROWSER:
-      meminfo = self.browser.meminfo()
+      meminfo = self.browser.meminfo(timeout)
       package_path = self.browser.unique_name
     elif package is not None:
-      meminfo = self.browser_platform.meminfo(package)
+      meminfo = self.browser_platform.meminfo(package, timeout)
       package_path = pth.safe_filename(package).lower()
     else:
       raise ValueError("Cannot dump meminfo without package name.")

tests/crossbench/mock_browser.py

@@ -263,7 +263,8 @@ def current_url(self) -> str:
     return self._current_url
 
   @override
-  def meminfo(self) -> dict[str, ProcessMeminfo]:
+  def meminfo(self, timeout: dt.timedelta) -> dict[str, ProcessMeminfo]:
+    del timeout
     return {
         "process_1": ProcessMeminfo(1, 2, 3, 4),
         "process_2": ProcessMeminfo(2, 3, 4, 5)

tests/crossbench/mock_helper.py

@@ -190,14 +190,16 @@ def download_to(self, url: str, path: pth.AnyPath) -> pth.AnyPath:
       self.touch(path)
     return path
 
-  def expect_sh(self, *args: CmdArg,
-                result: str | ShResult = ShResult()) -> None:
+  def expect_sh(
+      self, *args: CmdArg, result: bytes | str | ShResult = ShResult()) -> None:
     if args:
       if self._expected_sh_cmds is None:
         self._expected_sh_cmds = []
       self._expected_sh_cmds.append(self._convert_sh_args(*args))
     if isinstance(result, str):
       result = ShResult(result)
+    if isinstance(result, bytes):
+      result = ShResult(result)
     assert isinstance(result, ShResult)
     self._sh_results.append(result)
 

tests/crossbench/plt/pb/dumpsys_meminfo.pb

@@ -0,0 +1,96 @@
+���S���S"�
+���
&com.android.chrome:privileged_process0)
+
+Native Heap� �
(�0�HP�"�j�4 �"(
+
+Dalvik Heap� ,(�0�0HP�5�� �*
+Dalvik Other� (�0�P�	*
+Stack� (�0P�*
+Ashmem�
+ �(�
0P�*
+	Other dev
+ �
8P�
*#
+.so mmap�#� �(�0��8�HP��*
+	.jar mmap�D0��
8DP��
*
+	.apk mmap��((�0��8�(P��*
+	.dex mmap�0�$P�$*
+	.oat mmap�
0�EP�E*"
+	.art mmap� �(�0�g8H�
P̎
*
+
+Other mmap, (0�8P�2
+Unknown�' �
(�&0�8P�*:-
+"
+TOTAL���0 �D(�b0��	8�1H�
P���}�B �J'���: �0�-8��
@�
P��
X�"`��h�x�\(8@,HP"�
+���
Ycom.android.chrome:sandboxed_process0:org.chromium.content.app.SandboxedProcessService0:0)
+
+Native Heap� �(�0�H P��_�* �"(
+
+Dalvik Heap� d(�0�0HP�4�� �*
+Dalvik Other� 8(�0�HP�	*
+Stack� (�0P�*
+Ashmem� �((�0P�-*
+	Other dev �
8P�
*
+.so mmap� �(H0��
HP��
*
+	.jar mmap�0��
P��
*#
+	.apk mmap��
�S �(�0��8�SPД*
+	.ttf mmap�&|0�M8|P�N*
+	.dex mmap�0�P�*
+	.oat mmapN0�<P�<*
+	.art mmap�
+ �(�0�eH�
P��
*
+
+Other mmap�(0�8�P�2
+Unknown�= �
(�<0�P�A:-
+"
+TOTAL��T �W(�\0�	8�WH�
P���q�8 �J(���W �0�G8��
@�
P̺
X�`��h�xĘ
(8@HP"�
+���
Ycom.android.chrome:sandboxed_process0:org.chromium.content.app.SandboxedProcessService0:1)
+
+Native Heap� �(�0�HP��_�* �"(
+
+Dalvik Heap� d(�0�0HP�4�� �*
+Dalvik Other� 8(�0�HP�	*
+Stack� (�0P�*
+Ashmem�
+ �#0P�#*
+	Other dev �
8P�
*
+.so mmap� �(H0��
HP��
*
+	.jar mmap�0��
P��
* 
+	.apk mmap�Xx �(�0��8xP��*
+	.ttf mmap�&,0�M8,P�N*
+	.dex mmap�0�P�*
+	.oat mmapN0�<P�<*
+	.art mmap�
+ �(�0�eH�
P��
*
+
+Other mmap�	(0�P�2
+Unknown�( �
(�'0�P�,:-
+"
+TOTAL��
�
 �R(�A0Ĵ8�
H�
P��	�q�8 �J'��� �0�+8ì
@�
P̺
X�`��h�x�w(8@HP"�
+�֟
com.android.chrome0
+#
+Native Heap��
 �
(��
0�8$HP��
ԕ��
 �%"+
+
+Dalvik Heap��
 ,(�{0�/HP��
��
�} �}*
+Dalvik Other�$ �(�"0�P�)*
+Stack�
+ (�
+0P�
+*
+Ashmem� �)(�P�1*
+	Other dev�
 �8�
P�*#
+.so mmap�%� �(�0Ȼ8�H
+P��*
+	.jar mmap�00��80P��*"
+	.apk mmap���
(�0ܟ8��
P܄*
+	.ttf mmap��0�8�P�
+*
+	.dex mmap�o�g0�&8�gP��
*
+	.oat mmap�0�dP�d*#
+	.art mmap��
 �(�0�}8�
HhP��
*!
+
+Other mmap�� �
(�0�8��P��2
+Unknown�� �
(��0�P��:3
+%
+TOTAL���� �S(Ĺ0��8��H�
P������ �
J-�
��
�� �
+0��8��@�
P��X��
`��h�
+x��
 
(8�
@iHPMX