codegen scripts for android_protoc

Script to check out Android frameworks base and proto_logging to get all
the proto definitions needed for the --proto output from adb shell
dumpsys.

Change-Id: Id41161587effa13bcbcc2343113efbc5635829d3
Reviewed-on: https://chromium-review.googlesource.com/c/crossbench/+/6638793
Reviewed-by: Camillo Bruni <[email protected]>
Commit-Queue: Charles Dick <[email protected]>

Author: cwd-goog

.yapfignore

@@ -0,0 +1,2 @@
+android_protoc/frameworks/**
+android_protoc/__init__.py

PRESUBMIT.py

@@ -47,6 +47,7 @@ def CheckChange(input_api, output_api, on_commit):
       input_api,
       output_api,
       files_to_check=pylint_file_patterns_to_check,
+      files_to_skip=[r"^android_protoc/frameworks.*", r"^third_party/.*"],
       pylintrc=".pylintrc",
       version="3.2")
 
@@ -164,8 +165,11 @@ def MypyFilesToCheck(input_api, on_commit, modified_py_files) -> list[str]:
   # TODO: enable mypy on all tests
   result = []
   for file in mypy_files_to_check:
-    if not file.startswith("tests/"):
-      result.append(file)
+    if file.startswith("tests/"):
+      continue
+    if file.startswith("android_protoc/"):
+      continue
+    result.append(file)
   return result
 
 

android_protoc/README.md

@@ -0,0 +1,11 @@
+# Android Proto
+This directory contains compiled proto files from Android frameworks.
+
+To update the list of checked in proto files, add the module you want to
+PROTO_MODULES in codegen/gen.py and then run it. You will need git and protoc in
+your environment.
+
+To use generated protos, import directly from android_proto, e.g.:
+```
+from android_protoc import activitymanagerservice_pb2
+```

android_protoc/codegen/gen.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# Copyright 2025 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Script to generate Android proto parsing files.
+#
+# Does a sparse checkout of the required Android repos, and call protoc on the
+# minimum required proto files to have working implementations of the types
+# listed in PROTO_MODULES below.
+#
+# Also generates __init__.py that exports the types listed in PROTO_MODULES, for
+# easier importing into Crossbench.
+
+from contextlib import contextmanager
+import importlib
+import os
+import re
+import shutil
+import subprocess
+
+from pathlib import Path
+import sys
+
+# List of modules we want to depend on. We will protoc this module and all its
+# dependencies.
+PROTO_MODULES = [
+    "frameworks.base.core.proto.android.server.activitymanagerservice_pb2"
+]
+
+# Map of repo name to optional list of folders to sparse checkout
+FRAMEWORKS_REPOS = {
+    "base": ["core/proto"],
+    "proto_logging": None,
+}
+
+
+@contextmanager
+def change_cwd(cwd):
+  previous_cwd = os.getcwd()
+  try:
+    os.chdir(str(cwd))
+    yield
+  finally:
+    os.chdir(previous_cwd)
+
+
+def run_and_log(args):
+  print("Running: ", " ".join(args))
+  subprocess.run(args, check=True)
+
+
+def checkout_frameworks_repo(root_dir, repo_name, sparse_dirs):
+  repo_dir = root_dir / ".protos" / "frameworks" / repo_name
+  repo_dir.mkdir(parents=True)
+  with change_cwd(repo_dir):
+    run_and_log(["git", "init"])
+    url = f"https://android.googlesource.com/platform/frameworks/{repo_name}"
+    run_and_log(["git", "remote", "add", "origin", url])
+    if sparse_dirs:
+      run_and_log(["git", "config", "core.sparsecheckout", "true"])
+      with (repo_dir / ".git" / "info" / "sparse-checkout").open(
+          "a", encoding="utf8") as sparse_file:
+        sparse_file.writelines(sparse_dirs)
+    run_and_log([
+        "git", "fetch", "--filter=blob:none", "--depth", "1", "origin", "main"
+    ])
+    run_and_log(["git", "checkout", "main"])
+
+
+# Patterns to parse import errors when a module has not been compiled yet.
+MISSING_MODULE_RE = re.compile("^No module named '([^']*)'")
+MISSING_FILE_RE = re.compile("^cannot import name '([^']*)_pb2' from '([^']*)'")
+
+
+def compile_proto(protos_dir, proto_file):
+  # TODO: check that the proto file is inside one of the repos.
+  print(f"compiling '{proto_file}'")
+  run_and_log(
+      ["protoc", "--python_out=.", f"--proto_path={protos_dir}", proto_file])
+
+
+def compile_imports(root_dir, protos_dir, module_name):
+  with change_cwd(root_dir):
+    while True:
+      try:
+        importlib.invalidate_caches()
+        importlib.import_module(module_name)
+        break
+      except ImportError as err:
+        module_missing = MISSING_MODULE_RE.match(str(err))
+        if module_missing:
+          dir_name = module_missing[1].replace(".", "/")
+          if dir_name.endswith("_pb2"):
+            compile_proto(protos_dir, dir_name[:-4] + ".proto")
+            continue
+          print(f"making folder '{dir_name}'")
+
+          dir_path = root_dir / dir_name
+          dir_path.mkdir()
+          # this folder is a module
+          with (dir_path / "__init__.py").open("x", encoding="utf8"):
+            pass
+          continue
+
+        file_missing = MISSING_FILE_RE.match(str(err))
+        if file_missing:
+          file_name = f"{file_missing[1]}.proto"
+          dir_name = file_missing[2].replace(".", "/")
+          proto_file = os.path.join(dir_name, file_name)
+          compile_proto(protos_dir, proto_file)
+          continue
+        raise err
+
+
+ROOT_MODULE_PROLOGUE = '''# Copyright 2025 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is generated by android_protoc/codegen/gen.py
+
+from android_protoc.sys_path import android_protoc_in_sys_path
+
+with android_protoc_in_sys_path():
+'''
+
+
+def write_root_module(root_dir):
+  with (root_dir / "__init__.py").open("w", encoding="utf8") as root_module:
+    root_module.write(ROOT_MODULE_PROLOGUE)
+    for module_name in PROTO_MODULES:
+      module_parts = module_name.split(".")
+      from_name = ".".join(module_parts[0:-1])
+      import_name = module_parts[-1]
+      root_module.write(f"  from {from_name} import {import_name}\n")
+
+
+def main():
+  root_dir = Path(__file__).parent.parent.resolve()
+  protos_dir = root_dir / ".protos"
+  frameworks_dir = root_dir / "frameworks"
+
+  # Delete all checked out and generated code.
+  if protos_dir.exists():
+    shutil.rmtree(protos_dir)
+  if frameworks_dir.exists():
+    shutil.rmtree(frameworks_dir)
+
+  # Check out a fresh copy of all repos.
+  for repo, sparse in FRAMEWORKS_REPOS.items():
+    checkout_frameworks_repo(root_dir, repo, sparse)
+
+  # Put python_out dir in the python search path, so that once we protoc a file,
+  # python will find it.
+  sys.path.insert(0, str(root_dir))
+
+  for module in PROTO_MODULES:
+    print("Compiling all dependencies for ", module)
+    compile_imports(root_dir, protos_dir, module)
+
+  write_root_module(root_dir)
+
+  # Delete the checked out Android code.
+  if protos_dir.exists():
+    shutil.rmtree(protos_dir)
+
+
+if not __name__ == "__main__":
+  raise RuntimeError("protoc should not be imported. Execute it as a script")
+
+main()

android_protoc/sys_path.py

@@ -0,0 +1,17 @@
+# Copyright 2025 The Chromium Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from contextlib import contextmanager
+from pathlib import Path
+import sys
+
+
+@contextmanager
+def android_protoc_in_sys_path():
+  prev_path = sys.path
+  sys.path = [str(Path(__file__).parent.resolve())] + prev_path
+  try:
+    yield None
+  finally:
+    sys.path = prev_path

mypy.ini

@@ -1,5 +1,6 @@
 [mypy]
 disable_error_code = import
+exclude = '/android_protoc/frameworks/'
 
 [google.auth.*]
 follow_untyped_imports = True