Revert "build: Speed up JUnit test listing"

This reverts commit 5a395fa64508945192896b2a10cf97c730e51ac6.

Reason for revert: Speculative fix for flake endorser failures

Original change's description:
> build: Speed up JUnit test listing
>
> Listing JUnit tests can be very slow, 17+ seconds slow. This CL
> optimizes the process by parallelizing test class discovery and
> pre-filtering classes against the gtest filter(s) before passing the
> filtered classes to JUnit. This avoids expensive reflection on classes
> that would have been filtered out. Now it takes just 1 second.
>
> Since the underlying test listing is now significantly faster, the
> Python-side test list cache in local_machine_junit_test_run.py is no
> longer necessary and has been removed. This avoids staleness issues and
> reduces the number of things to examine when a test fails (i.e. no
> longer need to wonder whether the right tests ran due to the cache being
> stale).
>
> [email protected]
>
> Bug: 40878339, 448894165
> Change-Id: I7a59372fa2e940e60fbab2d36989333b5b422173
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/7173486
> Reviewed-by: Andrew Grieve <[email protected]>
> Auto-Submit: Peter Wen <[email protected]>
> Commit-Queue: Peter Wen <[email protected]>
> Reviewed-by: Henrique Nakashima <[email protected]>
> Cr-Commit-Position: refs/heads/main@{#1547984}

Bug: 40878339, 448894165, 462739429
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 40878339, 448894165
Change-Id: Id90abc870041d707c5cf154cecf9bc37b0ed4eba
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/7187501
Bot-Commit: Rubber Stamper <[email protected]>
Commit-Queue: Andrew Grieve <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1548572}
NOKEYCHECK=True
GitOrigin-RevId: 6847a7b2d60715df51f813afee0689f7ffcccdd3

Author: agrieve

android/pylib/local/machine/local_machine_junit_test_run.py

@@ -7,6 +7,7 @@
 import logging
 import multiprocessing
 import os
+import pathlib
 import queue
 import re
 import subprocess
@@ -16,6 +17,7 @@
 import time
 import zipfile
 from concurrent.futures import ThreadPoolExecutor
+import ast
 
 from devil.utils import cmd_helper
 from py_utils import tempfile_ext
@@ -25,6 +27,9 @@
 from pylib.constants import host_paths
 from pylib.results import json_results
 
+sys.path.append(str(pathlib.Path(__file__).parents[3] / 'gyp'))
+from util import md5_check
+
 # Chosen after timing test runs of chrome_junit_tests with 7,16,32,
 # and 64 workers in threadpool and different classes_per_job.
 _MAX_TESTS_PER_JOB = 128
@@ -210,15 +215,77 @@ def _MakeJob(self, shard_id, temp_dir, test_group, properties_jar_path,
                 json_config=job_json_config,
                 json_results_path=json_results_path)
 
-  def _GetJsonConfig(self):
-    with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
-      return self._QueryTestJsonConfig(temp_dir,
-                                       allow_debugging=False,
-                                       enable_shadow_allowlist=True)
+  def _GetJsonConfig(self, list_only: bool = False):
+    # The test list is expensive to generate, so cache it. The cache is
+    # invalidated when the classpath or this file changes.
+    cache_dir = pathlib.Path(
+        constants.GetOutDirectory()) / 'junit_test_list_cache'
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Cache --list-tests separately so that changing test filters does not
+    # invalidate its cache.
+    if list_only:
+      record_path = (cache_dir / f'{self._test_instance.suite}_list.stamp')
+      cached_test_list_path = (cache_dir /
+                               f'{self._test_instance.suite}_list.json')
+    else:
+      record_path = cache_dir / f'{self._test_instance.suite}.stamp'
+      cached_test_list_path = cache_dir / f'{self._test_instance.suite}.json'
+
+    # Extract the classpath from the wrapper script.
+    wrapper_script_source = ''
+    print(f'Wrapper path: {self._wrapper_path}')
+    with open(self._wrapper_path) as f:
+      wrapper_script_source = f.read()
+    classpath_re = re.compile(r'classpath = (\[[^\]]+\])', re.MULTILINE)
+    matches = classpath_re.search(wrapper_script_source)
+    classpath_str = matches.group(1)
+    classpath_list = ast.literal_eval(classpath_str)
+    # The paths in the wrapper script are relative to the script's location. We
+    # need to make them relative to the current working directory.
+    wrapper_dir = pathlib.Path(self._wrapper_path).parent
+    classpath = [str(wrapper_dir / p) for p in classpath_list]
+
+    # Filters affect the test list so changes in filter should invalidate the
+    # cache.
+    input_strings = self._GetFilterArgs()
+
+    # We don't want to rebuild the cache if only method implementations have
+    # been changed and no new tests have been added. This would be common when
+    # iterating on private methods or tests method content. We should only
+    # invalidate the cache if the public interface changes, which is likely
+    # when a new test is added or tests are removed. But avoid this
+    # optimization when listing tests as for parameterized tests, turbine jars
+    # do not change when parameters are updated (this does not affect modifying
+    # parameters and then running the test as robolectric loads the correct
+    # values at runtime).
+    if not list_only:
+      for i, p in enumerate(classpath):
+        if 'javac' in p:
+          turbine_p = p.replace('javac', 'turbine')
+          if os.path.exists(turbine_p):
+            classpath[i] = turbine_p
+
+    def do_query_test_json_config():
+      with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
+        result = self._QueryTestJsonConfig(temp_dir,
+                                           allow_debugging=False,
+                                           enable_shadow_allowlist=True)
+        with open(cached_test_list_path, 'w') as f:
+          json.dump(result, f)
+
+    md5_check.CallAndRecordIfStale(do_query_test_json_config,
+                                   record_path=str(record_path),
+                                   input_paths=classpath + [__file__],
+                                   input_strings=input_strings,
+                                   output_paths=[str(cached_test_list_path)])
+
+    with open(cached_test_list_path) as f:
+      return json.load(f)
 
   #override
   def GetTestsForListing(self):
-    json_config = self._GetJsonConfig()
+    json_config = self._GetJsonConfig(list_only=True)
     ret = []
     for config in json_config['configs'].values():
       for class_name, methods in config.items():