clang_wrapper: Handle multiple entries in a single CDB fragment.

infra/base-images/base-builder/indexer/clang_wrapper.py

@@ -68,6 +68,8 @@
 _INDEXER_THREADS_PER_MERGE_QUEUE = 16
 _INDEXER_PER_THREAD_MEMORY = 2 * 1024**3  # 2 GiB
 
+_CDB_FRAGMENT_DELIMITER = ",\n"
+
 SRC = Path(os.getenv("SRC", "/src"))
 # On OSS-Fuzz build infra, $OUT is not /out.
 OUT = Path(os.getenv("OUT", "/out"))
@@ -177,13 +179,13 @@ def files_by_creation_time(folder_path: Path) -> Sequence[Path]:
   return files
 
 
-def _wait_for_cdb_fragment(file: Path) -> str | None:
+def _wait_for_cdb_fragment(file: Path) -> Sequence[str]:
   """Returns the CDB fragment from the given file, waiting if needed."""
   num_retries = 3
   for i in range(1 + num_retries):
     data = file.read_text()
-    if data.endswith(",\n"):
-      return data.rstrip().rstrip(",")
+    if data.endswith(_CDB_FRAGMENT_DELIMITER):
+      return data.split(_CDB_FRAGMENT_DELIMITER)[:-1]
 
     if i < num_retries:
       print(
@@ -202,7 +204,7 @@ def _wait_for_cdb_fragment(file: Path) -> str | None:
       else:
         raise RuntimeError(error)
 
-  return None
+  return ()
 
 
 def read_cdb_fragments(cdb_path: Path) -> Any:
@@ -216,11 +218,10 @@ def read_cdb_fragments(cdb_path: Path) -> Any:
     if not file.name.endswith(".json"):
       continue
 
-    fragment = _wait_for_cdb_fragment(file)
-    if fragment:
-      contents.append(fragment)
+    fragments = _wait_for_cdb_fragment(file)
+    contents.extend(fragments)
 
-  contents = ",\n".join(contents)
+  contents = _CDB_FRAGMENT_DELIMITER.join(contents)
   contents = "[" + contents + "]"
   return json.loads(contents)
 
@@ -514,15 +515,13 @@ def load_cdbs(directory: Path) -> Iterator[tuple[Path, dict[str, Any]]]:
       if file.name.endswith("_linker_commands.json"):
         continue
 
-      fragment_data = _wait_for_cdb_fragment(file)
-      if not fragment_data:
-        continue
-
-      fragment = json.loads(fragment_data)
-      if "output" not in fragment:
-        continue
+      fragments_data = _wait_for_cdb_fragment(file)
+      for fragment_data in fragments_data:
+        fragment = json.loads(fragment_data)
+        if "output" not in fragment:
+          continue
 
-      yield file, fragment
+        yield file, fragment
 
   # We could be running multiple linking steps in parallel, so serialize merges.
   with _file_lock(merged_cdb_path / ".lock"):

infra/base-images/base-builder/indexer/clang_wrapper_test.py

@@ -90,12 +90,26 @@ def test_merge_incremental_cdb(self):
     }
 
     new_cdb_fragments = {
-        "test.c.aaa.json": {
+        "test.c.aaa.json": [{
             "directory": "/build/subdir",
             "file": "test.c",
             "output": "test.o",
             "arguments": ["-c", "test.c"],
-        },
+        }],
+        "bar.c.bbb.json": [
+            {
+                "directory": "/build/subdir",
+                "file": "bar.c",
+                "output": "bar.o",
+                "arguments": ["-c", "bar.c"],
+            },
+            {
+                "directory": "/build/subdir",
+                "file": "bar2.c",
+                "output": "bar2.o",
+                "arguments": ["-c", "bar2.c"],
+            },
+        ],
     }
 
     for cdb_fragment_path, cdb_fragment in old_cdb_fragments.items():
@@ -110,7 +124,7 @@ def test_merge_incremental_cdb(self):
 
     for cdb_fragment_path, cdb_fragment in new_cdb_fragments.items():
       (cdb_path / cdb_fragment_path).write_text(
-          json.dumps(cdb_fragment) + ",\n"
+          ",\n".join([json.dumps(frag) for frag in cdb_fragment]) + ",\n"
       )
 
     (cdb_path / "not_a_json").write_text("not a json")
@@ -125,6 +139,7 @@ def test_merge_incremental_cdb(self):
             pathlib.Path(merged_cdb_path) / "test.c.aaa.json",
             pathlib.Path(merged_cdb_path) / "foo.c.455.json",
             pathlib.Path(merged_cdb_path) / "foo.123_linker_commands.json",
+            pathlib.Path(merged_cdb_path) / "bar.c.bbb.json",
         ],
     )