Update indexer. (#13429)

Author: oliverchang

infra/base-images/base-builder/indexer/clang_wrapper.py

@@ -259,7 +259,7 @@ def read_cdb_fragments(cdb_path: Path) -> Any:
         )
         time.sleep(2**(i + 1))
     else:
-      error = "CDB fragment {file} is invalid even after retries: {data}"
+      error = f"CDB fragment {file} is invalid even after retries: {data}"
       if "test.c" in file.name or "conftest.c" in file.name:
         # Some build systems seem to have a weird issue where the autotools
         # generated `test.c` or `conftest.c` for testing compilers doesn't

infra/base-images/base-builder/indexer/index_build.py

@@ -450,8 +450,11 @@ def archive_target(target: BinaryMetadata) -> Path | None:
     manifest_types.Manifest(
         name=name,
         uuid=uuid,
-        binary_name=target.name,
-        binary_args=target.binary_args,
+        binary_config=manifest_types.CommandLineBinaryConfig(
+            kind=manifest_types.BinaryConfigKind.OSS_FUZZ,
+            binary_name=target.name,
+            binary_args=target.binary_args,
+        ),
         source_map=manifest_types.source_map_from_dict(json.loads(source_map)),
         lib_mount_path=lib_mount_path,
     ).save_build(
@@ -476,6 +479,9 @@ def test_and_archive(target_args: list[str],
   targets = enumerate_build_targets(target_args)
   if targets_to_index:
     targets = [t for t in targets if t.name in targets_to_index]
+    missing_targets = set(targets_to_index) - set(t.name for t in targets)
+    if missing_targets:
+      raise ValueError(f'Could not find specified targets {missing_targets}.')
 
   logging.info('targets %s', targets)
   for target in targets:
@@ -521,13 +527,24 @@ def main():
   )
   parser.add_argument(
       '--target-args',
-      default=manifest_types.INPUT_FILE,
+      default=None,
       help=('Arguments to pass to the target when executing it. '
             'This string is shell-escaped (interpreted with `shlex.split`). '
+            'The substring <input_file> will be replaced with the input path.'
+            'Note: This is deprecated, use --target-arg instead.'),
+  )
+  parser.add_argument(
+      '--target-arg',
+      action='append',
+      help=('An argument to pass to the target binary. '
             'The substring <input_file> will be replaced with the input path.'),
   )
   args = parser.parse_args()
 
+  if args.target_args and args.target_arg:
+    raise ValueError(
+        'Only one of --target-args or --target-arg can be specified.')
+
   targets_to_index = None
   if args.targets:
     targets_to_index = args.targets.split(',')
@@ -542,7 +559,17 @@ def main():
   # We don't have an existing /out dir on oss-fuzz's build infra.
   OUT.mkdir(parents=True, exist_ok=True)
   build_project(targets_to_index)
-  test_and_archive(shlex.split(args.target_args), targets_to_index)
+
+  if args.target_arg:
+    target_args = args.target_arg
+  elif args.target_args:
+    logging.warning('--target-args is deprecated, use --target-arg instead.')
+    target_args = shlex.split(args.target_args)
+  else:
+    logging.info('No target args specified.')
+    target_args = []
+
+  test_and_archive(target_args, targets_to_index)
 
   for snapshot in SNAPSHOT_DIR.iterdir():
     shutil.move(str(snapshot), OUT)

infra/base-images/base-builder/indexer/index_build_test.py

@@ -78,8 +78,9 @@ def _check_archive(self, archive_path: Path):
           if file:  # Make type checkers happy.
             manifest = json.load(file)
             self.assertTrue(manifest['lib_mount_path'])
-            self.assertIsNotNone(tar.getmember('obj/' +
-                                               manifest['binary_name']))
+            self.assertIsNotNone(
+                tar.getmember('obj/' +
+                              manifest['binary_config']['binary_name']))
 
     self.assertTrue(has_obj_lib, "obj/lib/ was not found in the archive.")
     self.assertTrue(has_idx_sqlite,

infra/base-images/base-builder/indexer/manifest_types.py

@@ -32,7 +32,8 @@
 import shutil
 import tarfile
 import tempfile
-from typing import Any, Callable, Self
+from typing import Any, Callable, Mapping, Self, Sequence
+import urllib.request
 
 import pathlib
 
@@ -51,7 +52,12 @@
 # Min archive version we currently support.
 _MIN_SUPPORTED_ARCHIVE_VERSION = 1
 # The current version of the build archive format.
-ARCHIVE_VERSION = 3
+ARCHIVE_VERSION = 4
+# OSS-Fuzz $OUT dir.
+OUT = pathlib.Path(os.getenv("OUT", "/out"))
+# OSS-Fuzz coverage info.
+_COVERAGE_INFO_URL = ("https://storage.googleapis.com/oss-fuzz-coverage/"
+                      f"latest_report_info/{os.getenv('PROJECT_NAME')}.json")
 
 # Will be replaced with the input file for target execution.
 INPUT_FILE = "<input_file>"
@@ -128,14 +134,85 @@ def from_dict(self, data: dict[str, Any]) -> Self:
     )
 
 
+class BinaryConfigKind(enum.StrEnum):
+  """The kind of binary configurations."""
+
+  OSS_FUZZ = enum.auto()
+  BINARY = enum.auto()
+
+  def validate_in(self, options: list[Self]):
+    if self not in options:
+      raise ValueError(
+          f"Expected one of the following binary config kinds: {options}, "
+          f"but got {self}")
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class BinaryConfig:
+  """Base binary configuration.
+
+  Attributes:
+    kind: The kind of binary configuration.
+    binary_args: The arguments to pass to the binary, for example
+      "<input_file>".
+  """
+
+  kind: BinaryConfigKind
+
+  @classmethod
+  def from_dict(cls, config_dict: Mapping[Any, Any]) -> Self:
+    """Deserializes the correct `BinaryConfig` subclass from a dict."""
+    mapping = {
+        BinaryConfigKind.OSS_FUZZ: CommandLineBinaryConfig,
+        BinaryConfigKind.BINARY: CommandLineBinaryConfig,
+    }
+    kind = config_dict["kind"]
+    if kind not in mapping:
+      raise ValueError(f"Unknown BinaryConfigKind: {kind}")
+    val = config_dict
+    if isinstance(val.get("binary_args"), str):
+      logging.warning(
+          "BinaryConfig: binary_args is type string instead of list."
+          " This is deprecated. Converting to list. Args: %s",
+          val["binary_args"],
+      )
+      val = dict(val, binary_args=shlex.split(val["binary_args"]))
+    return mapping[kind].from_dict(val)
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class CommandLineBinaryConfig(BinaryConfig):
+  """Configuration for a command-line userspace binary."""
+
+  binary_name: str
+  binary_args: list[str]
+
+  @classmethod
+  def from_dict(cls, config_dict: Mapping[Any, Any]) -> Self:
+    """Deserializes the `CommandLineBinaryConfig` from a dict."""
+    kind = BinaryConfigKind(config_dict["kind"])
+    kind.validate_in([BinaryConfigKind.OSS_FUZZ, BinaryConfigKind.BINARY])
+    return CommandLineBinaryConfig(
+        kind=kind,
+        binary_name=config_dict["binary_name"],
+        binary_args=config_dict["binary_args"],
+    )
+
+
 @dataclasses.dataclass(frozen=True)
 class Manifest:
   """Contains general meta-information about the snapshot."""
 
+  # The name of the target.
   name: str
+  # A unique identifier for the snapshot (not necessarily a valid UUID).
   uuid: str
-  binary_name: str
-  binary_args: list[str] | None = None
+  # A fixed path that shared libraries stored at `./obj/lib` should be mounted
+  # at before running the target.
+  lib_mount_path: pathlib.Path | None
+
+  # The binary configuration used to build the snapshot.
+  binary_config: BinaryConfig
 
   # The path prefix of the actual build directory (e.g., a temporary file in
   # the build host). It's used during replay to remove noisy source-file
@@ -154,7 +231,8 @@ class Manifest:
   #   }
   # }
   source_map: dict[pathlib.Path, SourceRef] | None = None
-  lib_mount_path: pathlib.Path | None = None
+
+  # Version of the manifest spec.
   version: int = ARCHIVE_VERSION
 
   @classmethod
@@ -172,22 +250,42 @@ def from_dict(cls, data: dict[str, Any]) -> Self:
       binary_args = _get_mapped(data, "binary_args", shlex.split)
     else:
       binary_args = data.get("binary_args")
+    if data["version"] < 4:
+      binary_config = CommandLineBinaryConfig(
+          kind=BinaryConfigKind.BINARY,
+          binary_name=data["binary_name"],
+          binary_args=binary_args or [],
+      )
+    else:
+      binary_config = _get_mapped(data, "binary_config", BinaryConfig.from_dict)
+
+    version = data["version"]
+    if _MIN_SUPPORTED_ARCHIVE_VERSION <= version <= ARCHIVE_VERSION:
+      # Upgrade archive version - we have upgraded all necessary fields.
+      version = ARCHIVE_VERSION
+    else:
+      logging.warning(
+          "Unsupported manifest version %s detected. Not upgrading.", version)
     return Manifest(
+        version=version,
         name=data["name"],
         uuid=data["uuid"],
-        binary_name=data["binary_name"],
-        binary_args=binary_args,
         lib_mount_path=lib_mount_path,
         source_map=_get_mapped(data, "source_map", source_map_from_dict),
         source_dir_prefix=data.get("source_dir_prefix"),
         reproducibility=_get_mapped(data, "reproducibility",
                                     Reproducibility.from_dict),
-        version=data["version"],
+        binary_config=binary_config,
     )
 
   def to_dict(self) -> dict[str, Any]:
     """Converts a Manifest object to a serializable dict."""
     data = dataclasses.asdict(self)
+
+    patches = data["binary_config"].get("patches")
+    if patches:
+      patches[:] = [path.as_posix() for path in patches]
+
     data["lib_mount_path"] = _get_mapped(data, "lib_mount_path",
                                          lambda x: x.as_posix())
     data["source_map"] = _get_mapped(data, "source_map", source_map_to_dict)
@@ -211,7 +309,7 @@ def validate(self) -> None:
       raise RuntimeError(
           "Build archive with version 1 has an alternative lib_mount_path set"
           f" ({self.lib_mount_path}). This is not a valid archive.")
-    if not self.name or not self.uuid or not self.binary_name:
+    if not self.name or not self.uuid or not self.binary_config:
       raise RuntimeError(
           "Attempting to load a manifest with missing fields. Expected all"
           " fields to be set, but got {self}")
@@ -229,10 +327,12 @@ def validate(self) -> None:
         raise RuntimeError(f"Type mismatch for field {k}: expected {v}, got"
                            f" {type(getattr(self, k))}")
     # We updated from string to list in version 3, make sure this propagated.
-    if self.binary_args is not None and not isinstance(self.binary_args, list):
-      raise RuntimeError(
-          "Type mismatch for field binary_args: expected list, got"
-          f" {type(self.binary_args)}")
+    binary_config = self.binary_config
+    if hasattr(binary_config, "binary_args"):
+      if not isinstance(binary_config.binary_args, list):
+        raise RuntimeError(
+            "Type mismatch for field binary_config.binary_args: expected list,"
+            f"got {type(binary_config.binary_args)}")
 
   def save_build(
       self,
@@ -246,6 +346,12 @@ def save_build(
   ) -> None:
     """Saves a build archive with this Manifest."""
     self.validate()
+
+    if not hasattr(self.binary_config, "binary_name"):
+      raise RuntimeError(
+          "Attempting to save a binary config type without binary_name."
+          " This is not yet supported. Kind: {self.binary_config.kind}.")
+
     with tempfile.NamedTemporaryFile() as tmp:
       mode = "w:gz" if archive_path.suffix.endswith("gz") else "w"
       with tarfile.open(tmp.name, mode) as tar:
@@ -294,15 +400,72 @@ def _save_dir(
         _save_dir(source_dir, SRC_DIR, exclude_build_artifacts=True)
         # Only include the relevant target for the snapshot, to save on disk
         # space.
-        _save_dir(build_dir, OBJ_DIR, only_include_target=self.binary_name)
+        _save_dir(
+            build_dir,
+            OBJ_DIR,
+            only_include_target=self.binary_config.binary_name,
+        )
         _save_dir(index_dir, INDEX_DIR)
+        if self.binary_config.kind == BinaryConfigKind.OSS_FUZZ:
+          copied_files = [tar_info.name for tar_info in tar.getmembers()]
+          try:
+            report_missing_source_files(self.binary_config.binary_name,
+                                        copied_files, tar)
+          except Exception:  # pylint: disable=broad-except
+            logging.exception("Failed to report missing source files.")
 
       if os.path.exists(archive_path) and not overwrite:
         logging.warning("Skipping existing archive %s", archive_path)
       else:
         shutil.copyfile(tmp.name, archive_path)
 
 
+def report_missing_source_files(binary_name: str, copied_files: list[str],
+                                tar: tarfile.TarFile):
+  """Saves a report of missing source files to the snapshot tarball."""
+  copied_files = {_get_comparable_path(file) for file in copied_files}
+  covered_files = {
+      _get_comparable_path(path): path
+      for path in get_covered_files(binary_name)
+  }
+  missing = set(covered_files) - copied_files
+  if not missing:
+    return
+  logging.info("Reporting missing files: %s", missing)
+  missing_report_lines = sorted([covered_files[k] for k in missing])
+  report_name = f"{binary_name}_missing_files.txt"
+  tar_info = tarfile.TarInfo(name=report_name)
+  missing_report = " ".join(missing_report_lines)
+  missing_report_bytes = missing_report.encode("utf-8")
+  tar.addfile(tarinfo=tar_info, fileobj=io.BytesIO(missing_report_bytes))
+  with open(os.path.join(OUT, report_name), "w") as fp:
+    fp.write(missing_report)
+
+
+def _get_comparable_path(path: str) -> tuple[str, str]:
+  return os.path.basename(os.path.dirname(path)), os.path.basename(path)
+
+
+def get_covered_files(target: str) -> Sequence[str]:
+  """Returns the files covered by fuzzing on OSS-Fuzz by the target."""
+  with urllib.request.urlopen(_COVERAGE_INFO_URL) as resp:
+    latest_info = json.load(resp)
+
+  stats_url = latest_info.get("fuzzer_stats_dir").replace(
+      "gs://", "https://storage.googleapis.com/")
+
+  target_url = f"{stats_url}/{target}.json"
+  with urllib.request.urlopen(target_url) as resp:
+    target_cov = json.load(resp)
+
+  files = target_cov["data"][0]["files"]
+  return [
+      file["filename"]
+      for file in files
+      if file["summary"]["regions"]["covered"]
+  ]
+
+
 def _get_mapped(data: dict[str, Any], key: str,
                 mapper: Callable[[Any], Any]) -> Any | None:
   """Get a value from a dict and apply a mapper to it, if it's not None."""