Merge branch 'master' into tests_PcapPlusPlus

Author: vitaliset

infra/base-images/base-builder/indexer/clang_wrapper.py

@@ -25,6 +25,8 @@
 import json
 import os
 from pathlib import Path  # pylint: disable=g-importing-member
+import shlex
+import shutil
 import subprocess
 import sys
 import time
@@ -49,7 +51,6 @@
     "-ffile-prefix-map=",
 )
 
-PROJECT = Path(os.environ["PROJECT_NAME"])
 SRC = Path(os.getenv("SRC", "/src"))
 # On OSS-Fuzz build infra, $OUT is not /out.
 OUT = Path(os.getenv("OUT", "/out"))
@@ -280,10 +281,6 @@ def read_cdb_fragments(cdb_path: Path) -> Any:
 
 def run_indexer(build_id: str, linker_commands: dict[str, Any]):
   """Run the indexer."""
-  index_dir = INDEXES_PATH / build_id
-  # TODO: check if this is correct.
-  index_dir.mkdir(exist_ok=True)
-
   # Use a build-specific compile commands directory, since there could be
   # parallel linking happening at the same time.
   compile_commands_dir = INDEXES_PATH / f"compile_commands_{build_id}"
@@ -300,6 +297,15 @@ def run_indexer(build_id: str, linker_commands: dict[str, Any]):
     )
     return
 
+  index_dir = INDEXES_PATH / build_id
+  if index_dir.exists():
+    # A previous indexer already ran for the same build ID.  Clear the directory
+    # so we can re-run the indexer, otherwise we might run into various issues
+    # (e.g. the indexer doesn't like it when source files already exist).
+    shutil.rmtree(index_dir)
+
+  index_dir.mkdir()
+
   with (compile_commands_dir / "compile_commands.json").open("wt") as f:
     json.dump(linker_commands["compile_commands"], f, indent=2)
 
@@ -429,8 +435,36 @@ def _write_filter_log(
       f.write(f"\t{cu_path}\n")
 
 
+def expand_rsp_file(argv: Sequence[str]) -> list[str]:
+  # https://llvm.org/docs/CommandLine.html#response-files
+  expanded = []
+  for arg in argv:
+    if arg.startswith("@"):
+      with open(arg[1:], "r") as f:
+        expanded_args = shlex.split(f.read())
+      expanded.extend(expanded_args)
+    else:
+      expanded.append(arg)
+
+  return expanded
+
+
+def force_optimization_flag(argv: Sequence[str]) -> list[str]:
+  """Forces -O0 in the given argument list."""
+  args = []
+  for arg in argv:
+    if arg.startswith("-O") and arg != "-O0":
+      arg = "-O0"
+
+    args.append(arg)
+
+  return args
+
+
 def main(argv: list[str]) -> None:
+  argv = expand_rsp_file(argv)
   argv = remove_flag_if_present(argv, "-gline-tables-only")
+  argv = force_optimization_flag(argv)
 
   if _has_disallowed_clang_flags(argv):
     raise ValueError("Disallowed clang flags found, aborting.")

infra/base-images/base-builder/indexer/clang_wrapper_test.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# Copyright 2025 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""clang_wrapper tests."""
+
+import clang_wrapper
+import unittest
+
+
+class ClangWrapperTest(unittest.TestCase):
+
+  def test_force_optimization_flag_no_optimization(self):
+    """Tests that optimization flags are not forced when not present."""
+    argv = ["clang", "-c", "test.c", "-o", "test.o"]
+    modified_argv = clang_wrapper.force_optimization_flag(argv)
+    self.assertCountEqual(modified_argv, argv)
+
+  def test_force_optimization_flag(self):
+    """Tests that optimization flags are forced when present."""
+    argv = ["clang", "-O2", "-c", "test.c", "-o", "test.o", "-O1"]
+    modified_argv = clang_wrapper.force_optimization_flag(argv)
+    self.assertCountEqual(
+        modified_argv, ["clang", "-O0", "-c", "test.c", "-o", "test.o", "-O0"]
+    )
+
+
+if __name__ == "__main__":
+  unittest.main()