Merge branch 'master' into feat/parallel-base-image-builds

Author: hunsche

infra/base-images/base-builder/indexer/clang_wrapper.py

@@ -68,6 +68,8 @@
 _INDEXER_THREADS_PER_MERGE_QUEUE = 16
 _INDEXER_PER_THREAD_MEMORY = 2 * 1024**3  # 2 GiB
 
+_CDB_FRAGMENT_DELIMITER = ",\n"
+
 SRC = Path(os.getenv("SRC", "/src"))
 # On OSS-Fuzz build infra, $OUT is not /out.
 OUT = Path(os.getenv("OUT", "/out"))
@@ -177,13 +179,13 @@ def files_by_creation_time(folder_path: Path) -> Sequence[Path]:
   return files
 
 
-def _wait_for_cdb_fragment(file: Path) -> str | None:
+def _wait_for_cdb_fragment(file: Path) -> Sequence[str]:
   """Returns the CDB fragment from the given file, waiting if needed."""
   num_retries = 3
   for i in range(1 + num_retries):
     data = file.read_text()
-    if data.endswith(",\n"):
-      return data.rstrip().rstrip(",")
+    if data.endswith(_CDB_FRAGMENT_DELIMITER):
+      return data.split(_CDB_FRAGMENT_DELIMITER)[:-1]
 
     if i < num_retries:
       print(
@@ -202,7 +204,7 @@ def _wait_for_cdb_fragment(file: Path) -> str | None:
       else:
         raise RuntimeError(error)
 
-  return None
+  return ()
 
 
 def read_cdb_fragments(cdb_path: Path) -> Any:
@@ -216,11 +218,10 @@ def read_cdb_fragments(cdb_path: Path) -> Any:
     if not file.name.endswith(".json"):
       continue
 
-    fragment = _wait_for_cdb_fragment(file)
-    if fragment:
-      contents.append(fragment)
+    fragments = _wait_for_cdb_fragment(file)
+    contents.extend(fragments)
 
-  contents = ",\n".join(contents)
+  contents = _CDB_FRAGMENT_DELIMITER.join(contents)
   contents = "[" + contents + "]"
   return json.loads(contents)
 
@@ -514,15 +515,13 @@ def load_cdbs(directory: Path) -> Iterator[tuple[Path, dict[str, Any]]]:
       if file.name.endswith("_linker_commands.json"):
         continue
 
-      fragment_data = _wait_for_cdb_fragment(file)
-      if not fragment_data:
-        continue
-
-      fragment = json.loads(fragment_data)
-      if "output" not in fragment:
-        continue
+      fragments_data = _wait_for_cdb_fragment(file)
+      for fragment_data in fragments_data:
+        fragment = json.loads(fragment_data)
+        if "output" not in fragment:
+          continue
 
-      yield file, fragment
+        yield file, fragment
 
   # We could be running multiple linking steps in parallel, so serialize merges.
   with _file_lock(merged_cdb_path / ".lock"):

infra/base-images/base-builder/indexer/clang_wrapper_test.py

@@ -90,12 +90,26 @@ def test_merge_incremental_cdb(self):
     }
 
     new_cdb_fragments = {
-        "test.c.aaa.json": {
+        "test.c.aaa.json": [{
             "directory": "/build/subdir",
             "file": "test.c",
             "output": "test.o",
             "arguments": ["-c", "test.c"],
-        },
+        }],
+        "bar.c.bbb.json": [
+            {
+                "directory": "/build/subdir",
+                "file": "bar.c",
+                "output": "bar.o",
+                "arguments": ["-c", "bar.c"],
+            },
+            {
+                "directory": "/build/subdir",
+                "file": "bar2.c",
+                "output": "bar2.o",
+                "arguments": ["-c", "bar2.c"],
+            },
+        ],
     }
 
     for cdb_fragment_path, cdb_fragment in old_cdb_fragments.items():
@@ -110,7 +124,7 @@ def test_merge_incremental_cdb(self):
 
     for cdb_fragment_path, cdb_fragment in new_cdb_fragments.items():
       (cdb_path / cdb_fragment_path).write_text(
-          json.dumps(cdb_fragment) + ",\n"
+          ",\n".join([json.dumps(frag) for frag in cdb_fragment]) + ",\n"
       )
 
     (cdb_path / "not_a_json").write_text("not a json")
@@ -125,6 +139,7 @@ def test_merge_incremental_cdb(self):
             pathlib.Path(merged_cdb_path) / "test.c.aaa.json",
             pathlib.Path(merged_cdb_path) / "foo.c.455.json",
             pathlib.Path(merged_cdb_path) / "foo.123_linker_commands.json",
+            pathlib.Path(merged_cdb_path) / "bar.c.bbb.json",
         ],
     )
 

infra/base-images/base-runner/Dockerfile

@@ -112,7 +112,7 @@ RUN /install_javascript.sh && rm /install_javascript.sh
 # Copy built ruby and ruzzy from builder
 COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
 COPY --from=base-ruby /install/ruzzy /install/ruzzy
-COPY ruzzy /usr/bin/ruzzy
+COPY ruzzy /usr/local/bin/ruzzy
 ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
 # RubyGems installation directory
 ENV GEM_HOME="$OUT/fuzz-gem"

projects/abseil-py/build.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 ################################################################################
-python3 setup.py install
+python3 -m pip install .
 cd ../
 mkdir fuzzbuilds
 cd fuzzbuilds

projects/cmake/project.yaml

@@ -11,7 +11,6 @@ sanitizers:
 #  - memory
 
 fuzzing_engines:
-  - afl
   - honggfuzz
   - libfuzzer
 

projects/glslang/build.sh

@@ -30,7 +30,7 @@ $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $SRC/compile_fuzzer.cc \
 	-I/src/glslang/glslang/Include/ -I/src/glslang/glslang/.. \
 	-I/src/glslang/build/include -I/src/glslang/SPIRV/.. \
 	./glslang/libglslang.a ./SPIRV/libSPIRV.a \
-	./glslang/libglslang-default-resource-limits.a ./SPIRV/libSPVRemapper.a \
+	./glslang/libglslang-default-resource-limits.a \
 	-lpthread ./glslang/libMachineIndependent.a \
 	./glslang/OSDependent/Unix/libOSDependent.a \
 	./glslang/libGenericCodeGen.a \

projects/jbig2dec/Dockerfile

@@ -19,8 +19,8 @@ RUN apt-get update && apt-get install -y make libtool pkg-config vim libreadline
 RUN git clone --recursive --depth 1 git://git.ghostscript.com/jbig2dec.git jbig2dec
 RUN mkdir tests
 RUN cp $SRC/jbig2dec/annex-h.jbig2 tests/annex-h.jb2
-RUN cd tests && wget -O t89-halftone.zip http://cgit.ghostscript.com/cgi-bin/cgit.cgi/tests.git/plain/jbig2/t89-halftone.zip && unzip t89-halftone.zip
-RUN cd tests && wget -O jb2streams.zip http://cgit.ghostscript.com/cgi-bin/cgit.cgi/tests.git/plain/jbig2/jb2streams.zip && unzip jb2streams.zip
+RUN cd tests && (wget -O t89-halftone.zip http://cgit.ghostscript.com/cgi-bin/cgit.cgi/tests.git/plain/jbig2/t89-halftone.zip && unzip t89-halftone.zip) || true
+RUN cd tests && (wget -O jb2streams.zip http://cgit.ghostscript.com/cgi-bin/cgit.cgi/tests.git/plain/jbig2/jb2streams.zip && unzip jb2streams.zip) || true
 RUN cd tests && zip -q $SRC/jbig2_fuzzer_seed_corpus.zip *.jb2
 RUN rm -rf tests
 COPY *.dict $SRC/

projects/jq/build.sh

@@ -53,7 +53,7 @@ $CXX $CXXFLAGS $LIB_FUZZING_ENGINE ./jq_fuzz_parse_stream.o \
     -o $OUT/jq_fuzz_parse_stream -I./src
 
 $CXX $CXXFLAGS $LIB_FUZZING_ENGINE ./tests/jq_fuzz_execute.cpp \
-    -I./src \
+    -I./src -I./vendor/oniguruma/src \
     ./.libs/libjq.a ./vendor/oniguruma/src/.libs/libonig.a\
     -o $OUT/jq_fuzz_execute -I./src
 

projects/libical/build.sh

@@ -15,7 +15,7 @@
 #
 ################################################################################
 
-cmake . -DSTATIC_ONLY=ON -DLIBICAL_GLIB=False -DLIBICAL_GLIB_BUILD_DOCS=False -DLIBICAL_GOBJECT_INTROSPECTION=False
+cmake . -DSTATIC_ONLY=ON -DLIBICAL_GLIB=False -DLIBICAL_GLIB_BUILD_DOCS=False -DLIBICAL_GOBJECT_INTROSPECTION=False -DLIBICAL_JAVA_BINDINGS=False
 make install -j$(nproc)
 
 

projects/liblouis/build.sh

@@ -15,4 +15,8 @@
 #
 ################################################################################
 
+sed -i 's/main/main2/g' tests/fuzzing/fuzz_backtranslate.c
+sed -i 's/main/main2/g' tests/fuzzing/fuzz_translate_generic.c
+sed -i 's/LLVMFuzzerRunDriver(&argc/\/\/LLVMFuzzerRunDriver(&argc/g' tests/fuzzing/fuzz_backtranslate.c
+sed -i 's/LLVMFuzzerRunDriver(&argc/\/\/LLVMFuzzerRunDriver(&argc/g' tests/fuzzing/fuzz_translate_generic.c
 $SRC/liblouis/tests/fuzzing/build.sh