Merge pull request #93 from rchildre3/feature/update-libfuzzer-15-deps

Update libFuzzer to compiler-rt 15 and Rust deps

Author: fitzgen

CHANGELOG.md

@@ -8,7 +8,14 @@ Released YYYY-MM-DD.
 
 ### Changed
 
-* TODO (or remove section if none)
+* Updated to `libFuzzer` commit `df90d22` (`release/15.x`).
+* LLVM 16's [upcoming change][llvm_cxx17] to build requirements to C++17
+  necessitate reflecting those changes. (`libFuzzer` updates contain C++14 code)
+* Drastically reduce build times by using parallel C++ compilation jobs
+* Updated `rand` dependency from 0.8.3 to 0.8.5
+* Updated `flate2` dependency from 1.0.20 to 1.0.24
+
+[llvm_cxx17]: https://llvm.org/docs/ReleaseNotes.html#update-on-required-toolchains-to-build-llvm
 
 ### Deprecated
 

Cargo.toml

@@ -13,7 +13,7 @@ arbitrary = "1"
 once_cell = "1"
 
 [build-dependencies]
-cc = "1.0"
+cc = { version = "1.0", features = ["parallel"] }
 
 [features]
 arbitrary-derive = ["arbitrary/derive"]
@@ -26,5 +26,5 @@ members = [
 ]
 
 [dev-dependencies]
-flate2 = "1.0.20"
-rand = "0.8.3"
+flate2 = "1.0.24"
+rand = "0.8.5"

build.rs

@@ -26,7 +26,7 @@ fn main() {
             println!("cargo:rerun-if-changed={}", source.display());
             build.file(source.to_str().unwrap());
         }
-        build.flag("-std=c++11");
+        build.flag("-std=c++17");
         build.flag("-fno-omit-frame-pointer");
         build.flag("-w");
         build.cpp(true);

example_mutator/fuzz/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 cargo-fuzz = true
 
 [dependencies]
-flate2 = "1.0.20"
+flate2 = "1.0.24"
 libfuzzer-sys = { path = "../.." }
 
 [[bin]]

libfuzzer/CMakeLists.txt

@@ -6,6 +6,8 @@ set(LIBFUZZER_SOURCES
   FuzzerExtFunctionsWeak.cpp
   FuzzerExtFunctionsWindows.cpp
   FuzzerExtraCounters.cpp
+  FuzzerExtraCountersDarwin.cpp
+  FuzzerExtraCountersWindows.cpp
   FuzzerFork.cpp
   FuzzerIO.cpp
   FuzzerIOPosix.cpp
@@ -64,18 +66,19 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND
   append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS)
 elseif(TARGET cxx-headers OR HAVE_LIBCXX)
   # libFuzzer uses C++ standard library headers.
+  list(APPEND LIBFUZZER_CFLAGS ${COMPILER_RT_CXX_CFLAGS})
   set(LIBFUZZER_DEPS cxx-headers)
 endif()
 
 append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
 
 if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
-  list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters)
+  list(APPEND LIBFUZZER_CFLAGS -fsanitize-coverage=0)
 endif()
 
 if(MSVC)
   # Silence warnings by turning off exceptions in MSVC headers and avoid an
-  # error by unecessarily defining thread_local when it isn't even used on
+  # error by unnecessarily defining thread_local when it isn't even used on
   # Windows.
   list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0)
 else()
@@ -136,15 +139,15 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
   macro(partially_link_libcxx name dir arch)
-    if(${arch} MATCHES "i386")
-      set(EMULATION_ARGUMENT "-m" "elf_i386")
-    else()
-      set(EMULATION_ARGUMENT "")
+    get_target_flags_for_arch(${arch} target_cflags)
+    if(CMAKE_CXX_COMPILER_ID MATCHES Clang)
+      get_compiler_rt_target(${arch} target)
+      set(target_cflags --target=${target} ${target_cflags})
     endif()
     set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
     file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
     add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
-      COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
+      COMMAND ${CMAKE_CXX_COMPILER} ${target_cflags} -Wl,--whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" -Wl,--no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
       COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
       COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
       COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
@@ -160,7 +163,8 @@ if(OS_NAME MATCHES "Linux|Fuchsia" AND
       CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON
                  -DCMAKE_POSITION_INDEPENDENT_CODE=ON
                  -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF
-                 -DLIBCXX_ABI_NAMESPACE=__Fuzzer)
+                 -DLIBCXX_ABI_NAMESPACE=__Fuzzer
+                 -DLIBCXX_ENABLE_EXCEPTIONS=OFF)
     target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
     add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
     target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)

libfuzzer/FuzzerBuiltinsMsvc.h

@@ -41,7 +41,8 @@ inline uint32_t Clzll(uint64_t X) {
 #if !defined(_M_ARM) && !defined(_M_X64)
   // Scan the high 32 bits.
   if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32)))
-    return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
+    return static_cast<int>(
+        63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
   // Scan the low 32 bits.
   if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X)))
     return static_cast<int>(63 - LeadZeroIdx);

libfuzzer/FuzzerCommand.h

@@ -33,7 +33,7 @@ class Command final {
 
   Command() : CombinedOutAndErr(false) {}
 
-  explicit Command(const Vector<std::string> &ArgsToAdd)
+  explicit Command(const std::vector<std::string> &ArgsToAdd)
       : Args(ArgsToAdd), CombinedOutAndErr(false) {}
 
   explicit Command(const Command &Other)
@@ -58,7 +58,7 @@ class Command final {
 
   // Gets all of the current command line arguments, **including** those after
   // "-ignore-remaining-args=1".
-  const Vector<std::string> &getArguments() const { return Args; }
+  const std::vector<std::string> &getArguments() const { return Args; }
 
   // Adds the given argument before "-ignore_remaining_args=1", or at the end
   // if that flag isn't present.
@@ -68,7 +68,7 @@ class Command final {
 
   // Adds all given arguments before "-ignore_remaining_args=1", or at the end
   // if that flag isn't present.
-  void addArguments(const Vector<std::string> &ArgsToAdd) {
+  void addArguments(const std::vector<std::string> &ArgsToAdd) {
     Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end());
   }
 
@@ -155,16 +155,16 @@ class Command final {
   Command(Command &&Other) = delete;
   Command &operator=(Command &&Other) = delete;
 
-  Vector<std::string>::iterator endMutableArgs() {
+  std::vector<std::string>::iterator endMutableArgs() {
     return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
   }
 
-  Vector<std::string>::const_iterator endMutableArgs() const {
+  std::vector<std::string>::const_iterator endMutableArgs() const {
     return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
   }
 
   // The command arguments.  Args[0] is the command name.
-  Vector<std::string> Args;
+  std::vector<std::string> Args;
 
   // True indicates stderr is redirected to stdout.
   bool CombinedOutAndErr;

libfuzzer/FuzzerCorpus.h

@@ -39,13 +39,13 @@ struct InputInfo {
   bool MayDeleteFile = false;
   bool Reduced = false;
   bool HasFocusFunction = false;
-  Vector<uint32_t> UniqFeatureSet;
-  Vector<uint8_t> DataFlowTraceForFocusFunction;
+  std::vector<uint32_t> UniqFeatureSet;
+  std::vector<uint8_t> DataFlowTraceForFocusFunction;
   // Power schedule.
   bool NeedsEnergyUpdate = false;
   double Energy = 0.0;
   double SumIncidence = 0.0;
-  Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
+  std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
 
   // Delete feature Idx and its frequency from FeatureFreqs.
   bool DeleteFeatureFreq(uint32_t Idx) {
@@ -209,7 +209,7 @@ class InputCorpus {
   InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
                          bool HasFocusFunction, bool NeverReduce,
                          std::chrono::microseconds TimeOfUnit,
-                         const Vector<uint32_t> &FeatureSet,
+                         const std::vector<uint32_t> &FeatureSet,
                          const DataFlowTrace &DFT, const InputInfo *BaseII) {
     assert(!U.empty());
     if (FeatureDebug)
@@ -258,7 +258,7 @@ class InputCorpus {
   }
 
   // Debug-only
-  void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) {
+  void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) {
     if (!FeatureDebug) return;
     Printf("{");
     for (uint32_t Feature: FeatureSet)
@@ -284,14 +284,16 @@ class InputCorpus {
     }
   }
 
-  void Replace(InputInfo *II, const Unit &U) {
+  void Replace(InputInfo *II, const Unit &U,
+               std::chrono::microseconds TimeOfUnit) {
     assert(II->U.size() > U.size());
     Hashes.erase(Sha1ToString(II->Sha1));
     DeleteFile(*II);
     ComputeSHA1(U.data(), U.size(), II->Sha1);
     Hashes.insert(Sha1ToString(II->Sha1));
     II->U = U;
     II->Reduced = true;
+    II->TimeOfUnit = TimeOfUnit;
     DistributionNeedsUpdate = true;
   }
 
@@ -325,7 +327,8 @@ class InputCorpus {
       const auto &II = *Inputs[i];
       Printf("  [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i,
              Sha1ToString(II.Sha1).c_str(), II.U.size(),
-             II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction);
+             II.NumExecutedMutations, II.NumSuccessfullMutations,
+             II.HasFocusFunction);
     }
   }
 
@@ -563,11 +566,11 @@ class InputCorpus {
   }
   std::piecewise_constant_distribution<double> CorpusDistribution;
 
-  Vector<double> Intervals;
-  Vector<double> Weights;
+  std::vector<double> Intervals;
+  std::vector<double> Weights;
 
   std::unordered_set<std::string> Hashes;
-  Vector<InputInfo*> Inputs;
+  std::vector<InputInfo *> Inputs;
 
   size_t NumAddedFeatures = 0;
   size_t NumUpdatedFeatures = 0;
@@ -577,7 +580,7 @@ class InputCorpus {
   bool DistributionNeedsUpdate = true;
   uint16_t FreqOfMostAbundantRareFeature = 0;
   uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {};
-  Vector<uint32_t> RareFeatures;
+  std::vector<uint32_t> RareFeatures;
 
   std::string OutputCorpus;
 };

libfuzzer/FuzzerDataFlowTrace.cpp

@@ -37,7 +37,7 @@ bool BlockCoverage::AppendCoverage(const std::string &S) {
 // Coverage lines have this form:
 // CN X Y Z T
 // where N is the number of the function, T is the total number of instrumented
-// BBs, and X,Y,Z, if present, are the indecies of covered BB.
+// BBs, and X,Y,Z, if present, are the indices of covered BB.
 // BB #0, which is the entry block, is not explicitly listed.
 bool BlockCoverage::AppendCoverage(std::istream &IN) {
   std::string L;
@@ -52,7 +52,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
       continue;
     }
     if (L[0] != 'C') continue;
-    Vector<uint32_t> CoveredBlocks;
+    std::vector<uint32_t> CoveredBlocks;
     while (true) {
       uint32_t BB = 0;
       SS >> BB;
@@ -68,7 +68,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
     auto It = Functions.find(FunctionId);
     auto &Counters =
         It == Functions.end()
-            ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)})
+            ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)})
                   .first->second
             : It->second;
 
@@ -86,8 +86,8 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
 //   * any uncovered function gets weight 0.
 //   * a function with lots of uncovered blocks gets bigger weight.
 //   * a function with a less frequently executed code gets bigger weight.
-Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
-  Vector<double> Res(NumFunctions);
+std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
+  std::vector<double> Res(NumFunctions);
   for (auto It : Functions) {
     auto FunctionID = It.first;
     auto Counters = It.second;
@@ -104,7 +104,7 @@ Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
 }
 
 void DataFlowTrace::ReadCoverage(const std::string &DirPath) {
-  Vector<SizedFile> Files;
+  std::vector<SizedFile> Files;
   GetSizedFilesFromDir(DirPath, &Files);
   for (auto &SF : Files) {
     auto Name = Basename(SF.File);
@@ -115,16 +115,16 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) {
   }
 }
 
-static void DFTStringAppendToVector(Vector<uint8_t> *DFT,
+static void DFTStringAppendToVector(std::vector<uint8_t> *DFT,
                                     const std::string &DFTString) {
   assert(DFT->size() == DFTString.size());
   for (size_t I = 0, Len = DFT->size(); I < Len; I++)
     (*DFT)[I] = DFTString[I] == '1';
 }
 
-// converts a string of '0' and '1' into a Vector<uint8_t>
-static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
-  Vector<uint8_t> DFT(DFTString.size());
+// converts a string of '0' and '1' into a std::vector<uint8_t>
+static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
+  std::vector<uint8_t> DFT(DFTString.size());
   DFTStringAppendToVector(&DFT, DFTString);
   return DFT;
 }
@@ -159,14 +159,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum,
 }
 
 bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
-                         Vector<SizedFile> &CorporaFiles, Random &Rand) {
+                         std::vector<SizedFile> &CorporaFiles, Random &Rand) {
   if (DirPath.empty()) return false;
   Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str());
-  Vector<SizedFile> Files;
+  std::vector<SizedFile> Files;
   GetSizedFilesFromDir(DirPath, &Files);
   std::string L;
   size_t FocusFuncIdx = SIZE_MAX;
-  Vector<std::string> FunctionNames;
+  std::vector<std::string> FunctionNames;
 
   // Collect the hashes of the corpus files.
   for (auto &SF : CorporaFiles)
@@ -191,7 +191,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
     // * chooses a random function according to the weights.
     ReadCoverage(DirPath);
     auto Weights = Coverage.FunctionWeights(NumFunctions);
-    Vector<double> Intervals(NumFunctions + 1);
+    std::vector<double> Intervals(NumFunctions + 1);
     std::iota(Intervals.begin(), Intervals.end(), 0);
     auto Distribution = std::piecewise_constant_distribution<double>(
         Intervals.begin(), Intervals.end(), Weights.begin());
@@ -247,7 +247,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
 }
 
 int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
-                    const Vector<SizedFile> &CorporaFiles) {
+                    const std::vector<SizedFile> &CorporaFiles) {
   Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n",
          DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size());
   if (CorporaFiles.empty()) {
@@ -265,7 +265,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
     // we then request tags in [0,Size/2) and [Size/2, Size), and so on.
     // Function number => DFT.
     auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File)));
-    std::unordered_map<size_t, Vector<uint8_t>> DFTMap;
+    std::unordered_map<size_t, std::vector<uint8_t>> DFTMap;
     std::unordered_set<std::string> Cov;
     Command Cmd;
     Cmd.addArgument(DFTBinary);