Merge pull request #68 from rust-fuzz/frewsxcv-bump

Upgrade libfuzzer to 7bf89c2.

Author: frewsxcv

libfuzzer/CMakeLists.txt

@@ -55,15 +55,15 @@ CHECK_CXX_SOURCE_COMPILES("
   }
   " HAS_THREAD_LOCAL)
 
-set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+set(LIBFUZZER_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
 
 if(OS_NAME MATCHES "Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
-  list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer)
-  # Remove -stdlib= which is unused when passing -nostdinc++.
-  string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+  list(APPEND LIBFUZZER_CFLAGS -D_LIBCPP_ABI_VERSION=Fuzzer)
+  append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS)
 elseif(TARGET cxx-headers OR HAVE_LIBCXX)
+  # libFuzzer uses C++ standard library headers.
   set(LIBFUZZER_DEPS cxx-headers)
 endif()
 

libfuzzer/FuzzerCorpus.h

@@ -18,6 +18,7 @@
 #include "FuzzerSHA1.h"
 #include "FuzzerTracePC.h"
 #include <algorithm>
+#include <chrono>
 #include <numeric>
 #include <random>
 #include <unordered_set>
@@ -26,13 +27,15 @@ namespace fuzzer {
 
 struct InputInfo {
   Unit U;  // The actual input data.
+  std::chrono::microseconds TimeOfUnit;
   uint8_t Sha1[kSHA1NumBytes];  // Checksum.
   // Number of features that this input has and no smaller input has.
   size_t NumFeatures = 0;
   size_t Tmp = 0; // Used by ValidateFeatureSet.
   // Stats.
   size_t NumExecutedMutations = 0;
   size_t NumSuccessfullMutations = 0;
+  bool NeverReduce = false;
   bool MayDeleteFile = false;
   bool Reduced = false;
   bool HasFocusFunction = false;
@@ -61,11 +64,15 @@ struct InputInfo {
   }
 
   // Assign more energy to a high-entropy seed, i.e., that reveals more
-  // information about the globally rare features in the neighborhood
-  // of the seed. Since we do not know the entropy of a seed that has
-  // never been executed we assign fresh seeds maximum entropy and
-  // let II->Energy approach the true entropy from above.
-  void UpdateEnergy(size_t GlobalNumberOfFeatures) {
+  // information about the globally rare features in the neighborhood of the
+  // seed. Since we do not know the entropy of a seed that has never been
+  // executed we assign fresh seeds maximum entropy and let II->Energy approach
+  // the true entropy from above. If ScalePerExecTime is true, the computed
+  // entropy is scaled based on how fast this input executes compared to the
+  // average execution time of inputs. The faster an input executes, the more
+  // energy gets assigned to the input.
+  void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime,
+                    std::chrono::microseconds AverageUnitExecutionTime) {
     Energy = 0.0;
     SumIncidence = 0;
 
@@ -88,6 +95,27 @@ struct InputInfo {
     // Normalize.
     if (SumIncidence != 0)
       Energy = (Energy / SumIncidence) + logl(SumIncidence);
+
+    if (ScalePerExecTime) {
+      // Scaling to favor inputs with lower execution time.
+      uint32_t PerfScore = 100;
+      if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 10)
+        PerfScore = 10;
+      else if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 4)
+        PerfScore = 25;
+      else if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 2)
+        PerfScore = 50;
+      else if (TimeOfUnit.count() * 3 > AverageUnitExecutionTime.count() * 4)
+        PerfScore = 75;
+      else if (TimeOfUnit.count() * 4 < AverageUnitExecutionTime.count())
+        PerfScore = 300;
+      else if (TimeOfUnit.count() * 3 < AverageUnitExecutionTime.count())
+        PerfScore = 200;
+      else if (TimeOfUnit.count() * 2 < AverageUnitExecutionTime.count())
+        PerfScore = 150;
+
+      Energy *= PerfScore;
+    }
   }
 
   // Increment the frequency of the feature Idx.
@@ -120,6 +148,7 @@ struct EntropicOptions {
   bool Enabled;
   size_t NumberOfRarestFeatures;
   size_t FeatureFrequencyThreshold;
+  bool ScalePerExecTime;
 };
 
 class InputCorpus {
@@ -177,7 +206,8 @@ class InputCorpus {
   bool empty() const { return Inputs.empty(); }
   const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
   InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
-                         bool HasFocusFunction,
+                         bool HasFocusFunction, bool NeverReduce,
+                         std::chrono::microseconds TimeOfUnit,
                          const Vector<uint32_t> &FeatureSet,
                          const DataFlowTrace &DFT, const InputInfo *BaseII) {
     assert(!U.empty());
@@ -187,6 +217,8 @@ class InputCorpus {
     InputInfo &II = *Inputs.back();
     II.U = U;
     II.NumFeatures = NumFeatures;
+    II.NeverReduce = NeverReduce;
+    II.TimeOfUnit = TimeOfUnit;
     II.MayDeleteFile = MayDeleteFile;
     II.UniqFeatureSet = FeatureSet;
     II.HasFocusFunction = HasFocusFunction;
@@ -268,6 +300,15 @@ class InputCorpus {
     return II;
   }
 
+  InputInfo &ChooseUnitToCrossOverWith(Random &Rand, bool UniformDist) {
+    if (!UniformDist) {
+      return ChooseUnitToMutate(Rand);
+    }
+    InputInfo &II = *Inputs[Rand(Inputs.size())];
+    assert(!II.U.empty());
+    return II;
+  }
+
   // Returns an index of random unit from the corpus to mutate.
   size_t ChooseUnitIdxToMutate(Random &Rand) {
     UpdateCorpusDistribution(Rand);
@@ -460,12 +501,19 @@ class InputCorpus {
     Weights.resize(N);
     std::iota(Intervals.begin(), Intervals.end(), 0);
 
+    std::chrono::microseconds AverageUnitExecutionTime(0);
+    for (auto II : Inputs) {
+      AverageUnitExecutionTime += II->TimeOfUnit;
+    }
+    AverageUnitExecutionTime /= N;
+
     bool VanillaSchedule = true;
     if (Entropic.Enabled) {
       for (auto II : Inputs) {
         if (II->NeedsEnergyUpdate && II->Energy != 0.0) {
           II->NeedsEnergyUpdate = false;
-          II->UpdateEnergy(RareFeatures.size());
+          II->UpdateEnergy(RareFeatures.size(), Entropic.ScalePerExecTime,
+                           AverageUnitExecutionTime);
         }
       }
 

libfuzzer/FuzzerDataFlowTrace.cpp

@@ -253,7 +253,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
     return 1;
   }
 
-  static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0";
+  static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0";
   putenv(DFSanEnv);
   MkDir(DirPath);
   for (auto &F : CorporaFiles) {

libfuzzer/FuzzerDriver.cpp

@@ -250,6 +250,28 @@ static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter,
   }
 }
 
+static void ValidateDirectoryExists(const std::string &Path,
+                                    bool CreateDirectory) {
+  if (Path.empty()) {
+    Printf("ERROR: Provided directory path is an empty string\n");
+    exit(1);
+  }
+
+  if (IsDirectory(Path))
+    return;
+
+  if (CreateDirectory) {
+    if (!MkDirRecursive(Path)) {
+      Printf("ERROR: Failed to create directory \"%s\"\n", Path.c_str());
+      exit(1);
+    }
+    return;
+  }
+
+  Printf("ERROR: The required directory \"%s\" does not exist\n", Path.c_str());
+  exit(1);
+}
+
 std::string CloneArgsWithoutX(const Vector<std::string> &Args,
                               const char *X1, const char *X2) {
   std::string Cmd;
@@ -299,7 +321,12 @@ int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) {
   if (MaxLen && MaxLen < U.size())
     U.resize(MaxLen);
   F->ExecuteCallback(U.data(), U.size());
-  F->TryDetectingAMemoryLeak(U.data(), U.size(), true);
+  if (Flags.print_full_coverage) {
+    // Leak detection is not needed when collecting full coverage data.
+    F->TPCUpdateObservedPCs();
+  } else {
+    F->TryDetectingAMemoryLeak(U.data(), U.size(), true);
+  }
   return 0;
 }
 
@@ -649,6 +676,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.Verbosity = Flags.verbosity;
   Options.MaxLen = Flags.max_len;
   Options.LenControl = Flags.len_control;
+  Options.KeepSeed = Flags.keep_seed;
   Options.UnitTimeoutSec = Flags.timeout;
   Options.ErrorExitCode = Flags.error_exitcode;
   Options.TimeoutExitCode = Flags.timeout_exitcode;
@@ -657,6 +685,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.IgnoreCrashes = Flags.ignore_crashes;
   Options.MaxTotalTimeSec = Flags.max_total_time;
   Options.DoCrossOver = Flags.cross_over;
+  Options.CrossOverUniformDist = Flags.cross_over_uniform_dist;
   Options.MutateDepth = Flags.mutate_depth;
   Options.ReduceDepth = Flags.reduce_depth;
   Options.UseCounters = Flags.use_counters;
@@ -678,13 +707,33 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
     Options.MallocLimitMb = Options.RssLimitMb;
   if (Flags.runs >= 0)
     Options.MaxNumberOfRuns = Flags.runs;
-  if (!Inputs->empty() && !Flags.minimize_crash_internal_step)
-    Options.OutputCorpus = (*Inputs)[0];
+  if (!Inputs->empty() && !Flags.minimize_crash_internal_step) {
+    // Ensure output corpus assumed to be the first arbitrary argument input
+    // is not a path to an existing file.
+    std::string OutputCorpusDir = (*Inputs)[0];
+    if (!IsFile(OutputCorpusDir)) {
+      Options.OutputCorpus = OutputCorpusDir;
+      ValidateDirectoryExists(Options.OutputCorpus, Flags.create_missing_dirs);
+    }
+  }
   Options.ReportSlowUnits = Flags.report_slow_units;
-  if (Flags.artifact_prefix)
+  if (Flags.artifact_prefix) {
     Options.ArtifactPrefix = Flags.artifact_prefix;
-  if (Flags.exact_artifact_path)
+
+    // Since the prefix could be a full path to a file name prefix, assume
+    // that if the path ends with the platform's separator that a directory
+    // is desired
+    std::string ArtifactPathDir = Options.ArtifactPrefix;
+    if (!IsSeparator(ArtifactPathDir[ArtifactPathDir.length() - 1])) {
+      ArtifactPathDir = DirName(ArtifactPathDir);
+    }
+    ValidateDirectoryExists(ArtifactPathDir, Flags.create_missing_dirs);
+  }
+  if (Flags.exact_artifact_path) {
     Options.ExactArtifactPath = Flags.exact_artifact_path;
+    ValidateDirectoryExists(DirName(Options.ExactArtifactPath),
+                            Flags.create_missing_dirs);
+  }
   Vector<Unit> Dictionary;
   if (Flags.dict)
     if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
@@ -699,6 +748,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.PrintFinalStats = Flags.print_final_stats;
   Options.PrintCorpusStats = Flags.print_corpus_stats;
   Options.PrintCoverage = Flags.print_coverage;
+  Options.PrintFullCoverage = Flags.print_full_coverage;
   if (Flags.exit_on_src_pos)
     Options.ExitOnSrcPos = Flags.exit_on_src_pos;
   if (Flags.exit_on_item)
@@ -707,8 +757,12 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
     Options.FocusFunction = Flags.focus_function;
   if (Flags.data_flow_trace)
     Options.DataFlowTrace = Flags.data_flow_trace;
-  if (Flags.features_dir)
+  if (Flags.features_dir) {
     Options.FeaturesDir = Flags.features_dir;
+    ValidateDirectoryExists(Options.FeaturesDir, Flags.create_missing_dirs);
+  }
+  if (Flags.mutation_graph_file)
+    Options.MutationGraphFile = Flags.mutation_graph_file;
   if (Flags.collect_data_flow)
     Options.CollectDataFlow = Flags.collect_data_flow;
   if (Flags.stop_file)
@@ -718,21 +772,19 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
       (size_t)Flags.entropic_feature_frequency_threshold;
   Options.EntropicNumberOfRarestFeatures =
       (size_t)Flags.entropic_number_of_rarest_features;
-  if (Options.Entropic) {
-    if (!Options.FocusFunction.empty()) {
-      Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot "
-             "be used together.\n");
-      exit(1);
-    }
+  Options.EntropicScalePerExecTime = Flags.entropic_scale_per_exec_time;
+  if (!Options.FocusFunction.empty())
+    Options.Entropic = false; // FocusFunction overrides entropic scheduling.
+  if (Options.Entropic)
     Printf("INFO: Running with entropic power schedule (0x%X, %d).\n",
            Options.EntropicFeatureFrequencyThreshold,
            Options.EntropicNumberOfRarestFeatures);
-  }
   struct EntropicOptions Entropic;
   Entropic.Enabled = Options.Entropic;
   Entropic.FeatureFrequencyThreshold =
       Options.EntropicFeatureFrequencyThreshold;
   Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures;
+  Entropic.ScalePerExecTime = Options.EntropicScalePerExecTime;
 
   unsigned Seed = Flags.seed;
   // Initialize Seed.
@@ -767,6 +819,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
 #endif // LIBFUZZER_EMSCRIPTEN
 
   Options.HandleAbrt = Flags.handle_abrt;
+  Options.HandleAlrm = !Flags.minimize_crash;
   Options.HandleBus = Flags.handle_bus;
   Options.HandleFpe = Flags.handle_fpe;
   Options.HandleIll = Flags.handle_ill;
@@ -776,6 +829,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.HandleXfsz = Flags.handle_xfsz;
   Options.HandleUsr1 = Flags.handle_usr1;
   Options.HandleUsr2 = Flags.handle_usr2;
+  Options.HandleWinExcept = Flags.handle_winexcept;
+
   SetSignalHandler(Options);
 
   std::atexit(Fuzzer::StaticExitCallback);
@@ -858,6 +913,12 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   exit(0);  // Don't let F destroy itself.
 }
 
+extern "C" ATTRIBUTE_INTERFACE int
+LLVMFuzzerRunDriver(int *argc, char ***argv,
+                    int (*UserCb)(const uint8_t *Data, size_t Size)) {
+  return FuzzerDriver(argc, argv, UserCb);
+}
+
 // Storage for global ExternalFunctions object.
 ExternalFunctions *EF = nullptr;
 

libfuzzer/FuzzerExtFunctionsWeak.cpp

@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 #include "FuzzerPlatform.h"
 #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA ||                \
-    LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN
+    LIBFUZZER_FREEBSD || LIBFUZZER_EMSCRIPTEN
 
 #include "FuzzerExtFunctions.h"
 #include "FuzzerIO.h"

libfuzzer/FuzzerExtraCounters.cpp

@@ -12,7 +12,7 @@
 #include <cstdint>
 
 #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD ||                \
-    LIBFUZZER_OPENBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN
+    LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN
 __attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters;
 __attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters;