Update libfuzzer code to compiler-rt 18

Update the LLVM compiler-rt C++ code to the top of the 18.x branch

The summary of the changes from compiler-rt 15 (the last time this code
was updated) are:

1. Improvements to Ctrl+C detection and closing out of a fuzzing session
2. Loop optimizations, in preventing copy operations
3. Dynamic pagesize detection
4. Setting Worker thread names (on Linux)
5. Use of a bitset data structure to track rarity of features "improves
   fuzzer throughput rather dramatically (close to 2x) in early
   exploratory phases"

The LLVM commits which make up the changes in this commit come from the
squash of the following LLVM commits:

```sh
git log --format=reference origin/release/15.x..release/18.x -- compiler-rt/lib/fuzzer/
```

```
3747cde5e84f ([Fuzzer] Enable custom libc++ for Android (#70407), 2023-11-02)
747e0d9f0aad ([compiler-rt] Use std::clamp (NFC), 2023-10-22)
bede46f57eec ([Fuzzer] Optimize UpdateFeatureFrequency (#65288), 2023-09-07)
d3440304b110 ([Fuzzer] Update build.sh to fix build errors (#65496), 2023-09-07)
dd3aa26fc8e9 (Revert "[Fuzzer] SetThreadName implementation for Windows", 2023-08-28)
151e33c768a6 ([NFC][fuzzer] Cleanup SetThreadName after D156317, 2023-08-28)
45eb6026d979 ([fuzzer] Try to fix windows build after D156317, 2023-08-28)
e0e8695a608f (Revert "[fuzzer] Try to fix windows build after D156317", 2023-08-28)
d364597f8a47 ([fuzzer] Try to fix windows build after D156317, 2023-08-28)
cf76ddcbeb10 ([Fuzzer] SetThreadName implementation for Windows, 2023-08-28)
52526065bc80 ([NFC][compiler-rt] Add missing space in libfuzzer -help docs, 2023-07-28)
cb924ddca514 (Revert "[Fuzzer] SetThreadName implementation for Windows", 2023-07-28)
885275bff215 ([Fuzzer] SetThreadName implementation for Windows, 2023-07-27)
f4aa7b5b8d90 (Revert "[Fuzzer] SetThreadName implementation for Windows", 2023-07-27)
e3f935c7f80c ([Fuzzer] SetThreadName implementation for Windows, 2023-07-27)
b2a253855f4e ([Fuzzer] Assign names to workers, 2023-07-26)
9c2f792dceb6 ([fuzzer] Enable loongarch64, 2023-07-18)
42564f97e8ca ([fuzzer][fuchsia] Support RISC-V, 2023-06-15)
facf22b8b07b ([fuzzer] Platfom specific version of PageSize, 2023-05-25)
baa1488c1693 ([fuzzer] Don't hard-code page size in FuzzerUtil.h, 2023-05-25)
cb9f2de2e802 (Revert "[fuzzer] Don't hard-code page size in FuzzerUtil.h", 2023-05-25)
a2b677e81537 ([fuzzer] Don't hard-code page size in FuzzerUtil.h, 2023-05-25)
f98ee40f4b5d ([NFC][Py Reformat] Reformat python files in the rest of the dirs, 2023-05-17)
cdfcf1aa38a4 ([libfuzzer] avoid unneccessary auto-copy, 2023-04-26)
8a986abe98ff ([fuzzer] Fix -Wunreachable-code-aggressive warning in FuzzerDriver.cpp, 2023-04-11)
f2c9d24e8c8d ([compiler-rt] [test] [fuzzer] Don't pass msvc/clang-cl specific flags to mingw tools, 2023-03-31)
90b4d1bcb201 ([fuzzer] Use puts() rather than printf() in CopyFileToErr(), 2023-03-15)
c2df1d8a6d1c ([libfuzzer] add test of cov file-id in control file, 2023-03-22)
d02ff3d57831 (Revert "[fuzzer] Use puts() rather than printf() in CopyFileToErr()", 2023-03-17)
03aa02adb03c ([fuzzer] Use puts() rather than printf() in CopyFileToErr(), 2023-03-15)
0b327814d86c ([libfuzzer] Fix build error due to out-of-line definition of 'Fuzzer' does not match any declaration in 'fuzzer::Fuzzer' (NFC), 2023-03-10)
513d9b9f3d67 ([libfuzzer] avoid unneccessary copy, 2023-03-09)
e09b3be411b0 ([fuzzer] Make sure we never delete Fuzzer, 2023-03-07)
67f5b05cdcdc (Add test for Flags.data_flow_trace, 2023-03-07)
91985c2ee3b3 (Use the right printf format specifiers, 2023-03-07)
6c485409de52 (Adding missing colon, 2023-01-19)
beb3fa2d2efb (Revert "Reland "[compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests"", 2022-12-13)
255c3e3dcb06 (Reland "[compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests", 2022-12-13)
68f4ceaf9b4e (Revert "[compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests", 2022-10-05)
93b1256e38f6 ([compiler-rt][test] Heed COMPILER_RT_DEBUG when compiling unittests, 2022-10-05)
98c2754abd8f ([Libfuzzer] Include signal.h for SIGINT, 2022-09-04)
aa0e9046c16b (Libfuzzer fix for Ctrl + c not working with -fork and -ignore_crashes=1, 2022-08-15)
```

Author: R. Elliott Childre

CHANGELOG.md

@@ -9,7 +9,7 @@ Released YYYY-MM-DD.
 
 ### Changed
 
-* TODO (or remove section if none)
+* Updated to `libFuzzer` commit `3747cde5e84f` (`release/18.x`).
 
 ### Deprecated
 

libfuzzer/CMakeLists.txt

@@ -59,7 +59,7 @@ CHECK_CXX_SOURCE_COMPILES("
 
 set(LIBFUZZER_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
 
-if(OS_NAME MATCHES "Linux|Fuchsia" AND
+if(OS_NAME MATCHES "Android|Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
   list(APPEND LIBFUZZER_CFLAGS -D_LIBCPP_ABI_VERSION=Fuzzer)
@@ -135,7 +135,7 @@ add_compiler_rt_runtime(clang_rt.fuzzer_interceptors
   CFLAGS ${LIBFUZZER_CFLAGS}
   PARENT_TARGET fuzzer)
 
-if(OS_NAME MATCHES "Linux|Fuchsia" AND
+if(OS_NAME MATCHES "Android|Linux|Fuchsia" AND
    COMPILER_RT_LIBCXX_PATH AND
    COMPILER_RT_LIBCXXABI_PATH)
   macro(partially_link_libcxx name dir arch)

libfuzzer/FuzzerCommand.h

@@ -19,6 +19,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+#include <thread>
 
 namespace fuzzer {
 
@@ -139,7 +140,7 @@ class Command final {
   // be the equivalent command line.
   std::string toString() const {
     std::stringstream SS;
-    for (auto arg : getArguments())
+    for (const auto &arg : getArguments())
       SS << arg << " ";
     if (hasOutputFile())
       SS << ">" << getOutputFile() << " ";

libfuzzer/FuzzerCorpus.h

@@ -18,6 +18,7 @@
 #include "FuzzerSHA1.h"
 #include "FuzzerTracePC.h"
 #include <algorithm>
+#include <bitset>
 #include <chrono>
 #include <numeric>
 #include <random>
@@ -77,7 +78,7 @@ struct InputInfo {
     SumIncidence = 0.0;
 
     // Apply add-one smoothing to locally discovered features.
-    for (auto F : FeatureFreqs) {
+    for (const auto &F : FeatureFreqs) {
       double LocalIncidence = F.second + 1;
       Energy -= LocalIncidence * log(LocalIncidence);
       SumIncidence += LocalIncidence;
@@ -382,6 +383,7 @@ class InputCorpus {
       }
 
       // Remove most abundant rare feature.
+      IsRareFeature[Delete] = false;
       RareFeatures[Delete] = RareFeatures.back();
       RareFeatures.pop_back();
 
@@ -397,6 +399,7 @@ class InputCorpus {
 
     // Add rare feature, handle collisions, and update energy.
     RareFeatures.push_back(Idx);
+    IsRareFeature[Idx] = true;
     GlobalFeatureFreqs[Idx] = 0;
     for (auto II : Inputs) {
       II->DeleteFeatureFreq(Idx);
@@ -450,9 +453,7 @@ class InputCorpus {
     uint16_t Freq = GlobalFeatureFreqs[Idx32]++;
 
     // Skip if abundant.
-    if (Freq > FreqOfMostAbundantRareFeature ||
-        std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) ==
-            RareFeatures.end())
+    if (Freq > FreqOfMostAbundantRareFeature || !IsRareFeature[Idx32])
       return;
 
     // Update global frequencies.
@@ -581,6 +582,7 @@ class InputCorpus {
   uint16_t FreqOfMostAbundantRareFeature = 0;
   uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {};
   std::vector<uint32_t> RareFeatures;
+  std::bitset<kFeatureSetSize> IsRareFeature;
 
   std::string OutputCorpus;
 };

libfuzzer/FuzzerDataFlowTrace.cpp

@@ -88,7 +88,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
 //   * a function with a less frequently executed code gets bigger weight.
 std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
   std::vector<double> Res(NumFunctions);
-  for (auto It : Functions) {
+  for (const auto &It : Functions) {
     auto FunctionID = It.first;
     auto Counters = It.second;
     assert(FunctionID < NumFunctions);

libfuzzer/FuzzerDriver.cpp

@@ -293,9 +293,12 @@ static int RunInMultipleProcesses(const std::vector<std::string> &Args,
   std::vector<std::thread> V;
   std::thread Pulse(PulseThread);
   Pulse.detach();
-  for (unsigned i = 0; i < NumWorkers; i++)
-    V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
-                            &HasErrors));
+  V.resize(NumWorkers);
+  for (unsigned i = 0; i < NumWorkers; i++) {
+    V[i] = std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
+                            &HasErrors);
+    SetThreadName(V[i], "FuzzerWorker");
+  }
   for (auto &T : V)
     T.join();
   return HasErrors ? 1 : 0;
@@ -463,7 +466,7 @@ int MinimizeCrashInput(const std::vector<std::string> &Args,
         CurrentFilePath = Flags.exact_artifact_path;
         WriteToFile(U, CurrentFilePath);
       }
-      Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n",
+      Printf("CRASH_MIN: failed to minimize beyond %s (%zu bytes), exiting\n",
              CurrentFilePath.c_str(), U.size());
       break;
     }
@@ -501,7 +504,6 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) {
   F->MinimizeCrashLoop(U);
   Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n");
   exit(0);
-  return 0;
 }
 
 void Merge(Fuzzer *F, FuzzingOptions &Options,
@@ -535,7 +537,7 @@ void Merge(Fuzzer *F, FuzzingOptions &Options,
 
 int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict,
                       UnitVector &Corpus) {
-  Printf("Started dictionary minimization (up to %d tests)\n",
+  Printf("Started dictionary minimization (up to %zu tests)\n",
          Dict.size() * Corpus.size() * 2);
 
   // Scores and usage count for each dictionary unit.
@@ -779,7 +781,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   if (!Options.FocusFunction.empty())
     Options.Entropic = false; // FocusFunction overrides entropic scheduling.
   if (Options.Entropic)
-    Printf("INFO: Running with entropic power schedule (0x%X, %d).\n",
+    Printf("INFO: Running with entropic power schedule (0x%zX, %zu).\n",
            Options.EntropicFeatureFrequencyThreshold,
            Options.EntropicNumberOfRarestFeatures);
   struct EntropicOptions Entropic;
@@ -797,7 +799,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   if (Flags.verbosity)
     Printf("INFO: Seed: %u\n", Seed);
 
-  if (Flags.collect_data_flow && !Flags.fork &&
+  if (Flags.collect_data_flow && Flags.data_flow_trace && !Flags.fork &&
       !(Flags.merge || Flags.set_cover_merge)) {
     if (RunIndividualFiles)
       return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
@@ -860,7 +862,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
         RunOneTest(F, Path.c_str(), Options.MaxLen);
       auto StopTime = system_clock::now();
       auto MS = duration_cast<milliseconds>(StopTime - StartTime).count();
-      Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS);
+      Printf("Executed %s in %ld ms\n", Path.c_str(), (long)MS);
     }
     Printf("***\n"
            "*** NOTE: fuzzing was not performed, you have only\n"

libfuzzer/FuzzerFlags.def

@@ -167,7 +167,7 @@ FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and "
     "purge_allocator_interval=-1 to disable this functionality.")
 FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. "
     "If >= 2 will also print stack traces.")
-FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon"
+FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon "
     "reaching this limit of RSS memory usage.")
 FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit "
     "if the target tries to allocate this number of Mb with one malloc call. "

libfuzzer/FuzzerFork.cpp

@@ -220,7 +220,7 @@ struct GlobalEnv {
       }
     }
     // if (!FilesToAdd.empty() || Job->ExitCode != 0)
-    Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd "
+    Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s: %zd "
            "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n",
            NumRuns, Cov.size(), Features.size(), Files.size(),
            Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes,

libfuzzer/FuzzerIO.cpp

@@ -65,7 +65,7 @@ std::string FileToString(const std::string &Path) {
 }
 
 void CopyFileToErr(const std::string &Path) {
-  Printf("%s", FileToString(Path).c_str());
+  Puts(FileToString(Path).c_str());
 }
 
 void WriteToFile(const Unit &U, const std::string &Path) {
@@ -151,6 +151,11 @@ void CloseStdout() {
   DiscardOutput(1);
 }
 
+void Puts(const char *Str) {
+  fputs(Str, OutputFile);
+  fflush(OutputFile);
+}
+
 void Printf(const char *Fmt, ...) {
   va_list ap;
   va_start(ap, Fmt);

libfuzzer/FuzzerIO.h

@@ -58,6 +58,7 @@ void CloseStdout();
 FILE *GetOutputFile();
 void SetOutputFile(FILE *NewOutputFile);
 
+void Puts(const char *Str);
 void Printf(const char *Fmt, ...);
 void VPrintf(bool Verbose, const char *Fmt, ...);