Use the stop time to cap command deadlines when running batches for fuzzing.

xinhaoyuan · copybara-github · commit 4e53ed8fd4fb · 2025-08-25T08:23:16.000-07:00
Do not report crashes on stop conditions.

Exit early in batch condition when termination is requested.

Prolong the unit testing duration to 10s to reduce flakiness.

This allows the stop time to be enforced more precisely instead of having to wait for the entire batch, without reporting the potential crashes due to interrupting the command.

After this we don't need to set batch timeout to the test time limit - it was a quick workaround for the same requirement.

PiperOrigin-RevId: 791875139
diff --git a/centipede/BUILD b/centipede/BUILD
@@ -665,6 +665,7 @@ cc_library(
         ":runner_request",
         ":runner_result",
         ":shared_memory_blob_sequence",
+        ":stop",
         ":util",
         ":workdir",
         "@abseil-cpp//absl/base:nullability",
diff --git a/centipede/centipede.cc b/centipede/centipede.cc
@@ -362,9 +362,17 @@ bool Centipede::InputPassesFilter(const ByteArray &input) {
 bool Centipede::ExecuteAndReportCrash(std::string_view binary,
                                       const std::vector<ByteArray> &input_vec,
                                       BatchResult &batch_result) {
-  bool success = user_callbacks_.Execute(binary, input_vec, batch_result);
-  if (!success) ReportCrash(binary, input_vec, batch_result);
-  return success || batch_result.IsIgnoredFailure();
+  bool success =
+      user_callbacks_.Execute(binary, input_vec, batch_result, GetStopTime());
+  if (success) return true;
+  if (ShouldStop()) {
+    FUZZTEST_LOG_FIRST_N(WARNING, 1)
+        << "Stop condition met - not reporting further crashes possibly "
+           "related to the stop condition.";
+    return true;
+  }
+  ReportCrash(binary, input_vec, batch_result);
+  return batch_result.IsIgnoredFailure();
 }
 
 // *** Highly experimental and risky. May not scale well for large targets. ***
@@ -976,7 +984,8 @@ void Centipede::ReportCrash(std::string_view binary,
     if (ShouldStop()) return;
     const auto &one_input = input_vec[input_idx];
     BatchResult one_input_batch_result;
-    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result)) {
+    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result,
+                                 absl::InfiniteFuture())) {
       auto hash = Hash(one_input);
       auto crash_dir = wd_.CrashReproducerDirPaths().MyShard();
       FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir));
diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc
@@ -50,6 +50,7 @@
 #include "./centipede/mutation_input.h"
 #include "./centipede/runner_request.h"
 #include "./centipede/runner_result.h"
+#include "./centipede/stop.h"
 #include "./centipede/util.h"
 #include "./centipede/workdir.h"
 #include "./common/blob_file.h"
@@ -472,14 +473,15 @@ void CentipedeCallbacks::CleanUpPersistentMode() {
       command_contexts_.end());
 }
 
-int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
+int CentipedeCallbacks::RunBatchForBinary(std::string_view binary,
+                                          absl::Time deadline) {
   auto& command_context = GetOrCreateCommandContextForBinary(binary);
   auto& cmd = command_context.cmd;
   const absl::Duration amortized_timeout =
       env_.timeout_per_batch == 0
           ? absl::InfiniteDuration()
           : absl::Seconds(env_.timeout_per_batch) + absl::Seconds(5);
-  const auto deadline = absl::Now() + amortized_timeout;
+  deadline = std::min(absl::Now() + amortized_timeout, deadline);
   int exit_code = EXIT_SUCCESS;
   const bool should_clean_up = [&] {
     if (!cmd.is_executing() && !cmd.ExecuteAsync()) {
@@ -497,11 +499,12 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
   if (should_clean_up) {
     exit_code = [&] {
       if (!cmd.is_executing()) return EXIT_FAILURE;
-      FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution.";
+      FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution with timeout "
+                          << kCommandCleanupTimeout;
       cmd.RequestStop();
       const auto ret = cmd.Wait(absl::Now() + kCommandCleanupTimeout);
       if (ret.has_value()) return *ret;
-      FUZZTEST_LOG(ERROR) << "Batch execution cleanup failed to end in 60s.";
+      FUZZTEST_LOG(ERROR) << "Failed to wait for the batch execution cleanup.";
       return EXIT_FAILURE;
     }();
     command_contexts_.erase(
@@ -515,7 +518,7 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
 
 int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
     std::string_view binary, const std::vector<ByteArray>& inputs,
-    BatchResult& batch_result) {
+    BatchResult& batch_result, absl::Time deadline) {
   auto start_time = absl::Now();
   batch_result.ClearAndResize(inputs.size());
 
@@ -541,7 +544,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
   }
 
   // Run.
-  const int exit_code = RunBatchForBinary(binary);
+  const int exit_code = RunBatchForBinary(binary, deadline);
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   // Get results.
@@ -699,7 +702,8 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary(
       << VV(num_inputs_written) << VV(inputs.size());
 
   // Execute.
-  const int exit_code = RunBatchForBinary(binary);
+  const int exit_code =
+      RunBatchForBinary(binary, /*deadline=*/absl::InfiniteFuture());
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   if (exit_code != EXIT_SUCCESS) {
diff --git a/centipede/centipede_callbacks.h b/centipede/centipede_callbacks.h
@@ -65,8 +65,8 @@ class CentipedeCallbacks {
   // Post-condition:
   // `batch_result` has results for every `input`, even on failure.
   virtual bool Execute(std::string_view binary,
-                       const std::vector<ByteArray> &inputs,
-                       BatchResult &batch_result) = 0;
+                       const std::vector<ByteArray>& inputs,
+                       BatchResult& batch_result, absl::Time deadline) = 0;
 
   // Takes non-empty `inputs` and returns at most `num_mutants` mutated inputs.
   virtual std::vector<ByteArray> Mutate(
@@ -103,8 +103,8 @@ class CentipedeCallbacks {
   // Same as ExecuteCentipedeSancovBinary, but uses shared memory.
   // Much faster for fast targets since it uses fewer system calls.
   int ExecuteCentipedeSancovBinaryWithShmem(
-      std::string_view binary, const std::vector<ByteArray> &inputs,
-      BatchResult &batch_result);
+      std::string_view binary, const std::vector<ByteArray>& inputs,
+      BatchResult& batch_result, absl::Time deadline);
 
   // Constructs a string CENTIPEDE_RUNNER_FLAGS=":flag1:flag2:...",
   // where the flags are determined by `env` and also include `extra_flags`.
@@ -173,7 +173,7 @@ class CentipedeCallbacks {
   // Returns a CommandContext with matching `binary`. Creates one if needed.
   CommandContext& GetOrCreateCommandContextForBinary(std::string_view binary);
   // Runs a batch with the command `binary` and returns the exit code.
-  int RunBatchForBinary(std::string_view binary);
+  int RunBatchForBinary(std::string_view binary, absl::Time deadline);
 
   // Prints the execution log from the last executed binary.
   void PrintExecutionLog() const;
diff --git a/centipede/centipede_default_callbacks.cc b/centipede/centipede_default_callbacks.cc
@@ -46,10 +46,11 @@ CentipedeDefaultCallbacks::CentipedeDefaultCallbacks(const Environment &env)
 }
 
 bool CentipedeDefaultCallbacks::Execute(std::string_view binary,
-                                        const std::vector<ByteArray> &inputs,
-                                        BatchResult &batch_result) {
-  return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result) ==
-         0;
+                                        const std::vector<ByteArray>& inputs,
+                                        BatchResult& batch_result,
+                                        absl::Time deadline) {
+  return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result,
+                                               deadline) == 0;
 }
 
 size_t CentipedeDefaultCallbacks::GetSeeds(size_t num_seeds,
diff --git a/centipede/centipede_default_callbacks.h b/centipede/centipede_default_callbacks.h
@@ -40,8 +40,8 @@ class CentipedeDefaultCallbacks : public CentipedeCallbacks {
   explicit CentipedeDefaultCallbacks(const Environment &env);
   size_t GetSeeds(size_t num_seeds, std::vector<ByteArray> &seeds) override;
   absl::StatusOr<std::string> GetSerializedTargetConfig() override;
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override;
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override;
   std::vector<ByteArray> Mutate(const std::vector<MutationInputRef> &inputs,
                                 size_t num_mutants) override;
 
diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc
@@ -305,7 +305,7 @@ absl::flat_hash_set<std::string> PruneOldCrashesAndGetRemainingCrashSignatures(
     FUZZTEST_CHECK_OK(
         RemoteFileGetContents(crashing_input_file, crashing_input));
     const bool is_reproducible = !scoped_callbacks.callbacks()->Execute(
-        env.binary, {crashing_input}, batch_result);
+        env.binary, {crashing_input}, batch_result, absl::InfiniteFuture());
     const bool is_duplicate =
         is_reproducible && !batch_result.IsSetupFailure() &&
         !remaining_crash_signatures.insert(batch_result.failure_signature())
@@ -686,6 +686,8 @@ int UpdateCorpusDatabaseForFuzzTests(
           << "Skip updating corpus database due to early stop requested.";
       continue;
     }
+    // The test time limit does not apply for the rest of the steps.
+    ClearEarlyStopRequestAndSetStopTime(/*stop_time=*/absl::InfiniteFuture());
 
     // TODO(xinhaoyuan): Have a separate flag to skip corpus updating instead
     // of checking whether workdir is specified or not.
diff --git a/centipede/environment.cc b/centipede/environment.cc
@@ -285,17 +285,6 @@ void Environment::UpdateWithTargetConfig(
   timeout_per_input = time_limit_per_input_sec;
   UpdateTimeoutPerBatchIfEqualTo(autocomputed_timeout_per_batch);
 
-  // Adjust `timeout_per_batch` to never exceed the test time limit.
-  if (const auto test_time_limit = config.GetTimeLimitPerTest();
-      test_time_limit < absl::InfiniteDuration()) {
-    const size_t test_time_limit_seconds =
-        convert_to_seconds(test_time_limit, "Test time limit");
-    timeout_per_batch =
-        timeout_per_batch == 0
-            ? test_time_limit_seconds
-            : std::min(timeout_per_batch, test_time_limit_seconds);
-  }
-
   // Convert bytes to MB by rounding up.
   constexpr auto bytes_to_mb = [](size_t bytes) {
     return bytes == 0 ? 0 : (bytes - 1) / 1024 / 1024 + 1;
diff --git a/centipede/environment_test.cc b/centipede/environment_test.cc
@@ -147,30 +147,6 @@ TEST(Environment,
   EXPECT_EQ(env.timeout_per_batch, 0);
 }
 
-TEST(Environment, UpdatesTimeoutPerBatchFromTargetConfigTimeLimit) {
-  Environment env;
-  fuzztest::internal::Configuration config;
-  config.time_limit = absl::Seconds(123);
-  config.time_budget_type = fuzztest::internal::TimeBudgetType::kPerTest;
-  FUZZTEST_CHECK(config.GetTimeLimitPerTest() == absl::Seconds(123));
-  env.UpdateWithTargetConfig(config);
-  EXPECT_EQ(env.timeout_per_batch, 123)
-      << "`timeout_per_batch` should be set to the test time limit when it was "
-         "previously unset";
-
-  env.timeout_per_batch = 456;
-  env.UpdateWithTargetConfig(config);
-  EXPECT_EQ(env.timeout_per_batch, 123)
-      << "`timeout_per_batch` should be set to test time limit when it is "
-         "shorter than the previous value";
-
-  env.timeout_per_batch = 56;
-  env.UpdateWithTargetConfig(config);
-  EXPECT_EQ(env.timeout_per_batch, 56)
-      << "`timeout_per_batch` should not be updated with the test time limit "
-         "when it is longer than the previous value";
-}
-
 TEST(Environment, UpdatesRssLimitMbFromTargetConfigRssLimit) {
   Environment env;
   env.rss_limit_mb = Environment::Default().rss_limit_mb;
diff --git a/centipede/minimize_crash.cc b/centipede/minimize_crash.cc
@@ -122,7 +122,8 @@ static void MinimizeCrash(const Environment &env,
     }
 
     // Execute all mutants. If a new crasher is found, add it to `queue`.
-    if (!callbacks->Execute(env.binary, smaller_mutants, batch_result)) {
+    if (!callbacks->Execute(env.binary, smaller_mutants, batch_result,
+                            absl::InfiniteFuture())) {
       size_t crash_inputs_idx = batch_result.num_outputs_read();
       FUZZTEST_CHECK_LT(crash_inputs_idx, smaller_mutants.size());
       const auto &new_crasher = smaller_mutants[crash_inputs_idx];
@@ -142,7 +143,8 @@ int MinimizeCrash(ByteSpan crashy_input, const Environment &env,
 
   BatchResult batch_result;
   ByteArray original_crashy_input(crashy_input.begin(), crashy_input.end());
-  if (callbacks->Execute(env.binary, {original_crashy_input}, batch_result)) {
+  if (callbacks->Execute(env.binary, {original_crashy_input}, batch_result,
+                         absl::InfiniteFuture())) {
     FUZZTEST_LOG(INFO) << "The original crashy input did not crash; exiting";
     return EXIT_FAILURE;
   }
diff --git a/centipede/runner.cc b/centipede/runner.cc
@@ -173,6 +173,8 @@ static void CheckWatchdogLimits() {
             resource.what, resource.limit, resource.units, resource.value);
         CentipedeSetFailureDescription(resource.failure);
         std::abort();
+      } else {
+        while (true) sleep(1);  // NOLINT
       }
     }
   }
@@ -493,12 +495,14 @@ static int ExecuteInputsFromShmem(BlobSequence &inputs_blobseq,
 
     RunOneInput(data.data(), data.size(), callbacks);
 
+    if (state->has_failure_description.test()) break;
+
     if (!FinishSendingOutputsToEngine(outputs_blobseq)) break;
   }
 
   CentipedeEndExecutionBatch();
 
-  return EXIT_SUCCESS;
+  return state->has_failure_description.test() ? EXIT_FAILURE : EXIT_SUCCESS;
 }
 
 // Dumps seed inputs to `output_dir`. Also see `GetSeedsViaExternalBinary()`.
@@ -1011,23 +1015,21 @@ extern "C" void CentipedeSetExecutionResult(const uint8_t *data, size_t size) {
 extern "C" void CentipedeSetFailureDescription(const char *description) {
   using fuzztest::internal::state;
   if (state->failure_description_path == nullptr) return;
-  // Make sure that the write is atomic and only happens once.
-  [[maybe_unused]] static int write_once = [=] {
-    FILE* f = fopen(state->failure_description_path, "w");
-    if (f == nullptr) {
-      perror("FAILURE: fopen()");
-      return 0;
-    }
-    const auto len = strlen(description);
-    if (fwrite(description, 1, len, f) != len) {
-      perror("FAILURE: fwrite()");
-    }
-    if (fflush(f) != 0) {
-      perror("FAILURE: fflush()");
-    }
-    if (fclose(f) != 0) {
-      perror("FAILURE: fclose()");
-    }
-    return 0;
-  }();
+  if (state->has_failure_description.test_and_set()) return;
+  FILE* f = fopen(state->failure_description_path, "w");
+  if (f == nullptr) {
+    perror("FAILURE: fopen()");
+    return;
+  }
+  const auto len = strlen(description);
+  if (fwrite(description, 1, len, f) != len) {
+    perror("FAILURE: fwrite()");
+  }
+  if (fflush(f) != 0) {
+    perror("FAILURE: fflush()");
+  }
+  if (fclose(f) != 0) {
+    perror("FAILURE: fclose()");
+  }
+  return;
 }
diff --git a/centipede/runner.h b/centipede/runner.h
@@ -80,6 +80,8 @@ struct GlobalRunnerState {
   const char *failure_description_path =
       flag_helper.GetStringFlag(":failure_description_path=");
 
+  std::atomic_flag has_failure_description = ATOMIC_FLAG_INIT;
+
   const char* persistent_mode_socket_path =
       flag_helper.GetStringFlag(":persistent_mode_socket=");
   int persistent_mode_socket = 0;
diff --git a/centipede/runner_interface.h b/centipede/runner_interface.h
@@ -132,6 +132,9 @@ extern "C" void CentipedeSetExecutionResult(const uint8_t *data, size_t size);
 
 // Set the failure description for the runner to propagate further. Only the
 // description from the first call will be used.
+//
+// If used during executing batch inputs, the rest of the inputs would be
+// skipped and the batch would be considered as failed.
 extern "C" void CentipedeSetFailureDescription(const char *description);
 
 namespace fuzztest::internal {
diff --git a/centipede/stop.cc b/centipede/stop.cc
@@ -46,6 +46,8 @@ void RequestEarlyStop(int exit_code) {
   early_stop.store({exit_code, true}, std::memory_order_release);
 }
 
+absl::Time GetStopTime() { return stop_time; }
+
 bool ShouldStop() { return EarlyStopRequested() || stop_time < absl::Now(); }
 
 int ExitCode() { return early_stop.load(std::memory_order_acquire).exit_code; }
diff --git a/centipede/stop.h b/centipede/stop.h
@@ -46,6 +46,13 @@ bool EarlyStopRequested();
 // ENSURES: Thread-safe.
 bool ShouldStop();
 
+// Returns the stop time set from the recent
+// `ClearEarlyStopRequestAndSetStopTime()`, or `absl::InfiniteFuture()` it was
+// not set.
+//
+// ENSURES: Thread-safe.
+absl::Time GetStopTime();
+
 // Returns the value most recently passed to `RequestEarlyStop()` or 0 if
 // `RequestEarlyStop()` was not called since the most recent call to
 // `ClearEarlyStopRequestAndSetStopTime()` (if any).
diff --git a/e2e_tests/corpus_database_test.cc b/e2e_tests/corpus_database_test.cc
@@ -38,7 +38,7 @@
 #define EXPECT_THAT_LOG(log, matcher)                                  \
   EXPECT_TRUE(testing::Value(log, matcher))                            \
       << "Matcher: " << testing::DescribeMatcher<std::string>(matcher) \
-      << "Contents of " #log ":\n"                                     \
+      << "\nContents of " #log ":\n"                                   \
       << log
 
 namespace fuzztest::internal {
@@ -123,6 +123,7 @@ class UpdateCorpusDatabaseTest
     // Dumping stack trace in gtest would slow down the execution, causing
     // test flakiness.
     options.flags[GTEST_FLAG_PREFIX_ "stack_trace_depth"] = "0";
+    options.flags["symbolize_stacktrace"] = "0";
     switch (GetParam()) {
       case ExecutionModelParam::kTestBinary:
         return RunBinary(binary_path, options);
diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc
diff --git a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc
diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc

Original file line number	Diff line number	Diff line change
`@@ -46,10 +46,11 @@ CentipedeDefaultCallbacks::CentipedeDefaultCallbacks(const Environment &env)`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`bool CentipedeDefaultCallbacks::Execute(std::string_view binary,`
`49`		`- const std::vector<ByteArray> &inputs,`
`50`		`- BatchResult &batch_result) {`
`51`		`- return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result) ==`
`52`		`- 0;`
	`49`	`+ const std::vector<ByteArray>& inputs,`
	`50`	`+ BatchResult& batch_result,`
	`51`	`+ absl::Time deadline) {`
	`52`	`+ return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result,`
	`53`	`+ deadline) == 0;`
`53`	`54`	`}`
`54`	`55`
`55`	`56`	`size_t CentipedeDefaultCallbacks::GetSeeds(size_t num_seeds,`