Use the stop time to set batch execution deadlines.

Do not report crashes on stop conditions.

This allows the stop time to be enforced more precisely, and avoids reporting the potential crashes due to early stopping.

After this we don't need to set batch timeout to the test time limit - it was a quick workaround for the same purpose.

Change the e2e unit test time to 10s to reduce flakiness.

PiperOrigin-RevId: 791875139

Author: xinhaoyuan

centipede/BUILD

@@ -389,6 +389,7 @@ cc_library(
         ":workdir",
         "@abseil-cpp//absl/base:core_headers",
         "@abseil-cpp//absl/synchronization",
+        "@abseil-cpp//absl/time",
         "@com_google_fuzztest//common:defs",
         "@com_google_fuzztest//common:hash",
         "@com_google_fuzztest//common:logging",
@@ -909,6 +910,7 @@ cc_library(
         ":stop",
         "@abseil-cpp//absl/status",
         "@abseil-cpp//absl/status:statusor",
+        "@abseil-cpp//absl/time",
         "@com_google_fuzztest//common:defs",
         "@com_google_fuzztest//common:logging",
     ],
@@ -1270,6 +1272,7 @@ cc_library(
         ":mutation_input",
         ":runner_result",
         ":util",
+        "@abseil-cpp//absl/time",
         "@com_google_fuzztest//common:defs",
         "@com_google_fuzztest//common:logging",
     ],
@@ -1474,6 +1477,7 @@ cc_test(
         ":util",
         ":workdir",
         "@abseil-cpp//absl/base:nullability",
+        "@abseil-cpp//absl/time",
         "@com_google_fuzztest//common:defs",
         "@com_google_fuzztest//common:test_util",
         "@googletest//:gtest_main",

centipede/centipede.cc

@@ -362,9 +362,17 @@ bool Centipede::InputPassesFilter(const ByteArray &input) {
 bool Centipede::ExecuteAndReportCrash(std::string_view binary,
                                       const std::vector<ByteArray> &input_vec,
                                       BatchResult &batch_result) {
-  bool success = user_callbacks_.Execute(binary, input_vec, batch_result);
-  if (!success) ReportCrash(binary, input_vec, batch_result);
-  return success || batch_result.IsIgnoredFailure();
+  bool success =
+      user_callbacks_.Execute(binary, input_vec, batch_result, GetStopTime());
+  if (success) return true;
+  if (ShouldStop()) {
+    FUZZTEST_LOG_FIRST_N(WARNING, 1)
+        << "Stop condition met - not reporting further crashes possibly "
+           "related to the stop condition.";
+    return true;
+  }
+  ReportCrash(binary, input_vec, batch_result);
+  return batch_result.IsIgnoredFailure();
 }
 
 // *** Highly experimental and risky. May not scale well for large targets. ***
@@ -986,7 +994,8 @@ void Centipede::ReportCrash(std::string_view binary,
     if (ShouldStop()) return;
     const auto &one_input = input_vec[input_idx];
     BatchResult one_input_batch_result;
-    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result)) {
+    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result,
+                                 absl::InfiniteFuture())) {
       auto hash = Hash(one_input);
       auto crash_dir = wd_.CrashReproducerDirPaths().MyShard();
       FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir));

centipede/centipede_callbacks.cc

@@ -472,14 +472,15 @@ void CentipedeCallbacks::CleanUpPersistentMode() {
       command_contexts_.end());
 }
 
-int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
+int CentipedeCallbacks::RunBatchForBinary(std::string_view binary,
+                                          absl::Time deadline) {
   auto& command_context = GetOrCreateCommandContextForBinary(binary);
   auto& cmd = command_context.cmd;
   const absl::Duration amortized_timeout =
       env_.timeout_per_batch == 0
           ? absl::InfiniteDuration()
           : absl::Seconds(env_.timeout_per_batch) + absl::Seconds(5);
-  const auto deadline = absl::Now() + amortized_timeout;
+  deadline = std::min(absl::Now() + amortized_timeout, deadline);
   int exit_code = EXIT_SUCCESS;
   const bool should_clean_up = [&] {
     if (!cmd.is_executing() && !cmd.ExecuteAsync()) {
@@ -497,11 +498,12 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
   if (should_clean_up) {
     exit_code = [&] {
       if (!cmd.is_executing()) return EXIT_FAILURE;
-      FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution.";
+      FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution with timeout "
+                          << kCommandCleanupTimeout;
       cmd.RequestStop();
       const auto ret = cmd.Wait(absl::Now() + kCommandCleanupTimeout);
       if (ret.has_value()) return *ret;
-      FUZZTEST_LOG(ERROR) << "Batch execution cleanup failed to end in 60s.";
+      FUZZTEST_LOG(ERROR) << "Failed to wait for the batch execution cleanup.";
       return EXIT_FAILURE;
     }();
     command_contexts_.erase(
@@ -515,7 +517,7 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) {
 
 int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
     std::string_view binary, const std::vector<ByteArray>& inputs,
-    BatchResult& batch_result) {
+    BatchResult& batch_result, absl::Time deadline) {
   auto start_time = absl::Now();
   batch_result.ClearAndResize(inputs.size());
 
@@ -541,7 +543,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem(
   }
 
   // Run.
-  const int exit_code = RunBatchForBinary(binary);
+  const int exit_code = RunBatchForBinary(binary, deadline);
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   // Get results.
@@ -699,7 +701,8 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary(
       << VV(num_inputs_written) << VV(inputs.size());
 
   // Execute.
-  const int exit_code = RunBatchForBinary(binary);
+  const int exit_code =
+      RunBatchForBinary(binary, /*deadline=*/absl::InfiniteFuture());
   inputs_blobseq_.ReleaseSharedMemory();  // Inputs are already consumed.
 
   if (exit_code != EXIT_SUCCESS) {

centipede/centipede_callbacks.h

@@ -65,8 +65,8 @@ class CentipedeCallbacks {
   // Post-condition:
   // `batch_result` has results for every `input`, even on failure.
   virtual bool Execute(std::string_view binary,
-                       const std::vector<ByteArray> &inputs,
-                       BatchResult &batch_result) = 0;
+                       const std::vector<ByteArray>& inputs,
+                       BatchResult& batch_result, absl::Time deadline) = 0;
 
   // Takes non-empty `inputs` and returns at most `num_mutants` mutated inputs.
   virtual std::vector<ByteArray> Mutate(
@@ -103,8 +103,8 @@ class CentipedeCallbacks {
   // Same as ExecuteCentipedeSancovBinary, but uses shared memory.
   // Much faster for fast targets since it uses fewer system calls.
   int ExecuteCentipedeSancovBinaryWithShmem(
-      std::string_view binary, const std::vector<ByteArray> &inputs,
-      BatchResult &batch_result);
+      std::string_view binary, const std::vector<ByteArray>& inputs,
+      BatchResult& batch_result, absl::Time deadline);
 
   // Constructs a string CENTIPEDE_RUNNER_FLAGS=":flag1:flag2:...",
   // where the flags are determined by `env` and also include `extra_flags`.
@@ -173,7 +173,7 @@ class CentipedeCallbacks {
   // Returns a CommandContext with matching `binary`. Creates one if needed.
   CommandContext& GetOrCreateCommandContextForBinary(std::string_view binary);
   // Runs a batch with the command `binary` and returns the exit code.
-  int RunBatchForBinary(std::string_view binary);
+  int RunBatchForBinary(std::string_view binary, absl::Time deadline);
 
   // Prints the execution log from the last executed binary.
   void PrintExecutionLog() const;

centipede/centipede_default_callbacks.cc

@@ -23,6 +23,7 @@
 
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
+#include "absl/time/time.h"
 #include "./centipede/centipede_callbacks.h"
 #include "./centipede/environment.h"
 #include "./centipede/mutation_input.h"
@@ -46,10 +47,11 @@ CentipedeDefaultCallbacks::CentipedeDefaultCallbacks(const Environment &env)
 }
 
 bool CentipedeDefaultCallbacks::Execute(std::string_view binary,
-                                        const std::vector<ByteArray> &inputs,
-                                        BatchResult &batch_result) {
-  return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result) ==
-         0;
+                                        const std::vector<ByteArray>& inputs,
+                                        BatchResult& batch_result,
+                                        absl::Time deadline) {
+  return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result,
+                                               deadline) == 0;
 }
 
 size_t CentipedeDefaultCallbacks::GetSeeds(size_t num_seeds,

centipede/centipede_default_callbacks.h

@@ -40,8 +40,8 @@ class CentipedeDefaultCallbacks : public CentipedeCallbacks {
   explicit CentipedeDefaultCallbacks(const Environment &env);
   size_t GetSeeds(size_t num_seeds, std::vector<ByteArray> &seeds) override;
   absl::StatusOr<std::string> GetSerializedTargetConfig() override;
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override;
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override;
   std::vector<ByteArray> Mutate(const std::vector<MutationInputRef> &inputs,
                                 size_t num_mutants) override;
 

centipede/centipede_interface.cc

@@ -306,7 +306,7 @@ absl::flat_hash_set<std::string> PruneOldCrashesAndGetRemainingCrashSignatures(
     FUZZTEST_CHECK_OK(
         RemoteFileGetContents(crashing_input_file, crashing_input));
     const bool is_reproducible = !scoped_callbacks.callbacks()->Execute(
-        env.binary, {crashing_input}, batch_result);
+        env.binary, {crashing_input}, batch_result, absl::InfiniteFuture());
     const bool is_duplicate =
         is_reproducible && !batch_result.IsSetupFailure() &&
         !remaining_crash_signatures.insert(batch_result.failure_signature())
@@ -692,6 +692,8 @@ int UpdateCorpusDatabaseForFuzzTests(
           << "Skip updating corpus database due to early stop requested.";
       continue;
     }
+    // The test time limit does not apply for the rest of the steps.
+    ClearEarlyStopRequestAndSetStopTime(/*stop_time=*/absl::InfiniteFuture());
 
     // TODO(xinhaoyuan): Have a separate flag to skip corpus updating instead
     // of checking whether workdir is specified or not.

centipede/centipede_test.cc

@@ -68,8 +68,8 @@ class CentipedeMock : public CentipedeCallbacks {
   // Doesn't execute anything
   // Sets `batch_result.results()` based on the values of `inputs`:
   // Collects various stats about the inputs, to be checked in tests.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.results().clear();
     // For every input, we create a 256-element array `counters`, where
     // i-th element is the number of bytes with the value 'i' in the input.
@@ -356,8 +356,8 @@ class MutateCallbacks : public CentipedeCallbacks {
  public:
   explicit MutateCallbacks(const Environment &env) : CentipedeCallbacks(env) {}
   // Will not be called.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     FUZZTEST_LOG(FATAL);
     return false;
   }
@@ -499,8 +499,8 @@ class MergeMock : public CentipedeCallbacks {
   // Doesn't execute anything.
   // All inputs are 1-byte long.
   // For an input {X}, the feature output is {X}.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.results().resize(inputs.size());
     for (size_t i = 0, n = inputs.size(); i < n; ++i) {
       FUZZTEST_CHECK_EQ(inputs[i].size(), 1);
@@ -588,10 +588,10 @@ class FunctionFilterMock : public CentipedeCallbacks {
   }
 
   // Executes the target in the normal way.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
-    return ExecuteCentipedeSancovBinaryWithShmem(env_.binary, inputs,
-                                                 batch_result) == EXIT_SUCCESS;
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
+    return ExecuteCentipedeSancovBinaryWithShmem(
+               env_.binary, inputs, batch_result, deadline) == EXIT_SUCCESS;
   }
 
   // Sets the inputs to one of 3 pre-defined values.
@@ -696,8 +696,8 @@ class ExtraBinariesMock : public CentipedeCallbacks {
 
   // Doesn't execute anything.
   // On certain combinations of {binary,input} returns false.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     bool res = true;
     for (const auto &input : inputs) {
       if (input.size() != 1) continue;
@@ -814,8 +814,8 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks {
   // Doesn't execute anything.
   // Crash when 0th char of input to binary b1 equals `crashing_input_idx_`, but
   // only on 1st exec.
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.ClearAndResize(inputs.size());
     bool res = true;
     if (!first_pass_) {
@@ -982,7 +982,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, CleansUpMetadataAfterStartup) {
 
   BatchResult batch_result;
   const std::vector<ByteArray> inputs = {{0}};
-  ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result));
+  ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result,
+                                /*deadline=*/absl::InfiniteFuture()));
   ASSERT_EQ(batch_result.results().size(), 1);
   bool found_startup_cmp_entry = false;
   batch_result.results()[0].metadata().ForEachCmpEntry(
@@ -999,8 +1000,8 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
                                           std::thread::id execute_thread_id)
       : CentipedeCallbacks(env), execute_thread_id_(execute_thread_id) {}
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     batch_result.ClearAndResize(inputs.size());
     thread_check_passed_ = thread_check_passed_ &&
                            std::this_thread::get_id() == execute_thread_id_;
@@ -1044,7 +1045,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, DetectsStackOverflow) {
   BatchResult batch_result;
   const std::vector<ByteArray> inputs = {ByteArray{'s', 't', 'k'}};
 
-  ASSERT_FALSE(callbacks.Execute(env.binary, inputs, batch_result));
+  ASSERT_FALSE(callbacks.Execute(env.binary, inputs, batch_result,
+                                 /*deadline=*/absl::InfiniteFuture()));
   EXPECT_THAT(batch_result.log(), HasSubstr("Stack limit exceeded"));
   EXPECT_EQ(batch_result.failure_description(), "stack-limit-exceeded");
 }
@@ -1053,8 +1055,8 @@ class SetupFailureCallbacks : public CentipedeCallbacks {
  public:
   using CentipedeCallbacks::CentipedeCallbacks;
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     ++execute_count_;
     batch_result.ClearAndResize(inputs.size());
     batch_result.exit_code() = EXIT_FAILURE;
@@ -1090,8 +1092,8 @@ class SkippedTestCallbacks : public CentipedeCallbacks {
  public:
   using CentipedeCallbacks::CentipedeCallbacks;
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time deadline) override {
     ++execute_count_;
     batch_result.ClearAndResize(inputs.size());
     batch_result.exit_code() = EXIT_FAILURE;
@@ -1128,8 +1130,8 @@ class IgnoredFailureCallbacks : public CentipedeCallbacks {
  public:
   using CentipedeCallbacks::CentipedeCallbacks;
 
-  bool Execute(std::string_view binary, const std::vector<ByteArray> &inputs,
-               BatchResult &batch_result) override {
+  bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs,
+               BatchResult& batch_result, absl::Time daedline) override {
     ++execute_count_;
     batch_result.ClearAndResize(inputs.size());
     batch_result.exit_code() = EXIT_FAILURE;
@@ -1224,7 +1226,24 @@ TEST_F(CentipedeWithTemporaryLocalDir, HangingFuzzTargetExitsAfterTimeout) {
   env.fork_server = false;
 
   // Test that the process does not get stuck and exits promptly.
-  EXPECT_FALSE(callbacks.Execute(env.binary, {{0}}, batch_result));
+  EXPECT_FALSE(callbacks.Execute(env.binary, {{0}}, batch_result,
+                                 /*deadline=*/absl::InfiniteFuture()));
+}
+
+TEST_F(CentipedeWithTemporaryLocalDir, RunnerExitsAfterFirstCustomFailure) {
+  Environment env;
+  env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target");
+  CentipedeDefaultCallbacks callbacks(env);
+  BatchResult result;
+  std::vector<ByteArray> inputs = {
+      {'c', 'u', 's', 't', 'o', 'm'},
+      {'c', 'u', 's', 't', 'o', 'm'},
+  };
+  EXPECT_FALSE(callbacks.Execute(env.binary, inputs, result,
+                                 /*deadline=*/absl::InfiniteFuture()));
+  EXPECT_THAT(result.failure_description(), HasSubstr("custom"));
+  EXPECT_THAT(result.log(), AllOf(HasSubstr("custom failure 0"),
+                                  Not(HasSubstr("custom failure 1"))));
 }
 
 }  // namespace

centipede/control_flow_test.cc

@@ -312,7 +312,7 @@ static void SymbolizeBinary(std::string_view test_dir,
       has_llvm_fuzzer_test_one_input = true;
       EXPECT_THAT(
           symbols.location(i),
-          testing::HasSubstr("centipede/testing/test_fuzz_target.cc:71"));
+          testing::HasSubstr("centipede/testing/test_fuzz_target.cc:73"));
     }
   }
   EXPECT_TRUE(has_llvm_fuzzer_test_one_input);

centipede/environment.cc

@@ -285,17 +285,6 @@ void Environment::UpdateWithTargetConfig(
   timeout_per_input = time_limit_per_input_sec;
   UpdateTimeoutPerBatchIfEqualTo(autocomputed_timeout_per_batch);
 
-  // Adjust `timeout_per_batch` to never exceed the test time limit.
-  if (const auto test_time_limit = config.GetTimeLimitPerTest();
-      test_time_limit < absl::InfiniteDuration()) {
-    const size_t test_time_limit_seconds =
-        convert_to_seconds(test_time_limit, "Test time limit");
-    timeout_per_batch =
-        timeout_per_batch == 0
-            ? test_time_limit_seconds
-            : std::min(timeout_per_batch, test_time_limit_seconds);
-  }
-
   // Convert bytes to MB by rounding up.
   constexpr auto bytes_to_mb = [](size_t bytes) {
     return bytes == 0 ? 0 : (bytes - 1) / 1024 / 1024 + 1;