From 218b4f2cec0bc6eb29a9c168e06de7201063401a Mon Sep 17 00:00:00 2001 From: Xinhao Yuan Date: Wed, 6 Aug 2025 15:48:06 -0700 Subject: [PATCH] Use the stop time to set batch execution deadlines. Do not report crashes on stop conditions. This allows the stop time to be enforced more precisely, and avoids reporting the potential crashes due to early stopping. After this we don't need to set batch timeout to the test time limit - it was a quick workaround for the same purpose. Change the e2e unit test time to 10s to reduce flakiness. PiperOrigin-RevId: 791875139 --- centipede/BUILD | 4 ++ centipede/centipede.cc | 17 +++-- centipede/centipede_callbacks.cc | 17 +++-- centipede/centipede_callbacks.h | 10 +-- centipede/centipede_default_callbacks.cc | 10 +-- centipede/centipede_default_callbacks.h | 4 +- centipede/centipede_interface.cc | 4 +- centipede/centipede_test.cc | 69 ++++++++++++------- centipede/control_flow_test.cc | 2 +- centipede/environment.cc | 11 --- centipede/environment_test.cc | 24 ------- centipede/minimize_crash.cc | 7 +- centipede/minimize_crash_test.cc | 5 +- centipede/runner.cc | 39 +++++------ centipede/runner.h | 2 + centipede/runner_interface.h | 3 + centipede/stop.cc | 2 + centipede/stop.h | 7 ++ centipede/test_coverage_util.cc | 4 +- centipede/test_coverage_util.h | 8 +-- centipede/testing/centipede_main_test.sh | 2 +- centipede/testing/test_fuzz_target.cc | 13 ++++ e2e_tests/corpus_database_test.cc | 1 + e2e_tests/functional_test.cc | 17 ++++- .../fuzz_tests_for_functional_testing.cc | 4 +- fuzztest/init_fuzztest.cc | 16 ++++- fuzztest/internal/centipede_adaptor.cc | 34 +++++---- 27 files changed, 199 insertions(+), 137 deletions(-) diff --git a/centipede/BUILD b/centipede/BUILD index 59982a3b5..da94bc019 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -389,6 +389,7 @@ cc_library( ":workdir", "@abseil-cpp//absl/base:core_headers", "@abseil-cpp//absl/synchronization", + "@abseil-cpp//absl/time", "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:hash", "@com_google_fuzztest//common:logging", @@ -909,6 +910,7 @@ cc_library( ":stop", "@abseil-cpp//absl/status", "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/time", "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:logging", ], @@ -1270,6 +1272,7 @@ cc_library( ":mutation_input", ":runner_result", ":util", + "@abseil-cpp//absl/time", "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:logging", ], @@ -1474,6 +1477,7 @@ cc_test( ":util", ":workdir", "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/time", "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:test_util", "@googletest//:gtest_main", diff --git a/centipede/centipede.cc b/centipede/centipede.cc index 97120d38c..517bfb481 100644 --- a/centipede/centipede.cc +++ b/centipede/centipede.cc @@ -362,9 +362,17 @@ bool Centipede::InputPassesFilter(const ByteArray &input) { bool Centipede::ExecuteAndReportCrash(std::string_view binary, const std::vector &input_vec, BatchResult &batch_result) { - bool success = user_callbacks_.Execute(binary, input_vec, batch_result); - if (!success) ReportCrash(binary, input_vec, batch_result); - return success || batch_result.IsIgnoredFailure(); + bool success = + user_callbacks_.Execute(binary, input_vec, batch_result, GetStopTime()); + if (success) return true; + if (ShouldStop()) { + FUZZTEST_LOG_FIRST_N(WARNING, 1) + << "Stop condition met - not reporting further crashes possibly " + "related to the stop condition."; + return true; + } + ReportCrash(binary, input_vec, batch_result); + return batch_result.IsIgnoredFailure(); } // *** Highly experimental and risky. May not scale well for large targets. *** @@ -986,7 +994,8 @@ void Centipede::ReportCrash(std::string_view binary, if (ShouldStop()) return; const auto &one_input = input_vec[input_idx]; BatchResult one_input_batch_result; - if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result)) { + if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result, + absl::InfiniteFuture())) { auto hash = Hash(one_input); auto crash_dir = wd_.CrashReproducerDirPaths().MyShard(); FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir)); diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc index 8110dce87..e15e369f1 100644 --- a/centipede/centipede_callbacks.cc +++ b/centipede/centipede_callbacks.cc @@ -472,14 +472,15 @@ void CentipedeCallbacks::CleanUpPersistentMode() { command_contexts_.end()); } -int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) { +int CentipedeCallbacks::RunBatchForBinary(std::string_view binary, + absl::Time deadline) { auto& command_context = GetOrCreateCommandContextForBinary(binary); auto& cmd = command_context.cmd; const absl::Duration amortized_timeout = env_.timeout_per_batch == 0 ? absl::InfiniteDuration() : absl::Seconds(env_.timeout_per_batch) + absl::Seconds(5); - const auto deadline = absl::Now() + amortized_timeout; + deadline = std::min(absl::Now() + amortized_timeout, deadline); int exit_code = EXIT_SUCCESS; const bool should_clean_up = [&] { if (!cmd.is_executing() && !cmd.ExecuteAsync()) { @@ -497,11 +498,12 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) { if (should_clean_up) { exit_code = [&] { if (!cmd.is_executing()) return EXIT_FAILURE; - FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution."; + FUZZTEST_LOG(ERROR) << "Cleaning up the batch execution with timeout " + << kCommandCleanupTimeout; cmd.RequestStop(); const auto ret = cmd.Wait(absl::Now() + kCommandCleanupTimeout); if (ret.has_value()) return *ret; - FUZZTEST_LOG(ERROR) << "Batch execution cleanup failed to end in 60s."; + FUZZTEST_LOG(ERROR) << "Failed to wait for the batch execution cleanup."; return EXIT_FAILURE; }(); command_contexts_.erase( @@ -515,7 +517,7 @@ int CentipedeCallbacks::RunBatchForBinary(std::string_view binary) { int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem( std::string_view binary, const std::vector& inputs, - BatchResult& batch_result) { + BatchResult& batch_result, absl::Time deadline) { auto start_time = absl::Now(); batch_result.ClearAndResize(inputs.size()); @@ -541,7 +543,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem( } // Run. - const int exit_code = RunBatchForBinary(binary); + const int exit_code = RunBatchForBinary(binary, deadline); inputs_blobseq_.ReleaseSharedMemory(); // Inputs are already consumed. // Get results. @@ -699,7 +701,8 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary( << VV(num_inputs_written) << VV(inputs.size()); // Execute. - const int exit_code = RunBatchForBinary(binary); + const int exit_code = + RunBatchForBinary(binary, /*deadline=*/absl::InfiniteFuture()); inputs_blobseq_.ReleaseSharedMemory(); // Inputs are already consumed. if (exit_code != EXIT_SUCCESS) { diff --git a/centipede/centipede_callbacks.h b/centipede/centipede_callbacks.h index 2976cd1d4..ea85ea63e 100644 --- a/centipede/centipede_callbacks.h +++ b/centipede/centipede_callbacks.h @@ -65,8 +65,8 @@ class CentipedeCallbacks { // Post-condition: // `batch_result` has results for every `input`, even on failure. virtual bool Execute(std::string_view binary, - const std::vector &inputs, - BatchResult &batch_result) = 0; + const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) = 0; // Takes non-empty `inputs` and returns at most `num_mutants` mutated inputs. virtual std::vector Mutate( @@ -103,8 +103,8 @@ class CentipedeCallbacks { // Same as ExecuteCentipedeSancovBinary, but uses shared memory. // Much faster for fast targets since it uses fewer system calls. int ExecuteCentipedeSancovBinaryWithShmem( - std::string_view binary, const std::vector &inputs, - BatchResult &batch_result); + std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline); // Constructs a string CENTIPEDE_RUNNER_FLAGS=":flag1:flag2:...", // where the flags are determined by `env` and also include `extra_flags`. @@ -173,7 +173,7 @@ class CentipedeCallbacks { // Returns a CommandContext with matching `binary`. Creates one if needed. CommandContext& GetOrCreateCommandContextForBinary(std::string_view binary); // Runs a batch with the command `binary` and returns the exit code. - int RunBatchForBinary(std::string_view binary); + int RunBatchForBinary(std::string_view binary, absl::Time deadline); // Prints the execution log from the last executed binary. void PrintExecutionLog() const; diff --git a/centipede/centipede_default_callbacks.cc b/centipede/centipede_default_callbacks.cc index ee54c2aea..fa0cc3f1c 100644 --- a/centipede/centipede_default_callbacks.cc +++ b/centipede/centipede_default_callbacks.cc @@ -23,6 +23,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/time/time.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/environment.h" #include "./centipede/mutation_input.h" @@ -46,10 +47,11 @@ CentipedeDefaultCallbacks::CentipedeDefaultCallbacks(const Environment &env) } bool CentipedeDefaultCallbacks::Execute(std::string_view binary, - const std::vector &inputs, - BatchResult &batch_result) { - return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result) == - 0; + const std::vector& inputs, + BatchResult& batch_result, + absl::Time deadline) { + return ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result, + deadline) == 0; } size_t CentipedeDefaultCallbacks::GetSeeds(size_t num_seeds, diff --git a/centipede/centipede_default_callbacks.h b/centipede/centipede_default_callbacks.h index 0b7856261..a3031e0d2 100644 --- a/centipede/centipede_default_callbacks.h +++ b/centipede/centipede_default_callbacks.h @@ -40,8 +40,8 @@ class CentipedeDefaultCallbacks : public CentipedeCallbacks { explicit CentipedeDefaultCallbacks(const Environment &env); size_t GetSeeds(size_t num_seeds, std::vector &seeds) override; absl::StatusOr GetSerializedTargetConfig() override; - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override; + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override; std::vector Mutate(const std::vector &inputs, size_t num_mutants) override; diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc index 321f89a54..43cfb2a16 100644 --- a/centipede/centipede_interface.cc +++ b/centipede/centipede_interface.cc @@ -306,7 +306,7 @@ absl::flat_hash_set PruneOldCrashesAndGetRemainingCrashSignatures( FUZZTEST_CHECK_OK( RemoteFileGetContents(crashing_input_file, crashing_input)); const bool is_reproducible = !scoped_callbacks.callbacks()->Execute( - env.binary, {crashing_input}, batch_result); + env.binary, {crashing_input}, batch_result, absl::InfiniteFuture()); const bool is_duplicate = is_reproducible && !batch_result.IsSetupFailure() && !remaining_crash_signatures.insert(batch_result.failure_signature()) @@ -692,6 +692,8 @@ int UpdateCorpusDatabaseForFuzzTests( << "Skip updating corpus database due to early stop requested."; continue; } + // The test time limit does not apply for the rest of the steps. + ClearEarlyStopRequestAndSetStopTime(/*stop_time=*/absl::InfiniteFuture()); // TODO(xinhaoyuan): Have a separate flag to skip corpus updating instead // of checking whether workdir is specified or not. diff --git a/centipede/centipede_test.cc b/centipede/centipede_test.cc index 39ce84473..7cf27a289 100644 --- a/centipede/centipede_test.cc +++ b/centipede/centipede_test.cc @@ -68,8 +68,8 @@ class CentipedeMock : public CentipedeCallbacks { // Doesn't execute anything // Sets `batch_result.results()` based on the values of `inputs`: // Collects various stats about the inputs, to be checked in tests. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { batch_result.results().clear(); // For every input, we create a 256-element array `counters`, where // i-th element is the number of bytes with the value 'i' in the input. @@ -356,8 +356,8 @@ class MutateCallbacks : public CentipedeCallbacks { public: explicit MutateCallbacks(const Environment &env) : CentipedeCallbacks(env) {} // Will not be called. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { FUZZTEST_LOG(FATAL); return false; } @@ -499,8 +499,8 @@ class MergeMock : public CentipedeCallbacks { // Doesn't execute anything. // All inputs are 1-byte long. // For an input {X}, the feature output is {X}. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { batch_result.results().resize(inputs.size()); for (size_t i = 0, n = inputs.size(); i < n; ++i) { FUZZTEST_CHECK_EQ(inputs[i].size(), 1); @@ -588,10 +588,10 @@ class FunctionFilterMock : public CentipedeCallbacks { } // Executes the target in the normal way. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { - return ExecuteCentipedeSancovBinaryWithShmem(env_.binary, inputs, - batch_result) == EXIT_SUCCESS; + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { + return ExecuteCentipedeSancovBinaryWithShmem( + env_.binary, inputs, batch_result, deadline) == EXIT_SUCCESS; } // Sets the inputs to one of 3 pre-defined values. @@ -696,8 +696,8 @@ class ExtraBinariesMock : public CentipedeCallbacks { // Doesn't execute anything. // On certain combinations of {binary,input} returns false. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { bool res = true; for (const auto &input : inputs) { if (input.size() != 1) continue; @@ -814,8 +814,8 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks { // Doesn't execute anything. // Crash when 0th char of input to binary b1 equals `crashing_input_idx_`, but // only on 1st exec. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { batch_result.ClearAndResize(inputs.size()); bool res = true; if (!first_pass_) { @@ -982,7 +982,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, CleansUpMetadataAfterStartup) { BatchResult batch_result; const std::vector inputs = {{0}}; - ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result)); + ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result, + /*deadline=*/absl::InfiniteFuture())); ASSERT_EQ(batch_result.results().size(), 1); bool found_startup_cmp_entry = false; batch_result.results()[0].metadata().ForEachCmpEntry( @@ -999,8 +1000,8 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks { std::thread::id execute_thread_id) : CentipedeCallbacks(env), execute_thread_id_(execute_thread_id) {} - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { batch_result.ClearAndResize(inputs.size()); thread_check_passed_ = thread_check_passed_ && std::this_thread::get_id() == execute_thread_id_; @@ -1044,7 +1045,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, DetectsStackOverflow) { BatchResult batch_result; const std::vector inputs = {ByteArray{'s', 't', 'k'}}; - ASSERT_FALSE(callbacks.Execute(env.binary, inputs, batch_result)); + ASSERT_FALSE(callbacks.Execute(env.binary, inputs, batch_result, + /*deadline=*/absl::InfiniteFuture())); EXPECT_THAT(batch_result.log(), HasSubstr("Stack limit exceeded")); EXPECT_EQ(batch_result.failure_description(), "stack-limit-exceeded"); } @@ -1053,8 +1055,8 @@ class SetupFailureCallbacks : public CentipedeCallbacks { public: using CentipedeCallbacks::CentipedeCallbacks; - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { ++execute_count_; batch_result.ClearAndResize(inputs.size()); batch_result.exit_code() = EXIT_FAILURE; @@ -1090,8 +1092,8 @@ class SkippedTestCallbacks : public CentipedeCallbacks { public: using CentipedeCallbacks::CentipedeCallbacks; - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { ++execute_count_; batch_result.ClearAndResize(inputs.size()); batch_result.exit_code() = EXIT_FAILURE; @@ -1128,8 +1130,8 @@ class IgnoredFailureCallbacks : public CentipedeCallbacks { public: using CentipedeCallbacks::CentipedeCallbacks; - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time daedline) override { ++execute_count_; batch_result.ClearAndResize(inputs.size()); batch_result.exit_code() = EXIT_FAILURE; @@ -1224,7 +1226,24 @@ TEST_F(CentipedeWithTemporaryLocalDir, HangingFuzzTargetExitsAfterTimeout) { env.fork_server = false; // Test that the process does not get stuck and exits promptly. - EXPECT_FALSE(callbacks.Execute(env.binary, {{0}}, batch_result)); + EXPECT_FALSE(callbacks.Execute(env.binary, {{0}}, batch_result, + /*deadline=*/absl::InfiniteFuture())); +} + +TEST_F(CentipedeWithTemporaryLocalDir, RunnerExitsAfterFirstCustomFailure) { + Environment env; + env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + CentipedeDefaultCallbacks callbacks(env); + BatchResult result; + std::vector inputs = { + {'c', 'u', 's', 't', 'o', 'm'}, + {'c', 'u', 's', 't', 'o', 'm'}, + }; + EXPECT_FALSE(callbacks.Execute(env.binary, inputs, result, + /*deadline=*/absl::InfiniteFuture())); + EXPECT_THAT(result.failure_description(), HasSubstr("custom")); + EXPECT_THAT(result.log(), AllOf(HasSubstr("custom failure 0"), + Not(HasSubstr("custom failure 1")))); } } // namespace diff --git a/centipede/control_flow_test.cc b/centipede/control_flow_test.cc index 2d21d210c..5327b8567 100644 --- a/centipede/control_flow_test.cc +++ b/centipede/control_flow_test.cc @@ -312,7 +312,7 @@ static void SymbolizeBinary(std::string_view test_dir, has_llvm_fuzzer_test_one_input = true; EXPECT_THAT( symbols.location(i), - testing::HasSubstr("centipede/testing/test_fuzz_target.cc:71")); + testing::HasSubstr("centipede/testing/test_fuzz_target.cc:73")); } } EXPECT_TRUE(has_llvm_fuzzer_test_one_input); diff --git a/centipede/environment.cc b/centipede/environment.cc index 4cd2be62a..bed792209 100644 --- a/centipede/environment.cc +++ b/centipede/environment.cc @@ -285,17 +285,6 @@ void Environment::UpdateWithTargetConfig( timeout_per_input = time_limit_per_input_sec; UpdateTimeoutPerBatchIfEqualTo(autocomputed_timeout_per_batch); - // Adjust `timeout_per_batch` to never exceed the test time limit. - if (const auto test_time_limit = config.GetTimeLimitPerTest(); - test_time_limit < absl::InfiniteDuration()) { - const size_t test_time_limit_seconds = - convert_to_seconds(test_time_limit, "Test time limit"); - timeout_per_batch = - timeout_per_batch == 0 - ? test_time_limit_seconds - : std::min(timeout_per_batch, test_time_limit_seconds); - } - // Convert bytes to MB by rounding up. constexpr auto bytes_to_mb = [](size_t bytes) { return bytes == 0 ? 0 : (bytes - 1) / 1024 / 1024 + 1; diff --git a/centipede/environment_test.cc b/centipede/environment_test.cc index a906b998d..977619270 100644 --- a/centipede/environment_test.cc +++ b/centipede/environment_test.cc @@ -147,30 +147,6 @@ TEST(Environment, EXPECT_EQ(env.timeout_per_batch, 0); } -TEST(Environment, UpdatesTimeoutPerBatchFromTargetConfigTimeLimit) { - Environment env; - fuzztest::internal::Configuration config; - config.time_limit = absl::Seconds(123); - config.time_budget_type = fuzztest::internal::TimeBudgetType::kPerTest; - FUZZTEST_CHECK(config.GetTimeLimitPerTest() == absl::Seconds(123)); - env.UpdateWithTargetConfig(config); - EXPECT_EQ(env.timeout_per_batch, 123) - << "`timeout_per_batch` should be set to the test time limit when it was " - "previously unset"; - - env.timeout_per_batch = 456; - env.UpdateWithTargetConfig(config); - EXPECT_EQ(env.timeout_per_batch, 123) - << "`timeout_per_batch` should be set to test time limit when it is " - "shorter than the previous value"; - - env.timeout_per_batch = 56; - env.UpdateWithTargetConfig(config); - EXPECT_EQ(env.timeout_per_batch, 56) - << "`timeout_per_batch` should not be updated with the test time limit " - "when it is longer than the previous value"; -} - TEST(Environment, UpdatesRssLimitMbFromTargetConfigRssLimit) { Environment env; env.rss_limit_mb = Environment::Default().rss_limit_mb; diff --git a/centipede/minimize_crash.cc b/centipede/minimize_crash.cc index 66d417a20..10eaf2261 100644 --- a/centipede/minimize_crash.cc +++ b/centipede/minimize_crash.cc @@ -24,6 +24,7 @@ #include "absl/base/thread_annotations.h" #include "absl/synchronization/mutex.h" +#include "absl/time/time.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/environment.h" #include "./centipede/mutation_input.h" @@ -122,7 +123,8 @@ static void MinimizeCrash(const Environment &env, } // Execute all mutants. If a new crasher is found, add it to `queue`. - if (!callbacks->Execute(env.binary, smaller_mutants, batch_result)) { + if (!callbacks->Execute(env.binary, smaller_mutants, batch_result, + absl::InfiniteFuture())) { size_t crash_inputs_idx = batch_result.num_outputs_read(); FUZZTEST_CHECK_LT(crash_inputs_idx, smaller_mutants.size()); const auto &new_crasher = smaller_mutants[crash_inputs_idx]; @@ -142,7 +144,8 @@ int MinimizeCrash(ByteSpan crashy_input, const Environment &env, BatchResult batch_result; ByteArray original_crashy_input(crashy_input.begin(), crashy_input.end()); - if (callbacks->Execute(env.binary, {original_crashy_input}, batch_result)) { + if (callbacks->Execute(env.binary, {original_crashy_input}, batch_result, + absl::InfiniteFuture())) { FUZZTEST_LOG(INFO) << "The original crashy input did not crash; exiting"; return EXIT_FAILURE; } diff --git a/centipede/minimize_crash_test.cc b/centipede/minimize_crash_test.cc index 30a145a9e..d7d5ec296 100644 --- a/centipede/minimize_crash_test.cc +++ b/centipede/minimize_crash_test.cc @@ -23,6 +23,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/base/nullability.h" +#include "absl/time/time.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/environment.h" #include "./centipede/runner_result.h" @@ -40,8 +41,8 @@ class MinimizerMock : public CentipedeCallbacks { MinimizerMock(const Environment &env) : CentipedeCallbacks(env) {} // Runs FuzzMe() on every input, imitates failure if FuzzMe() returns true. - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { batch_result.ClearAndResize(inputs.size()); for (auto &input : inputs) { if (FuzzMe(input)) { diff --git a/centipede/runner.cc b/centipede/runner.cc index 2e73d8564..e5808e1bd 100644 --- a/centipede/runner.cc +++ b/centipede/runner.cc @@ -494,12 +494,14 @@ static int ExecuteInputsFromShmem(BlobSequence &inputs_blobseq, RunOneInput(data.data(), data.size(), callbacks); + if (state->has_failure_description.load()) break; + if (!FinishSendingOutputsToEngine(outputs_blobseq)) break; } CentipedeEndExecutionBatch(); - return EXIT_SUCCESS; + return state->has_failure_description.load() ? EXIT_FAILURE : EXIT_SUCCESS; } // Dumps seed inputs to `output_dir`. Also see `GetSeedsViaExternalBinary()`. @@ -1017,23 +1019,20 @@ extern "C" void CentipedeSetExecutionResult(const uint8_t *data, size_t size) { extern "C" void CentipedeSetFailureDescription(const char *description) { using fuzztest::internal::state; if (state->failure_description_path == nullptr) return; - // Make sure that the write is atomic and only happens once. - [[maybe_unused]] static int write_once = [=] { - FILE* f = fopen(state->failure_description_path, "w"); - if (f == nullptr) { - perror("FAILURE: fopen()"); - return 0; - } - const auto len = strlen(description); - if (fwrite(description, 1, len, f) != len) { - perror("FAILURE: fwrite()"); - } - if (fflush(f) != 0) { - perror("FAILURE: fflush()"); - } - if (fclose(f) != 0) { - perror("FAILURE: fclose()"); - } - return 0; - }(); + if (state->has_failure_description.exchange(true)) return; + FILE* f = fopen(state->failure_description_path, "w"); + if (f == nullptr) { + perror("FAILURE: fopen()"); + return; + } + const auto len = strlen(description); + if (fwrite(description, 1, len, f) != len) { + perror("FAILURE: fwrite()"); + } + if (fflush(f) != 0) { + perror("FAILURE: fflush()"); + } + if (fclose(f) != 0) { + perror("FAILURE: fclose()"); + } } diff --git a/centipede/runner.h b/centipede/runner.h index 7eac41606..fbe094166 100644 --- a/centipede/runner.h +++ b/centipede/runner.h @@ -80,6 +80,8 @@ struct GlobalRunnerState { const char *failure_description_path = flag_helper.GetStringFlag(":failure_description_path="); + std::atomic has_failure_description; + const char* persistent_mode_socket_path = flag_helper.GetStringFlag(":persistent_mode_socket="); int persistent_mode_socket = 0; diff --git a/centipede/runner_interface.h b/centipede/runner_interface.h index 96f1a8de5..eb004d5aa 100644 --- a/centipede/runner_interface.h +++ b/centipede/runner_interface.h @@ -132,6 +132,9 @@ extern "C" void CentipedeSetExecutionResult(const uint8_t *data, size_t size); // Set the failure description for the runner to propagate further. Only the // description from the first call will be used. +// +// If used during executing batch inputs, the rest of the inputs would be +// skipped and the batch would be considered as failed. extern "C" void CentipedeSetFailureDescription(const char *description); namespace fuzztest::internal { diff --git a/centipede/stop.cc b/centipede/stop.cc index 20f85bd35..6a76c17a2 100644 --- a/centipede/stop.cc +++ b/centipede/stop.cc @@ -46,6 +46,8 @@ void RequestEarlyStop(int exit_code) { early_stop.store({exit_code, true}, std::memory_order_release); } +absl::Time GetStopTime() { return stop_time; } + bool ShouldStop() { return EarlyStopRequested() || stop_time < absl::Now(); } int ExitCode() { return early_stop.load(std::memory_order_acquire).exit_code; } diff --git a/centipede/stop.h b/centipede/stop.h index f4244f8da..d3e255ddb 100644 --- a/centipede/stop.h +++ b/centipede/stop.h @@ -46,6 +46,13 @@ bool EarlyStopRequested(); // ENSURES: Thread-safe. bool ShouldStop(); +// Returns the stop time set from the recent +// `ClearEarlyStopRequestAndSetStopTime()`, or `absl::InfiniteFuture()` it was +// not set. +// +// ENSURES: Thread-safe. +absl::Time GetStopTime(); + // Returns the value most recently passed to `RequestEarlyStop()` or 0 if // `RequestEarlyStop()` was not called since the most recent call to // `ClearEarlyStopRequestAndSetStopTime()` (if any). diff --git a/centipede/test_coverage_util.cc b/centipede/test_coverage_util.cc index 2602a58c9..c14fda3df 100644 --- a/centipede/test_coverage_util.cc +++ b/centipede/test_coverage_util.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/time/time.h" #include "./centipede/corpus.h" #include "./centipede/environment.h" #include "./centipede/feature.h" @@ -41,7 +42,8 @@ std::vector RunInputsAndCollectCorpusRecords( } BatchResult batch_result; // Run. - CBs.Execute(env.binary, byte_array_inputs, batch_result); + CBs.Execute(env.binary, byte_array_inputs, batch_result, + /*deadline=*/absl::InfiniteFuture()); // Repackage execution results into a vector of CorpusRecords. std::vector corpus_records; diff --git a/centipede/test_coverage_util.h b/centipede/test_coverage_util.h index c06a9bf7a..ad8800b32 100644 --- a/centipede/test_coverage_util.h +++ b/centipede/test_coverage_util.h @@ -43,10 +43,10 @@ std::vector RunInputsAndCollectCoverage( class TestCallbacks : public CentipedeCallbacks { public: explicit TestCallbacks(const Environment &env) : CentipedeCallbacks(env) {} - bool Execute(std::string_view binary, const std::vector &inputs, - BatchResult &batch_result) override { - int result = - ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, batch_result); + bool Execute(std::string_view binary, const std::vector& inputs, + BatchResult& batch_result, absl::Time deadline) override { + int result = ExecuteCentipedeSancovBinaryWithShmem(binary, inputs, + batch_result, deadline); FUZZTEST_CHECK_EQ(EXIT_SUCCESS, result); return true; } diff --git a/centipede/testing/centipede_main_test.sh b/centipede/testing/centipede_main_test.sh index 3cdbc65ec..e8fc45dcd 100755 --- a/centipede/testing/centipede_main_test.sh +++ b/centipede/testing/centipede_main_test.sh @@ -71,7 +71,7 @@ test_debug_symbols() { --symbolizer_path="${LLVM_SYMBOLIZER}" | tee "${LOG}" fuzztest::internal::assert_regex_in_file 'Custom mutator detected; will use it' "${LOG}" # Note: the test assumes LLVMFuzzerTestOneInput is defined on a specific line. - fuzztest::internal::assert_regex_in_file "FUNC: LLVMFuzzerTestOneInput .*testing/test_fuzz_target.cc:71" "${LOG}" + fuzztest::internal::assert_regex_in_file "FUNC: LLVMFuzzerTestOneInput .*testing/test_fuzz_target.cc:73" "${LOG}" fuzztest::internal::assert_regex_in_file "EDGE: LLVMFuzzerTestOneInput .*testing/test_fuzz_target.cc" "${LOG}" echo "============ ${FUNC}: add func1/func2-A inputs to the corpus." diff --git a/centipede/testing/test_fuzz_target.cc b/centipede/testing/test_fuzz_target.cc index 32e07ebd2..56e7e1bd8 100644 --- a/centipede/testing/test_fuzz_target.cc +++ b/centipede/testing/test_fuzz_target.cc @@ -24,6 +24,8 @@ #include +#include "./centipede/runner_interface.h" + // Function with a single coverage edge. Used by coverage_test.cc. __attribute__((noinline)) extern "C" void SingleEdgeFunc() { [[maybe_unused]] static volatile int sink; @@ -243,6 +245,17 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { funcs[idx1](); funcs[idx2](); } + + // "custom" for failure with custom description + if (size == 6 && data[0] == 'c' && data[1] == 'u' && data[2] == 's' && + data[3] == 't' && data[4] == 'o' && data[5] == 'm') { + CentipedeSetFailureDescription("INPUT FAILURE: custom"); + static int count = 0; + printf("custom failure %d\n", count); + ++count; + fflush(stdout); + } + IndirectCallFunc(data[0]); return 0; } diff --git a/e2e_tests/corpus_database_test.cc b/e2e_tests/corpus_database_test.cc index c41f7d419..9e99b019f 100644 --- a/e2e_tests/corpus_database_test.cc +++ b/e2e_tests/corpus_database_test.cc @@ -122,6 +122,7 @@ class UpdateCorpusDatabaseTest // Dumping stack trace in gtest would slow down the execution, causing // test flakiness. options.flags[GTEST_FLAG_PREFIX_ "stack_trace_depth"] = "0"; + options.flags["symbolize_stacktrace"] = "0"; switch (GetParam()) { case ExecutionModelParam::kTestBinary: return RunBinary(binary_path, options); diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc index 49eec3782..622cd3e53 100644 --- a/e2e_tests/functional_test.cc +++ b/e2e_tests/functional_test.cc @@ -123,6 +123,10 @@ class UnitTestModeTest : public ::testing::Test { const absl::flat_hash_map& env = {}, absl::flat_hash_map fuzzer_flags = {}) { fuzzer_flags["print_subprocess_log"] = "true"; + fuzzer_flags["unguided"] = "true"; + if (!fuzzer_flags.contains("fuzz_for")) { + fuzzer_flags["fuzz_for"] = "10s"; + } RunOptions run_options; run_options.flags = { {GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}, @@ -142,6 +146,7 @@ TEST_F(UnitTestModeTest, InvalidSeedsAreSkippedAndReported) { auto [status, std_out, std_err] = Run(/*test_filter=*/"*", /*target_binary=*/"testdata/fuzz_tests_with_invalid_seeds"); + SCOPED_TRACE(std_err); EXPECT_THAT_LOG(std_err, HasSubstr("[!] Skipping WithSeeds() value in")); EXPECT_THAT_LOG(std_err, HasSubstr("Could not turn value into corpus type:\n{17}")); @@ -265,6 +270,7 @@ TEST_F(UnitTestModeTest, GlobalEnvironmentGoesThroughCompleteLifecycle) { TEST_F(UnitTestModeTest, FixtureGoesThroughCompleteLifecycle) { auto [status, std_out, std_err] = Run("FixtureTest.NeverFails"); + SCOPED_TRACE(std_err); EXPECT_GT(CountSubstrs(std_err, "<>"), 0); EXPECT_EQ(CountSubstrs(std_err, "<>"), CountSubstrs(std_err, "<>")); @@ -288,6 +294,7 @@ TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledInBalance) { auto [status, std_out, std_err] = Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount:" "CallCountPerFuzzTest.NeverFails"); + SCOPED_TRACE(std_err); EXPECT_GT(CountSubstrs(std_err, "<>"), 0); EXPECT_EQ( @@ -612,6 +619,7 @@ TEST_F(UnitTestModeTest, StackLimitWorks) { auto [status, std_out, std_err] = Run("MySuite.DataDependentStackOverflow", kDefaultTargetBinary, /*env=*/{}, /*fuzzer_flags=*/{{"stack_limit_kb", "1000"}}); + SCOPED_TRACE(std_err); EXPECT_THAT_LOG(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 1024000); ExpectTargetAbort(status, std_err); @@ -620,7 +628,7 @@ TEST_F(UnitTestModeTest, StackLimitWorks) { TEST_F(UnitTestModeTest, RssLimitFlagWorks) { auto [status, std_out, std_err] = Run("MySuite.LargeHeapAllocation", kDefaultTargetBinary, - /*env=*/{}, /*fuzzer_flags=*/{{"rss_limit_mb", "1024"}}); + /*env=*/{}, /*fuzzer_flags=*/{{"rss_limit_mb", "2048"}}); EXPECT_THAT_LOG(std_err, HasSubstr("argument 0: ")); EXPECT_THAT_LOG(std_err, ContainsRegex(absl::StrCat("RSS limit exceeded"))); ExpectTargetAbort(status, std_err); @@ -631,6 +639,7 @@ TEST_F(UnitTestModeTest, TimeLimitFlagWorks) { Run("MySuite.Sleep", kDefaultTargetBinary, /*env=*/{}, /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}}); + SCOPED_TRACE(std_err); EXPECT_THAT_LOG(std_err, HasSubstr("argument 0: ")); EXPECT_THAT_LOG(std_err, ContainsRegex("Per-input timeout exceeded")); ExpectTargetAbort(status, std_err); @@ -1176,9 +1185,10 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, TimeLimitFlagWorks) { // to restrict the filter to only fuzz tests. TEST_F(FuzzingModeCommandLineInterfaceTest, RunsOnlyFuzzTests) { auto [status, std_out, std_err] = - RunWith({{"fuzz_for", "1ns"}}, /*env=*/{}, /*timeout=*/absl::Seconds(10), + RunWith({{"fuzz_for", "1s"}}, /*env=*/{}, /*timeout=*/absl::Seconds(10), "testdata/unit_test_and_fuzz_tests"); + SCOPED_TRACE(std_err); EXPECT_THAT_LOG(std_out, Not(HasSubstr("[ RUN ] UnitTest.AlwaysPasses"))); EXPECT_THAT_LOG(std_out, HasSubstr("[ RUN ] FuzzTest.AlwaysPasses")); @@ -1190,7 +1200,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, RunsOnlyFuzzTests) { TEST_F(FuzzingModeCommandLineInterfaceTest, AllowsSpecifyingFilterWithFuzzForDuration) { auto [status, std_out, std_err] = - RunWith({{"fuzz_for", "1ns"}}, /*env=*/{}, /*timeout=*/absl::Seconds(10), + RunWith({{"fuzz_for", "1s"}}, /*env=*/{}, /*timeout=*/absl::Seconds(10), "testdata/unit_test_and_fuzz_tests", {{GTEST_FLAG_PREFIX_ "filter", "UnitTest.AlwaysPasses:FuzzTest.AlwaysPasses"}}); @@ -1239,6 +1249,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, UsesCentipedeBinaryWhenEnvIsSet) { std_err, HasSubstr("Starting the update of the corpus database for fuzz tests")); EXPECT_THAT_LOG(std_err, HasSubstr("FuzzTest.AlwaysPasses")); + SCOPED_TRACE(std_err); EXPECT_THAT(status, Eq(ExitCode(0))); } diff --git a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc index 26368c8a9..8403f8b8a 100644 --- a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc +++ b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc @@ -865,8 +865,8 @@ void LargeHeapAllocation(size_t allocation_size) { } FUZZ_TEST(MySuite, LargeHeapAllocation) .WithDomains(Just( - // 1 GiB - 1ULL << 30)); + // 2 GiB + 1ULL << 31)); // A fuzz test that is expected to accept and skip some inputs before hitting // the crash. diff --git a/fuzztest/init_fuzztest.cc b/fuzztest/init_fuzztest.cc index 5325456db..ea0b0f5c3 100644 --- a/fuzztest/init_fuzztest.cc +++ b/fuzztest/init_fuzztest.cc @@ -170,6 +170,13 @@ FUZZTEST_DEFINE_FLAG( bool, print_subprocess_log, false, "If set, print the log of the subprocesses spawned by FuzzTest."); +FUZZTEST_DEFINE_FLAG(bool, unguided, false, + "If used together with --" FUZZTEST_FLAG_PREFIX + "fuzz or --" FUZZTEST_FLAG_PREFIX + "fuzz_for, carries out fuzzing without coverage guidance. " + "When used with --" FUZZTEST_FLAG_PREFIX + "fuzz_for, regular tests also run by default."); + // Internal flags - not part of the user interface. // // These flags are meant to be set only by the parent controller process for its @@ -442,10 +449,13 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { GTEST_FLAG_SET(filter, filter); } } - const RunMode run_mode = - fuzzing_time_limit.has_value() ? RunMode::kFuzz : RunMode::kUnitTest; // TODO(b/307513669): Use the Configuration class instead of Runtime. - runtime.SetRunMode(run_mode); + if (!absl::GetFlag(FUZZTEST_FLAG(unguided)) && + fuzzing_time_limit.has_value()) { + runtime.SetRunMode(RunMode::kFuzz); + } else { + runtime.SetRunMode(RunMode::kUnitTest); + } } void ParseAbslFlags(int argc, char** argv) { diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc index 781b8b0df..edd699dec 100644 --- a/fuzztest/internal/centipede_adaptor.cc +++ b/fuzztest/internal/centipede_adaptor.cc @@ -463,25 +463,27 @@ class CentipedeAdaptorRunnerCallbacks prng_(GetRandomSeed()) {} bool Execute(fuzztest::internal::ByteSpan input) override { - [[maybe_unused]] static bool check_if_not_skipped_on_setup = [&] { - if (runtime_.skipping_requested()) { - absl::FPrintF(GetStderr(), - "[.] Skipping %s per request from the test setup.\n", - fuzzer_impl_.test_.full_name()); - CentipedeSetFailureDescription("SKIPPED TEST: Requested from setup"); - // It has to use _Exit(1) to avoid trigger the reporting of regular - // setup failure while let Centipede be aware of this. Note that this - // skips the fixture teardown. - std::_Exit(1); - } - return true; - }(); // Disable tracing until running the property function in // `CentipedeFxitureDriver::RunFuzzTestIteration()` const int old_traced = CentipedeSetCurrentThreadTraced(/*traced=*/0); absl::Cleanup tracing_restorer = [old_traced] { CentipedeSetCurrentThreadTraced(old_traced); }; + static const bool skipped_on_setup = runtime_.skipping_requested(); + if (skipped_on_setup) { + absl::FPrintF(GetStderr(), + "[.] Skipping %s per request from the test setup.\n", + fuzzer_impl_.test_.full_name()); + CentipedeSetFailureDescription("SKIPPED TEST: Requested from setup"); + return true; + } + if (runtime_.termination_requested()) { + absl::FPrintF(GetStderr(), + "[.] Termination requested - exiting without executing " + "further inputs.\n"); + CentipedeSetFailureDescription("IGNORED FAILURE: Termination requested"); + return false; + } // We should avoid doing anything other than executing the input here so // that we don't affect the execution time. auto parsed_input = @@ -636,7 +638,8 @@ class CentipedeFixtureDriver : public UntypedFixtureDriver { if (!runner_mode) CentipedePrepareProcessing(); std::move(run_iteration_once)(); }); - if (runtime_.skipping_requested()) { + if (runtime_.skipping_requested() || + runtime_.run_mode() == RunMode::kUnitTest) { CentipedeSetExecutionResult(nullptr, 0); } CentipedeFinalizeProcessing(); @@ -1021,7 +1024,8 @@ class CentipedeCallbacksForRunnerFlagsExtraction bool Execute(std::string_view binary, const std::vector& inputs, - fuzztest::internal::BatchResult& batch_result) override { + fuzztest::internal::BatchResult& batch_result, + absl::Time deadline) override { return false; }