Add AFL-style metrics(pending,pend_fav, own_finds,imported) (#1351)

* add the metrics(pending,own_finds,imported)

* add the pend_fav metrics

* push

* Add the feature that AFLStats is computed and reported in AFLStatsStage

* fix some cicd errors

* AFLStats migrates to stage/stats.rs

* fix the cicd error

* fix some bugs and resolve the conflicts

* fix some typos

---------

Co-authored-by: toseven <[email protected]>
Co-authored-by: toka <[email protected]>
Co-authored-by: Dominik Maier <[email protected]>

Author: 4 people

libafl/src/fuzzer/mod.rs

@@ -24,8 +24,8 @@ use crate::{
     stages::StagesTuple,
     start_timer,
     state::{
-        HasClientPerfMonitor, HasCorpus, HasExecutions, HasLastReportTime, HasMetadata,
-        HasSolutions, UsesState,
+        HasClientPerfMonitor, HasCorpus, HasExecutions, HasImported, HasLastReportTime,
+        HasMetadata, HasSolutions, UsesState,
     },
     Error,
 };
@@ -333,7 +333,8 @@ where
     F: Feedback<CS::State>,
     OF: Feedback<CS::State>,
     OT: ObserversTuple<CS::State> + Serialize + DeserializeOwned,
-    CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasCorpus,
+    CS::State:
+        HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasCorpus + HasImported,
 {
     /// Evaluate if a set of observation channels has an interesting state
     fn process_execution<EM>(
@@ -415,6 +416,9 @@ where
                             forward_id: None,
                         },
                     )?;
+                } else {
+                    // This testcase is from the other fuzzers.
+                    *state.imported_mut() += 1;
                 }
                 Ok((res, Some(idx)))
             }
@@ -450,7 +454,7 @@ where
     OT: ObserversTuple<CS::State> + Serialize + DeserializeOwned,
     F: Feedback<CS::State>,
     OF: Feedback<CS::State>,
-    CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions,
+    CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasImported,
 {
     /// Process one input, adding to the respective corpora if needed and firing the right events
     #[inline]
@@ -483,7 +487,7 @@ where
     F: Feedback<CS::State>,
     OF: Feedback<CS::State>,
     OT: ObserversTuple<CS::State> + Serialize + DeserializeOwned,
-    CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions,
+    CS::State: HasCorpus + HasSolutions + HasClientPerfMonitor + HasExecutions + HasImported,
 {
     /// Process one input, adding to the respective corpora if needed and firing the right events
     #[inline]
@@ -591,6 +595,7 @@ where
         + HasMetadata
         + HasCorpus
         + HasTestcase
+        + HasImported
         + HasLastReportTime,
     ST: StagesTuple<E, EM, CS::State, Self>,
 {
@@ -633,11 +638,9 @@ where
         {
             let mut testcase = state.testcase_mut(idx)?;
             let scheduled_count = testcase.scheduled_count();
-
             // increase scheduled count, this was fuzz_level in afl
             testcase.set_scheduled_count(scheduled_count + 1);
         }
-
         Ok(idx)
     }
 }

libafl/src/stages/mod.rs

@@ -27,6 +27,9 @@ pub use power::{PowerMutationalStage, StdPowerMutationalStage};
 pub mod generalization;
 pub use generalization::GeneralizationStage;
 
+pub mod stats;
+pub use stats::AflStatsStage;
+
 pub mod owned;
 pub use owned::StagesOwnedList;
 

libafl/src/stages/mutational.rs

@@ -146,6 +146,7 @@ where
             post.post_exec(state, i as i32, corpus_idx)?;
             mark_feature_time!(state, PerfFeature::MutatePostExec);
         }
+
         Ok(())
     }
 }

libafl/src/stages/stats.rs

@@ -0,0 +1,162 @@
+//! Stage to compute/report AFL stats
+
+#[cfg(feature = "std")]
+use alloc::string::ToString;
+use core::{marker::PhantomData, time::Duration};
+
+use libafl_bolts::current_time;
+#[cfg(feature = "std")]
+use serde_json::json;
+
+use crate::{
+    corpus::{Corpus, CorpusId},
+    events::EventFirer,
+    schedulers::minimizer::IsFavoredMetadata,
+    stages::Stage,
+    state::{HasCorpus, HasImported, HasMetadata, UsesState},
+    Error,
+};
+#[cfg(feature = "std")]
+use crate::{events::Event, monitors::UserStats};
+
+/// The [`AflStatsStage`] is a simple stage that computes and reports some stats.
+#[derive(Debug, Clone)]
+pub struct AflStatsStage<E, EM, Z>
+where
+    E: UsesState,
+    EM: EventFirer<State = E::State>,
+    Z: UsesState<State = E::State>,
+{
+    // the number of testcases that have been fuzzed
+    has_fuzzed_size: usize,
+    // the number of "favored" testcases
+    is_favored_size: usize,
+    // the number of testcases found by itself
+    own_finds_size: usize,
+    // the number of testcases imported by other fuzzers
+    imported_size: usize,
+    // the last time that we report all stats
+    last_report_time: Duration,
+    // the interval that we report all stats
+    stats_report_interval: Duration,
+
+    phantom: PhantomData<(E, EM, Z)>,
+}
+
+impl<E, EM, Z> UsesState for AflStatsStage<E, EM, Z>
+where
+    E: UsesState,
+    EM: EventFirer<State = E::State>,
+    Z: UsesState<State = E::State>,
+{
+    type State = E::State;
+}
+
+impl<E, EM, Z> Stage<E, EM, Z> for AflStatsStage<E, EM, Z>
+where
+    E: UsesState,
+    EM: EventFirer<State = E::State>,
+    Z: UsesState<State = E::State>,
+    E::State: HasImported + HasCorpus + HasMetadata,
+{
+    fn perform(
+        &mut self,
+        _fuzzer: &mut Z,
+        _executor: &mut E,
+        state: &mut E::State,
+        _manager: &mut EM,
+        corpus_idx: CorpusId,
+    ) -> Result<(), Error> {
+        // Report your stats every `STATS_REPORT_INTERVAL`
+        // compute pending, pending_favored, imported, own_finds
+        {
+            let testcase = state.corpus().get(corpus_idx)?.borrow();
+            if testcase.scheduled_count() == 0 {
+                self.has_fuzzed_size += 1;
+                if testcase.has_metadata::<IsFavoredMetadata>() {
+                    self.is_favored_size += 1;
+                }
+            } else {
+                return Ok(());
+            }
+        }
+
+        let corpus_size = state.corpus().count();
+        let pending_size = corpus_size - self.has_fuzzed_size;
+        let pend_favored_size = corpus_size - self.is_favored_size;
+        self.imported_size = *state.imported();
+        self.own_finds_size = corpus_size - self.imported_size;
+
+        let cur = current_time();
+
+        if cur.checked_sub(self.last_report_time).unwrap_or_default() > self.stats_report_interval {
+            #[cfg(feature = "std")]
+            {
+                let json = json!({
+                        "pending":pending_size,
+                        "pend_fav":pend_favored_size,
+                        "own_finds":self.own_finds_size,
+                        "imported":self.imported_size,
+                });
+                _manager.fire(
+                    state,
+                    Event::UpdateUserStats {
+                        name: "AflStats".to_string(),
+                        value: UserStats::String(json.to_string()),
+                        phantom: PhantomData,
+                    },
+                )?;
+            }
+            #[cfg(not(feature = "std"))]
+            log::info!(
+                "pending: {}, pend_favored: {}, own_finds: {}, imported: {}",
+                pending_size,
+                pend_favored_size,
+                self.own_finds_size,
+                self.imported_size
+            );
+            self.last_report_time = cur;
+        }
+
+        Ok(())
+    }
+}
+
+impl<E, EM, Z> AflStatsStage<E, EM, Z>
+where
+    E: UsesState,
+    EM: EventFirer<State = E::State>,
+    Z: UsesState<State = E::State>,
+    E::State: HasImported + HasCorpus + HasMetadata,
+{
+    /// create a new instance of the [`AflStatsStage`]
+    #[must_use]
+    pub fn new(interval: Duration) -> Self {
+        Self {
+            stats_report_interval: interval,
+            ..Default::default()
+        }
+    }
+}
+
+impl<E, EM, Z> Default for AflStatsStage<E, EM, Z>
+where
+    E: UsesState,
+    EM: EventFirer<State = E::State>,
+    Z: UsesState<State = E::State>,
+    E::State: HasImported + HasCorpus + HasMetadata,
+{
+    /// the default instance of the [`AflStatsStage`]
+    #[must_use]
+    fn default() -> Self {
+        Self {
+            has_fuzzed_size: 0,
+            is_favored_size: 0,
+            own_finds_size: 0,
+            imported_size: 0,
+            last_report_time: current_time(),
+            stats_report_interval: Duration::from_secs(15),
+            phantom: PhantomData,
+        }
+    }
+}

libafl/src/state/mod.rs

@@ -210,6 +210,15 @@ pub trait HasExecutions {
     fn executions_mut(&mut self) -> &mut usize;
 }
 
+/// Trait for some stats of AFL
+pub trait HasImported {
+    ///the imported testcases counter
+    fn imported(&self) -> &usize;
+
+    ///the imported testcases counter (mutable)
+    fn imported_mut(&mut self) -> &mut usize;
+}
+
 /// Trait for the starting time
 pub trait HasStartTime {
     /// The starting time
@@ -244,6 +253,8 @@ pub struct StdState<I, C, R, SC> {
     executions: usize,
     /// At what time the fuzzing started
     start_time: Duration,
+    /// the number of new paths that imported from other fuzzers
+    imported: usize,
     /// The corpus
     corpus: C,
     // Solutions corpus
@@ -407,6 +418,20 @@ impl<I, C, R, SC> HasExecutions for StdState<I, C, R, SC> {
     }
 }
 
+impl<I, C, R, SC> HasImported for StdState<I, C, R, SC> {
+    /// Return the number of new paths that imported from other fuzzers
+    #[inline]
+    fn imported(&self) -> &usize {
+        &self.imported
+    }
+
+    /// Return the number of new paths that imported from other fuzzers
+    #[inline]
+    fn imported_mut(&mut self) -> &mut usize {
+        &mut self.imported
+    }
+}
+
 impl<I, C, R, SC> HasLastReportTime for StdState<I, C, R, SC> {
     /// The last time we reported progress,if available/used.
     /// This information is used by fuzzer `maybe_report_progress`.
@@ -812,6 +837,7 @@ where
         let mut state = Self {
             rand,
             executions: 0,
+            imported: 0,
             start_time: Duration::from_millis(0),
             metadata: SerdeAnyMap::default(),
             named_metadata: NamedSerdeAnyMap::default(),