Merge pull request AFLplusplus#2248 from AFLplusplus/dev

push to stable

Author: vanhauser-thc

GNUmakefile

@@ -34,7 +34,7 @@ VERSION     = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f
 
 PROGS       = afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze
 SH_PROGS    = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-addseeds afl-system-config afl-persistent-config afl-cc
-HEADERS     = include/config.h include/types.h
+HEADERS     = include/afl-as.h include/afl-fuzz.h include/afl-mutations.h include/afl-persistent-replay.h include/afl-prealloc.h include/afl-record-compat.h include/alloc-inl.h include/android-ashmem.h include/cmplog.h include/common.h include/config.h include/coverage-32.h include/coverage-64.h include/debug.h include/envs.h include/forkserver.h include/hash.h include/list.h include/sharedmem.h include/snapshot-inl.h include/t1ha.h include/t1ha0_ia32aes_b.h include/t1ha_bits.h include/t1ha_selfcheck.h include/types.h include/xxhash.h
 MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) afl-as.8
 ASAN_OPTIONS=detect_leaks=0
 
@@ -119,7 +119,7 @@ endif
 COMPILER_TYPE=$(shell $(CC) --version|grep "Free Software Foundation")
 ifneq "$(COMPILER_TYPE)" ""
   #$(info gcc is being used)
-  override CFLAGS_OPT += -Wno-error=format-truncation -Wno-format-truncation
+  override CFLAGS_OPT += -Wno-format-truncation
 endif
 
 ifeq "$(SYS)" "SunOS"
@@ -578,7 +578,7 @@ code-format:
 ifndef AFL_NO_X86
 test_build: afl-cc afl-gcc afl-as afl-showmap
 	@echo "[*] Testing the CC wrapper afl-cc and its instrumentation output..."
-	@unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_LSAN AFL_USE_ASAN AFL_USE_MSAN; ASAN_OPTIONS=detect_leaks=0 AFL_INST_RATIO=100 AFL_PATH=. ./afl-cc test-instr.c $(LDFLAGS) -o test-instr 2>&1 || (echo "Oops, afl-cc failed"; exit 1 )
+	@unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_LSAN AFL_USE_ASAN AFL_USE_MSAN AFL_LLVM_ALLOWLIST AFL_LLVM_DENYLIST; ASAN_OPTIONS=detect_leaks=0 AFL_INST_RATIO=100 AFL_PATH=. ./afl-cc test-instr.c $(LDFLAGS) -o test-instr 2>&1 || (echo "Oops, afl-cc failed"; exit 1 )
 	-ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -q -m none -o .test-instr0 ./test-instr < /dev/null
 	-echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr
 	@rm -f test-instr

GNUmakefile.gcc_plugin

@@ -163,7 +163,7 @@ $(PASSES): instrumentation/afl-gcc-common.h
 .PHONY: test_build
 test_build: $(PROGS)
 	@echo "[*] Testing the CC wrapper and instrumentation output..."
-	unset AFL_USE_ASAN AFL_USE_MSAN; ASAN_OPTIONS=detect_leaks=0 AFL_QUIET=1 AFL_INST_RATIO=100 AFL_PATH=. AFL_CC=$(CC) ./afl-gcc-fast $(CFLAGS) $(CPPFLAGS) ./test-instr.c -o test-instr $(LDFLAGS)
+	unset AFL_USE_ASAN AFL_USE_MSAN AFL_LLVM_ALLOWLIST AFL_LLVM_DENYLIST; ASAN_OPTIONS=detect_leaks=0 AFL_QUIET=1 AFL_INST_RATIO=100 AFL_PATH=. AFL_CC=$(CC) ./afl-gcc-fast $(CFLAGS) $(CPPFLAGS) ./test-instr.c -o test-instr $(LDFLAGS)
 	ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr0 ./test-instr </dev/null
 	echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr
 	@rm -f test-instr

GNUmakefile.llvm

@@ -69,7 +69,7 @@ endif
 
 LLVM_STDCXX                  := gnu++11
 LLVM_LTO                     := 0
-LLVM_UNSUPPORTED             := $(shell echo "$(LLVMVER)" | grep -E -q '^[0-2]\.|^3\.[0-8]\.' && echo 1 || echo 0)
+LLVM_UNSUPPORTED             := $(shell echo "$(LLVMVER)" | grep -E -q '^[0-2]\.|^3\.[0-7]\.' && echo 1 || echo 0)
 # Uncomment to see the values assigned above
 # $(foreach var,LLVM_CONFIG LLVMVER LLVM_MAJOR LLVM_MINOR LLVM_TOO_NEW LLVM_TOO_OLD LLVM_TOO_NEW_DEFAULT LLVM_TOO_OLD_DEFAULT LLVM_NEW_API LLVM_NEWER_API LLVM_13_OK LLVM_HAVE_LTO LLVM_BINDIR LLVM_LIBDIR LLVM_STDCXX LLVM_APPLE_XCODE LLVM_LTO LLVM_UNSUPPORTED,$(warning $(var) = $($(var))))
 
@@ -508,7 +508,7 @@ document:
 .PHONY: test_build
 test_build: $(PROGS)
 	@echo "[*] Testing the CC wrapper and instrumentation output..."
-	unset AFL_USE_ASAN AFL_USE_MSAN AFL_INST_RATIO; ASAN_OPTIONS=detect_leaks=0 AFL_QUIET=1 AFL_PATH=. AFL_LLVM_LAF_ALL=1 ./afl-cc $(CFLAGS) $(CPPFLAGS) ./test-instr.c -o test-instr $(LDFLAGS)
+	unset AFL_USE_ASAN AFL_USE_MSAN AFL_INST_RATIO AFL_LLVM_ALLOWLIST AFL_LLVM_DENYLIST; ASAN_OPTIONS=detect_leaks=0 AFL_QUIET=1 AFL_PATH=. AFL_LLVM_LAF_ALL=1 ./afl-cc $(CFLAGS) $(CPPFLAGS) ./test-instr.c -o test-instr $(LDFLAGS)
 	ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr0 ./test-instr < /dev/null
 	echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr
 	@rm -f test-instr

custom_mutators/rust/Cargo.toml

@@ -5,4 +5,5 @@ members = [
     "example",
     # Lain needs a nightly toolchain
     # "example_lain",
-]
+    # "example_lain_post_process",
+]

custom_mutators/rust/README.md

@@ -5,7 +5,15 @@ Bindings to create custom mutators in Rust.
 These bindings are documented with rustdoc. To view the documentation run
 ```cargo doc -p custom_mutator --open```.
 
-A minimal example can be found in `example`. Build it using `cargo build --example example_mutator`. 
+A minimal example can be found in `example`. Build it using `cargo build --example example_mutator`.
 
 An example using [lain](https://github.com/microsoft/lain) for structured fuzzing can be found in `example_lain`.
 Since lain requires a nightly rust toolchain, you need to set one up before you can play with it.
+
+An example for the use of the post_process function, using [lain](https://github.com/microsoft/lain) with [serde](https://github.com/serde-rs/serde) and [bincode](https://github.com/bincode-org/bincode) can be found in `example_lain_post_process`.
+In order for it to work you need to:
+
+- disable input trimming with `AFL_DISABLE_TRIM=1`
+- provide an initial instance serialized with `bincode` or use the `AFL_NO_STARTUP_CALIBRATION=1` environment variable.
+
+Note that `bincode` can also be used to serialize/deserialize the lain-generated structure and mutate it rather than generating a new one at each iteration, but it requires some structure serialized with `bincode` as input seed.

custom_mutators/rust/custom_mutator/src/lib.rs

@@ -73,6 +73,8 @@ pub trait RawCustomMutator {
         None
     }
 
+    fn post_process<'b, 's: 'b>(&'s mut self, buffer: &'b mut [u8]) -> Option<&'b [u8]>;
+
     /*fn post_process(&self, buffer: &[u8], unsigned char **out_buf)-> usize;
     int afl_custom_init_trim(&self, buffer: &[u8]);
     size_t afl_custom_trim(&self, unsigned char **out_buf);
@@ -353,6 +355,33 @@ pub mod wrappers {
             Err(err) => panic_handler("afl_custom_queue_get", &err),
         }
     }
+
+    /// Internal function used in the macro
+    pub unsafe fn afl_custom_post_process<M: RawCustomMutator>(
+        data: *mut c_void,
+        buf: *mut u8,
+        buf_size: usize,
+        out_buf: *mut *const u8,
+    ) -> usize {
+        match catch_unwind(|| {
+            let mut context = FFIContext::<M>::from(data);
+
+            assert!(!buf.is_null(), "null buf passed to afl_custom_post_process");
+            assert!(
+                !out_buf.is_null(),
+                "null out_buf passed to afl_custom_post_process"
+            );
+            let buff_slice = slice::from_raw_parts_mut(buf, buf_size);
+            if let Some(buffer) = context.mutator.post_process(buff_slice) {
+                *out_buf = buffer.as_ptr();
+                return buffer.len();
+            }
+            0
+        }) {
+            Ok(ret) => ret,
+            Err(err) => panic_handler("afl_custom_post_process", &err),
+        }
+    }
 }
 
 /// An exported macro to defined afl_custom_init meant for insternal usage
@@ -480,6 +509,16 @@ macro_rules! export_mutator {
         pub unsafe extern "C" fn afl_custom_deinit(data: *mut ::std::os::raw::c_void) {
             $crate::wrappers::afl_custom_deinit_::<$mutator_type>(data)
         }
+
+        #[no_mangle]
+        pub unsafe extern "C" fn afl_custom_post_process(
+            data: *mut ::std::os::raw::c_void,
+            buf: *mut u8,
+            buf_size: usize,
+            out_buf: *mut *const u8,
+        ) -> usize {
+            $crate::wrappers::afl_custom_post_process::<$mutator_type>(data, buf, buf_size, out_buf)
+        }
     };
 }
 
@@ -512,6 +551,10 @@ mod sanity_test {
         ) -> Option<&'b [u8]> {
             unimplemented!()
         }
+
+        fn post_process<'b, 's: 'b>(&'s mut self, buffer: &'b mut [u8]) -> Option<&'b [u8]> {
+            unimplemented!()
+        }
     }
 
     export_mutator!(ExampleMutator);
@@ -579,6 +622,13 @@ pub trait CustomMutator {
     fn introspection(&mut self) -> Result<Option<&str>, Self::Error> {
         Ok(None)
     }
+
+    fn post_process<'b, 's: 'b>(
+        &'s mut self,
+        buffer: &'b mut [u8],
+    ) -> Result<Option<&'b [u8]>, Self::Error> {
+        Ok(Some(buffer))
+    }
 }
 
 impl<M> RawCustomMutator for M
@@ -682,6 +732,16 @@ where
             }
         }
     }
+
+    fn post_process<'b, 's: 'b>(&'s mut self, buffer: &'b mut [u8]) -> Option<&'b [u8]> {
+        match self.post_process(buffer) {
+            Ok(r) => r,
+            Err(e) => {
+                Self::handle_error(e);
+                None
+            }
+        }
+    }
 }
 
 /// the default value to return from [`CustomMutator::describe`].

custom_mutators/rust/example_lain/Cargo.toml

@@ -8,9 +8,9 @@ edition = "2021"
 
 [dependencies]
 custom_mutator = { path = "../custom_mutator" }
-lain="0.5"
+lain = { git = "https://github.com/AFLplusplus/lain.git" }
 
 [[example]]
 name = "example_lain"
 path = "./src/lain_mutator.rs"
-crate-type = ["cdylib"]
+crate-type = ["cdylib"]

custom_mutators/rust/example_lain_post_process/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "example_lain_post_process"
+version = "0.1.0"
+authors = [
+    "Julius Hohnerlein <julihoh@users.noreply.github.com>",
+    "jma <94166787+jma-qb@users.noreply.github.com>",
+]
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+custom_mutator = { path = "../custom_mutator" }
+lain = { git = "https://github.com/AFLplusplus/lain.git" }
+bincode = "1.3.3"
+serde = { version = "1.0.214", features = ["derive"] }
+
+[[example]]
+name = "example_lain_post_process"
+path = "./src/lain_mutator.rs"
+crate-type = ["cdylib"]

custom_mutators/rust/example_lain_post_process/rust-toolchain

@@ -0,0 +1 @@
+nightly

custom_mutators/rust/example_lain_post_process/src/lain_mutator.rs

@@ -0,0 +1,70 @@
+#![cfg(unix)]
+
+use custom_mutator::{export_mutator, CustomMutator};
+use lain::{
+    mutator::Mutator,
+    prelude::*,
+    rand::{rngs::StdRng, SeedableRng},
+};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize, Serialize, Mutatable, NewFuzzed, BinarySerialize)]
+struct MyStruct {
+    tag: u8,
+    #[lain(ignore)]
+    length: u32,
+    #[lain(min = 0, max = 10)]
+    data: Vec<u8>,
+}
+
+struct LainMutator {
+    mutator: Mutator<StdRng>,
+    buffer: Vec<u8>,
+    post_buffer: Vec<u8>,
+}
+
+impl CustomMutator for LainMutator {
+    type Error = ();
+
+    fn init(seed: u32) -> Result<Self, ()> {
+        Ok(Self {
+            mutator: Mutator::new(StdRng::seed_from_u64(seed as u64)),
+            buffer: Vec::new(),
+            post_buffer: Vec::new(),
+        })
+    }
+
+    fn fuzz<'b, 's: 'b>(
+        &'s mut self,
+        _buffer: &'b mut [u8],
+        _add_buff: Option<&[u8]>,
+        max_size: usize,
+    ) -> Result<Option<&'b [u8]>, ()> {
+        // we just sample an instance of MyStruct, ignoring the current input
+        let instance = MyStruct::new_fuzzed(&mut self.mutator, None);
+        let serialized = bincode::serialize(&instance).unwrap();
+        let size = serialized.len();
+        if size > max_size {
+            return Err(());
+        }
+        self.buffer.clear();
+        self.buffer.reserve(size);
+        self.buffer.extend_from_slice(&serialized);
+        Ok(Some(self.buffer.as_slice()))
+    }
+
+    fn post_process<'b, 's: 'b>(
+        &'s mut self,
+        buffer: &'b mut [u8],
+    ) -> Result<Option<&'b [u8]>, Self::Error> {
+        let mut instance = bincode::deserialize::<MyStruct>(&buffer).unwrap();
+        instance.length = instance.data.len() as u32;
+        let size = instance.serialized_size();
+        self.post_buffer.clear();
+        self.post_buffer.reserve(size);
+        instance.binary_serialize::<_, BigEndian>(&mut self.post_buffer);
+        Ok(Some(&self.post_buffer))
+    }
+}
+
+export_mutator!(LainMutator);