Merge pull request AFLplusplus#2259 from AFLplusplus/removeoutdated

Remove outdated afl-gcc/afl-clang

Author: vanhauser-thc

GNUmakefile

@@ -30,12 +30,10 @@ INCLUDE_PATH = $(PREFIX)/include/afl
 PROGNAME    = afl
 VERSION     = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f2)
 
-# PROGS intentionally omit afl-as, which gets installed elsewhere.
-
 PROGS       = afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze
 SH_PROGS    = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-addseeds afl-system-config afl-persistent-config afl-cc
-HEADERS     = include/afl-as.h include/afl-fuzz.h include/afl-mutations.h include/afl-persistent-replay.h include/afl-prealloc.h include/afl-record-compat.h include/alloc-inl.h include/android-ashmem.h include/cmplog.h include/common.h include/config.h include/coverage-32.h include/coverage-64.h include/debug.h include/envs.h include/forkserver.h include/hash.h include/list.h include/sharedmem.h include/snapshot-inl.h include/t1ha.h include/t1ha0_ia32aes_b.h include/t1ha_bits.h include/t1ha_selfcheck.h include/types.h include/xxhash.h
-MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) afl-as.8
+HEADERS     = include/afl-fuzz.h include/afl-mutations.h include/afl-persistent-replay.h include/afl-prealloc.h include/afl-record-compat.h include/alloc-inl.h include/android-ashmem.h include/cmplog.h include/common.h include/config.h include/coverage-32.h include/coverage-64.h include/debug.h include/envs.h include/forkserver.h include/hash.h include/list.h include/sharedmem.h include/snapshot-inl.h include/t1ha.h include/t1ha0_ia32aes_b.h include/t1ha_bits.h include/t1ha_selfcheck.h include/types.h include/xxhash.h
+MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8)
 ASAN_OPTIONS=detect_leaks=0
 
 SYS = $(shell uname -s)
@@ -327,10 +325,12 @@ ifdef TEST_MMAP
 endif
 
 .PHONY: all
-all:	test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_build all_done
+all:	test_x86 test_shm test_python ready $(PROGS) llvm gcc_plugin test_build all_done
 	-$(MAKE) -C utils/aflpp_driver
 	@echo
 	@echo
+	@echo
+	@echo
 	@echo Build Summary:
 	@test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler"
 	@test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md"
@@ -339,6 +339,7 @@ all:	test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu
 ifneq "$(SYS)" "Darwin"
 	@test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this"
 endif
+	@test -e afl-cc || echo "[-] AFL++ instrumentation compilers could not be built! Install llvm-VERSION-dev or gcc-VERSION-plugin-dev, see docs/INSTALL.md!"
 	@echo
 
 .PHONY: llvm
@@ -465,10 +466,6 @@ endif
 ready:
 	@echo "[+] Everything seems to be working, ready to compile. ($(shell $(CC) --version 2>&1|head -n 1))"
 
-afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86
-	$(CC) $(CFLAGS) src/$@.c -o $@ $(LDFLAGS)
-	@ln -sf afl-as as
-
 src/afl-performance.o : $(COMM_HDR) src/afl-performance.c include/hash.h
 	$(CC) $(CFLAGS) $(CFLAGS_OPT) $(SPECIAL_PERFORMANCE) -Iinclude -c src/afl-performance.c -o src/afl-performance.o
 
@@ -576,7 +573,7 @@ code-format:
 
 .PHONY: test_build
 ifndef AFL_NO_X86
-test_build: afl-cc afl-gcc afl-as afl-showmap
+test_build: afl-cc afl-showmap
 	@echo "[*] Testing the CC wrapper afl-cc and its instrumentation output..."
 	@unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_LSAN AFL_USE_ASAN AFL_USE_MSAN AFL_LLVM_ALLOWLIST AFL_LLVM_DENYLIST; ASAN_OPTIONS=detect_leaks=0 AFL_INST_RATIO=100 AFL_PATH=. ./afl-cc test-instr.c $(LDFLAGS) -o test-instr 2>&1 || (echo "Oops, afl-cc failed"; exit 1 )
 	-ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -q -m none -o .test-instr0 ./test-instr < /dev/null
@@ -585,18 +582,18 @@ test_build: afl-cc afl-gcc afl-as afl-showmap
 	@cmp -s .test-instr0 .test-instr1; DR="$$?"; rm -f .test-instr0 .test-instr1; if [ "$$DR" = "0" ]; then echo; echo "Oops, the instrumentation of afl-cc does not seem to be behaving correctly!"; echo; echo "Please post to https://github.com/AFLplusplus/AFLplusplus/issues to troubleshoot the issue."; echo; exit 1; fi
 	@echo
 	@echo "[+] All right, the instrumentation of afl-cc seems to be working!"
-#	@echo "[*] Testing the CC wrapper afl-gcc and its instrumentation output..."
-#	@unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_LSAN AFL_USE_ASAN AFL_USE_MSAN; AFL_CC=$(CC) ASAN_OPTIONS=detect_leaks=0 AFL_INST_RATIO=100 AFL_PATH=. ./afl-gcc test-instr.c -o test-instr 2>&1 || (echo "Oops, afl-gcc failed"; exit 1 )
+#	@echo "[*] Testing the CC wrapper and its instrumentation output..."
+#	@unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_LSAN AFL_USE_ASAN AFL_USE_MSAN; AFL_CC=$(CC) ASAN_OPTIONS=detect_leaks=0 AFL_INST_RATIO=100 AFL_PATH=. ./afl-clang-fast test-instr.c -o test-instr 2>&1 || (echo "Oops, afl-clang-fast failed"; exit 1 )
 #	ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr0 ./test-instr < /dev/null
 #	echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr
 #	@rm -f test-instr
-#	@cmp -s .test-instr0 .test-instr1; DR="$$?"; rm -f .test-instr0 .test-instr1; if [ "$$DR" = "0" ]; then echo; echo "Oops, the instrumentation of afl-gcc does not seem to be behaving correctly!"; \
+#	@cmp -s .test-instr0 .test-instr1; DR="$$?"; rm -f .test-instr0 .test-instr1; if [ "$$DR" = "0" ]; then echo; echo "Oops, the instrumentation of afl-clang-fast does not seem to be behaving correctly!"; \
 #		gcc -v 2>&1 | grep -q -- --with-as= && ( echo; echo "Gcc is configured not to use an external assembler with the -B option." ) || \
 #		( echo; echo "Please post to https://github.com/AFLplusplus/AFLplusplus/issues to troubleshoot the issue." ); echo; exit 0; fi
 #	@echo
-#	@echo "[+] All right, the instrumentation of afl-gcc seems to be working!"
+#	@echo "[+] All right, the instrumentation of afl-clang-fast seems to be working!"
 else
-test_build: afl-cc afl-as afl-showmap
+test_build: afl-cc afl-showmap
 	@echo "[!] Note: skipping build tests (you may need to use LLVM or QEMU mode)."
 endif
 
@@ -606,15 +603,16 @@ all_done: test_build
 	@test -e cmplog-instructions-pass.so && echo "[+] LLVM mode for 'afl-cc' successfully built!" || echo "[-] LLVM mode for 'afl-cc'  failed to build, likely you either don't have llvm installed, or you need to set LLVM_CONFIG, to point to e.g. llvm-config-11. See instrumentation/README.llvm.md how to do this. Highly recommended!"
 	@test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode for 'afl-cc' successfully built!" || echo "[-] LLVM LTO mode for 'afl-cc'  failed to build, this would need LLVM 11+, see instrumentation/README.lto.md how to build it"
 	@test -e afl-gcc-pass.so && echo "[+] gcc_plugin for 'afl-cc' successfully built!" || echo "[-] gcc_plugin for 'afl-cc'  failed to build, unless you really need it that is fine - or read instrumentation/README.gcc_plugin.md how to build it"
-	@echo "[+] All done! Be sure to review the README.md - it's pretty short and useful."
+	@test -e afl-cc && echo "[+] All done! Be sure to review the README.md - it's pretty short and useful."
+	@test -e afl-cc || echo "[-] ERROR  - neither afl-clang-fast or afl-gcc-fast could be compiled - YOU ARE MISSING PACKAGES! Read docs/INSTALL.md!"
 	@if [ "$(SYS)" = "Darwin" ]; then printf "\nWARNING: Fuzzing on MacOS X is slow because of the unusually high overhead of\nfork() on this OS. Consider using Linux or *BSD for fuzzing software not\nspecifically for MacOS.\n\n"; fi
 	@! tty <&1 >/dev/null || printf "\033[0;30mNOTE: If you can read this, your terminal probably uses white background.\nThis will make the UI hard to read. See docs/status_screen.md for advice.\033[0m\n" 2>/dev/null
 
 .NOTPARALLEL: clean all
 
 .PHONY: clean
 clean:
-	rm -rf $(PROGS) afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM lib*.a
+	rm -rf $(PROGS) afl-fuzz-document as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM lib*.a
 	-$(MAKE) -f GNUmakefile.llvm clean
 	-$(MAKE) -f GNUmakefile.gcc_plugin clean
 	-$(MAKE) -C utils/libdislocator clean
@@ -831,18 +829,16 @@ endif
 	install -m 644 $(HEADERS) $${DESTDIR}$(INCLUDE_PATH)
 	@mkdir -m 0755 -p ${DESTDIR}$(MAN_PATH)
 	install -m0644 *.8 ${DESTDIR}$(MAN_PATH)
-	install -m 755 afl-as $${DESTDIR}$(HELPER_PATH)
-	ln -sf afl-as $${DESTDIR}$(HELPER_PATH)/as
 	install -m 644 docs/*.md $${DESTDIR}$(DOC_PATH)
 	cp -r testcases/ $${DESTDIR}$(MISC_PATH)
 	cp -r dictionaries/ $${DESTDIR}$(MISC_PATH)
 	cp injections.dic $${DESTDIR}$(MISC_PATH)
 
 .PHONY: uninstall
 uninstall:
-	-cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-client afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto*
+	-cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-client afl-network-server afl-g* afl-plot.sh afl-ld-lto afl-c* afl-lto*
 	-cd $${DESTDIR}$(INCLUDE_PATH) && rm -f $(HEADERS:include/%=%)
-	-cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so libnyx.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a as afl-as SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt injections.dic
+	-cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so libnyx.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt injections.dic
 	-rm -rf $${DESTDIR}$(MISC_PATH)/testcases $${DESTDIR}$(MISC_PATH)/dictionaries
 	-sh -c "ls docs/*.md | sed 's|^docs/|$${DESTDIR}$(DOC_PATH)/|' | xargs rm -f"
 	-cd $${DESTDIR}$(MAN_PATH) && rm -f $(MANPAGES)

docs/env_variables.md

@@ -45,14 +45,10 @@ fairly broad use of environment variables instead:
           make
     ```
 
-  - Setting `AFL_AS`, `AFL_CC`, and `AFL_CXX` lets you use alternate downstream
-    compilation tools, rather than the default 'as', 'clang', or 'gcc' binaries
+  - Setting `AFL_CC`, and `AFL_CXX` lets you use alternate downstream
+    compilation tools, rather than the default 'clang', or 'gcc' binaries
     in your `$PATH`.
 
-  - If you are a weird person that wants to compile and instrument asm text
-    files, then use the `AFL_AS_FORCE_INSTRUMENT` variable:
-    `AFL_AS_FORCE_INSTRUMENT=1 afl-gcc foo.s -o foo`
-
   - Most AFL tools do not print any output if stdout/stderr are redirected. If
     you want to get the output into a file, then set the `AFL_DEBUG` environment
     variable. This is sadly necessary for various build processes which fail
@@ -83,17 +79,13 @@ fairly broad use of environment variables instead:
     Setting `AFL_INST_RATIO` to 0 is a valid choice. This will instrument only
     the transitions between function entry points, but not individual branches.
 
-    Note that this is an outdated variable. A few instances (e.g., afl-gcc)
-    still support these, but state-of-the-art (e.g., LLVM LTO and LLVM PCGUARD)
-    do not need this.
+    Note that this is an outdated variable. Only LLVM CLASSIC pass can use this.
 
   - `AFL_NO_BUILTIN` causes the compiler to generate code suitable for use with
     libtokencap.so (but perhaps running a bit slower than without the flag).
 
-  - `AFL_PATH` can be used to point afl-gcc to an alternate location of afl-as.
-    One possible use of this is utils/clang_asm_normalize/, which lets you
-    instrument hand-written assembly when compiling clang code by plugging a
-    normalizer into the chain. (There is no equivalent feature for GCC.)
+  - `AFL_PATH` can be used to point a directory that contains LLVM/GCC plugins
+    for AFL++, AFL++'s runtime objects and QEMU/Frida support files.
 
   - Setting `AFL_QUIET` will prevent afl-as and afl-cc banners from being
     displayed during compilation, in case you find them distracting.

docs/features.md

@@ -6,20 +6,22 @@ QEMU 5.1 with laf-intel and Redqueen, FRIDA mode, unicorn mode, gcc plugin, full
 
 ## Features and instrumentation
 
-| Feature/Instrumentation       | afl-gcc  | llvm      | gcc_plugin | FRIDA mode(9)  | QEMU mode(10)    | unicorn_mode(10) | nyx_mode(12) | coresight_mode(11) |
-| ------------------------------|:--------:|:---------:|:----------:|:--------------:|:----------------:|:----------------:|:------------:|:------------------:|
-| Threadsafe counters [A]       |          |    x(3)   |            |                |                  |                  |       x      |                    |
-| NeverZero           [B]       | x86[_64] |    x(1)   |      x     |        x       |         x        |         x        |              |                    |
-| Persistent Mode     [C]       |          |     x     |      x     | x86[_64]/arm64 | x86[_64]/arm[64] |         x        |              |                    |
-| LAF-Intel / CompCov [D]       |          |     x     |            |                | x86[_64]/arm[64] | x86[_64]/arm[64] |   x86[_64]   |                    |
-| CmpLog              [E]       |          |     x     |      x     | x86[_64]/arm64 | x86[_64]/arm[64] |                  |              |                    |
-| Selective Instrumentation [F] |          |     x     |      x     |        x       |         x        |                  |              |                    |
-| Non-Colliding Coverage    [G] |          |    x(4)   |            |                |       (x)(5)     |                  |              |                    |
-| Ngram prev_loc Coverage   [H] |          |    x(6)   |            |                |                  |                  |              |                    |
-| Context Coverage    [I]       |          |    x(6)   |            |                |                  |                  |              |                    |
-| Auto Dictionary     [J]       |          |    x(7)   |            |                |                  |                  |              |                    |
-| Snapshot Support    [K]       |          |   (x)(8)  |   (x)(8)   |                |       (x)(5)     |                  |       x      |                    |
-| Shared Memory Test cases  [L] |          |     x     |      x     | x86[_64]/arm64 |         x        |         x        |       x      |                    |
+Note that afl-gcc and afl-clang have been removed because their instrumentation is absolutely outdated.
+
+| Feature/Instrumentation       | llvm      | gcc_plugin | FRIDA mode(9)  | QEMU mode(10)    | unicorn_mode(10) | nyx_mode(12) | coresight_mode(11) |
+| ------------------------------|:---------:|:----------:|:--------------:|:----------------:|:----------------:|:------------:|:------------------:|
+| Threadsafe counters [A]       |    x(3)   |            |                |                  |                  |       x      |                    |
+| NeverZero           [B]       |    x(1)   |      x     |        x       |         x        |         x        |              |                    |
+| Persistent Mode     [C]       |     x     |      x     | x86[_64]/arm64 | x86[_64]/arm[64] |         x        |              |                    |
+| LAF-Intel / CompCov [D]       |     x     |            |                | x86[_64]/arm[64] | x86[_64]/arm[64] |   x86[_64]   |                    |
+| CmpLog              [E]       |     x     |      x     | x86[_64]/arm64 | x86[_64]/arm[64] |                  |              |                    |
+| Selective Instrumentation [F] |     x     |      x     |        x       |         x        |                  |              |                    |
+| Non-Colliding Coverage    [G] |    x(4)   |            |                |       (x)(5)     |                  |              |                    |
+| Ngram prev_loc Coverage   [H] |    x(6)   |            |                |                  |                  |              |                    |
+| Context Coverage    [I]       |    x(6)   |            |                |                  |                  |              |                    |
+| Auto Dictionary     [J]       |    x(7)   |            |                |                  |                  |              |                    |
+| Snapshot Support    [K]       |   (x)(8)  |   (x)(8)   |                |       (x)(5)     |                  |       x      |                    |
+| Shared Memory Test cases  [L] |     x     |      x     | x86[_64]/arm64 |         x        |         x        |       x      |                    |
 
 ## More information about features
 
@@ -94,7 +96,7 @@ L. Faster fuzzing and less kernel syscall overhead by in-memory fuzz testcase
 
 Among others, the following features and patches have been integrated:
 
-* NeverZero patch for afl-gcc, instrumentation, QEMU mode and unicorn_mode which
+* NeverZero for llvm/gcc instrumentation, QEMU mode and unicorn_mode which
   prevents a wrapping map value to zero, increases coverage
 * Persistent mode, deferred forkserver and in-memory fuzzing for QEMU mode
 * Unicorn mode which allows fuzzing of binaries from completely different

docs/fuzzing_binary-only_targets.md

@@ -199,6 +199,7 @@ have an x86_64 or arm64 binary that does not contain C++ exceptions and - if
 x86_64 - still has it's symbols and compiled with position independent code
 (PIC/PIE), then the RetroWrite solution might be for you.
 It decompiles to ASM files which can then be instrumented with afl-gcc.
+Note that afl-gcc is only present until AFL++ v4.21c and was subsequently removed as it is obsolete.
 
 Binaries that are statically instrumented for fuzzing using RetroWrite are close
 in performance to compiler-instrumented binaries and outperform the QEMU-based