diff --git a/Dockerfile.cn b/Dockerfile.cn index a4be645..5613b02 100644 --- a/Dockerfile.cn +++ b/Dockerfile.cn @@ -1,6 +1,9 @@ FROM ubuntu -RUN cp /etc/apt/sources.list /etc/apt/sources.list.backup && sed -i "s/archive.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list +RUN apt-get update && apt-get install -y ca-certificates +RUN cp /etc/apt/sources.list /etc/apt/sources.list.backup \ + && sed -i "s@http://.*archive.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list \ + && sed -i "s@http://.*security.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list RUN apt-get update && apt-get install -y build-essential parallel wget clang libelf1 libelf-dev zlib1g-dev COPY . /root/ diff --git a/README.md b/README.md index 9396a2e..6a3c0b0 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ A file is usually read by syscalls including `openat`, `newstatat` and `read` or ## Prerequisites -To compile eBPF programs and preload shared objects, you will need `clang`, `libelf`, `zlib`, `build-essential`. +The kernel version should be at least 5.8. To compile eBPF programs and preload shared objects, you will need `clang`, `libelf`, `zlib`, `build-essential`. On Ubuntu: ``` @@ -58,6 +58,20 @@ docker run -it deterministic_builds ## Examples +Run examples with docker: + +```bash +# build images normally +docker build -t example . +# build images use Chinese package manager mirrors +docker build -t example . -f Dockerfile.cn + +# run container +docker run -it example +# enter the shell +docker exec -it /bin/bash +``` + ### gcc macros Some macros in gcc can lead to nondeterminacy in compilation. diff --git a/bpf/README.md b/bpf/README.md index 288736a..aae6daa 100644 --- a/bpf/README.md +++ b/bpf/README.md @@ -35,6 +35,10 @@ make `modify_file_name` intercepts `openat` syscall to modify the names of read program files suffixed with ".c", ".cc", ".cpp" or ".cxx" to empty, making `__FILE__` macro return a fixed value. +## `modify_random` + +`modify_random` intercepts the reading syscalls of `/dev/random` and `/dev/urandom`, and `getrandom` syscall, making the random generation of names in the compiler (e.g. random names in Link Time Optimization) fixed. + ## `preload_filter` `preload_filter` intercepts the reading syscalls of `/etc/ld.so.preload` when loading an executable file. The reading syscalls return origin content of `/etc/ld.so.preload` when the specified processes calling them, or they return empty content. diff --git a/bpf/src/.gitignore b/bpf/src/.gitignore index 8fb920f..b59118f 100644 --- a/bpf/src/.gitignore +++ b/bpf/src/.gitignore @@ -3,4 +3,5 @@ /preload_filter /modify_file_timestamp /modify_file_name -/modify_file_read \ No newline at end of file +/modify_file_read +/modify_random \ No newline at end of file diff --git a/bpf/src/Makefile b/bpf/src/Makefile index 4df8516..92fa1b8 100644 --- a/bpf/src/Makefile +++ b/bpf/src/Makefile @@ -16,7 +16,7 @@ INCLUDES := -I$(OUTPUT) -I../libbpf/include/uapi -I$(dir $(VMLINUX)) CFLAGS := -g -Wall ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) -APPS = modify_time preload_filter modify_file_timestamp modify_file_name modify_file_read +APPS = modify_time preload_filter modify_file_timestamp modify_file_name modify_file_read modify_random # Get Clang's default includes on this system. We'll explicitly add these dirs # to the includes list when compiling with `-target bpf` because otherwise some diff --git a/bpf/src/common.bpf.h b/bpf/src/common.bpf.h index 3951b49..dc356a6 100644 --- a/bpf/src/common.bpf.h +++ b/bpf/src/common.bpf.h @@ -1,7 +1,21 @@ -bool comm_filter(char *comm) { - if (!(__builtin_memcmp("cc1\0", comm, 4) == 0)) { - return 0; +__inline__ __attribute__((always_inline)) bool comm_filter(char *comm) { + if ((__builtin_memcmp("cc1\0", comm, 4) == 0)) { + return 1; } - return 1; + if ((__builtin_memcmp("cc1p", comm, 4) == 0) && + (__builtin_memcmp("lus\0", comm + 4, 4) == 0)) { + return 1; + } + if ((__builtin_memcmp("lto1", comm, 4) == 0) && + (__builtin_memcmp("-wpa", comm + 4, 4) == 0)) { + return 1; + } + if ((__builtin_memcmp("g++\0", comm, 4) == 0)) { + return 1; + } + if ((__builtin_memcmp("gcc\0", comm, 4) == 0)) { + return 1; + } + return 0; } diff --git a/bpf/src/modify_file_read.bpf.c b/bpf/src/modify_file_read.bpf.c index ed9d43f..d341b7a 100644 --- a/bpf/src/modify_file_read.bpf.c +++ b/bpf/src/modify_file_read.bpf.c @@ -142,14 +142,19 @@ int handle_exit_newfstatat(struct trace_event_raw_sys_exit *ctx) { SEC("tracepoint/syscalls/sys_enter_read") int handle_enter_read(struct trace_event_raw_sys_enter *ctx) { - tid_t tid = bpf_get_current_pid_tgid(); - char comm[16]; bpf_get_current_comm(comm, 16); if (!comm_filter(comm)) { return 0; } + tid_t tid = bpf_get_current_pid_tgid(); + + long unsigned int *blank_p = bpf_map_lookup_elem(&tids, &tid); + if (blank_p == NULL) { + return 0; + } + char *content_p = (void *)ctx->args[1]; bpf_map_update_elem(&content_ps, &tid, &content_p, BPF_ANY); diff --git a/bpf/src/modify_random.bpf.c b/bpf/src/modify_random.bpf.c new file mode 100644 index 0000000..ea407e1 --- /dev/null +++ b/bpf/src/modify_random.bpf.c @@ -0,0 +1,180 @@ +#include "vmlinux.h" +#include +#include +#include +#include "modify_file_read.h" +#include "common.bpf.h" +#include "../../config/time_config.h" + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +char random_name[] = "/dev/random"; +char urandom_name[] = "/dev/urandom"; +char replace_buf[] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + +struct read_prop { + void *buf; + size_t count; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024); +} rb SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, tid_t); + __type(value, u8); +} tids SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, tid_t); + __type(value, struct read_prop); +} read_props SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, tid_t); + __type(value, u64); +} random_bufs SEC(".maps"); + +SEC("tracepoint/syscalls/sys_enter_openat") +int handle_enter_openat(struct trace_event_raw_sys_enter *ctx) { + tid_t tid = bpf_get_current_pid_tgid(); + + char comm[16]; + bpf_get_current_comm(comm, 16); + + if (!comm_filter(comm)) { + return 0; + } + + char *filename_p = (char *)ctx->args[1]; + + char filename[20]; + long success = bpf_probe_read_user(filename, 20, filename_p); + + if (((__builtin_memcmp(random_name, filename, 4) == 0) && + (__builtin_memcmp(random_name + 4, filename + 4, 4) == 0) && + (random_name[9] == filename[9] && random_name[10] == filename[10] && + random_name[11] == filename[11]))) { + bpf_printk("[sys_enter_openat] Detected reading %s", filename); + + u8 blank = 0; + bpf_map_update_elem(&tids, &tid, &blank, BPF_ANY); + } + + if (((__builtin_memcmp(urandom_name, filename, 4) == 0) && + (__builtin_memcmp(urandom_name + 4, filename + 4, 4) == 0) && + (__builtin_memcmp(urandom_name + 8, filename + 8, 4) == 0))) { + bpf_printk("[sys_enter_openat] Detected reading %s", filename); + + u8 blank = 0; + bpf_map_update_elem(&tids, &tid, &blank, BPF_ANY); + } + + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_read") +int handle_enter_read(struct trace_event_raw_sys_enter *ctx) { + char comm[16]; + bpf_get_current_comm(comm, 16); + if (!comm_filter(comm)) { + return 0; + } + + tid_t tid = bpf_get_current_pid_tgid(); + + long unsigned int *blank_p = bpf_map_lookup_elem(&tids, &tid); + if (blank_p == NULL) { + return 0; + } + + struct read_prop a_read_prop; + a_read_prop.buf = (void *)ctx->args[1]; + a_read_prop.count = (size_t)ctx->args[2]; + + bpf_map_update_elem(&read_props, &tid, &a_read_prop, BPF_ANY); + + return 0; +} + +SEC("tracepoint/syscalls/sys_exit_read") +int handle_exit_read(struct trace_event_raw_sys_exit *ctx) { + char comm[16]; + bpf_get_current_comm(comm, 16); + if (!comm_filter(comm)) { + return 0; + } + + tid_t tid = bpf_get_current_pid_tgid(); + + long unsigned int *blank_p = bpf_map_lookup_elem(&tids, &tid); + if (blank_p == NULL) { + return 0; + } + bpf_map_delete_elem(&tids, &tid); + + long unsigned int *a_read_prop_p = bpf_map_lookup_elem(&read_props, &tid); + if (a_read_prop_p == NULL) { + return 0; + } + + struct read_prop a_read_prop = *(struct read_prop *)a_read_prop_p; + + bpf_printk("[sys_exit_read] OVERWRITING read buf at %p size %d to 0", + a_read_prop.buf, a_read_prop.count); + bool success = bpf_probe_write_user(a_read_prop.buf, replace_buf, 8); + bpf_printk("[sys_exit_read] RESULT %d", success); + + return 0; +} + +SEC("tracepoint/syscalls/sys_enter_getrandom") +int handle_enter_getrandom(struct trace_event_raw_sys_enter *ctx) { + char comm[16]; + bpf_get_current_comm(comm, 16); + + if (!comm_filter(comm)) { + return 0; + } + + tid_t tid = bpf_get_current_pid_tgid(); + + void *random_buf = (void *)ctx->args[0]; + bpf_map_update_elem(&random_bufs, &tid, &random_buf, BPF_ANY); + + return 0; +} + +SEC("tracepoint/syscalls/sys_exit_getrandom") +int handle_exit_getrandom(struct trace_event_raw_sys_exit *ctx) { + char comm[16]; + bpf_get_current_comm(comm, 16); + if (!comm_filter(comm)) { + return 0; + } + + tid_t tid = bpf_get_current_pid_tgid(); + + long unsigned int *random_buf_p = bpf_map_lookup_elem(&random_bufs, &tid); + if (random_buf_p == NULL) { + return 0; + } + bpf_map_delete_elem(&random_bufs, &tid); + + char *random_buf = (char *)*random_buf_p; + + bpf_printk("[sys_exit_getrandom] OVERWRITING random buf at %p size 8 to 0", + random_buf); + bool success = bpf_probe_write_user(random_buf, replace_buf, 8); + bpf_printk("[sys_exit_getrandom] RESULT %d", success); + + return 0; +} diff --git a/bpf/src/modify_random.c b/bpf/src/modify_random.c new file mode 100644 index 0000000..3f4a284 --- /dev/null +++ b/bpf/src/modify_random.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (c) 2020 Facebook */ +#include +#include +#include +#include +#include +#include +#include "modify_random.h" +#include "modify_random.skel.h" + +static struct env { + bool verbose; + // long min_duration_ms; +} env; + +const char *argp_program_version = "modify_random 0.0"; +const char *argp_program_bug_address = ""; +const char argp_program_doc[] = + "BPF modify_random demo application.\n" + "\n" + "It traces process start and exits and shows associated \n" + "information (filename, process duration, PID and PPID, etc).\n" + "\n" + "USAGE: ./modify_random [-d ] [-v]\n"; + +static const struct argp_option opts[] = { + {"verbose", 'v', NULL, 0, "Verbose debug output"}, + {"duration", 'd', "DURATION-MS", 0, + "Minimum process duration (ms) to report"}, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) { + switch (key) { + case 'v': + env.verbose = true; + break; + // case 'd': + // errno = 0; + // env.min_duration_ms = strtol(arg, NULL, 10); + // if (errno || env.min_duration_ms <= 0) { + // fprintf(stderr, "Invalid duration: %s\n", arg); + // argp_usage(state); + // } + // break; + case ARGP_KEY_ARG: + argp_usage(state); + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static const struct argp argp = { + .options = opts, + .parser = parse_arg, + .doc = argp_program_doc, +}; + +static int libbpf_print_fn(enum libbpf_print_level level, const char *format, + va_list args) { + if (level == LIBBPF_DEBUG && !env.verbose) + return 0; + return vfprintf(stderr, format, args); +} + +static volatile bool exiting = false; + +static void sig_handler(int sig) { exiting = true; } + +static int handle_event(void *ctx, void *data, size_t data_sz) { + const struct event *e = data; + struct tm *tm; + char ts[32]; + time_t t; + + time(&t); + tm = localtime(&t); + strftime(ts, sizeof(ts), "%H:%M:%S", tm); + + if (e->exit_event) { + printf("%-8s %-5s %-16s %-7d %-7d [%u]", ts, "EXIT", e->comm, e->pid, + e->ppid, e->exit_code); + if (e->duration_ns) + printf(" (%llums)", e->duration_ns / 1000000); + printf("\n"); + } else { + printf("%-8s %-5s %-16s %-7d %-7d %s\n", ts, "EXEC", e->comm, e->pid, + e->ppid, e->filename); + } + + return 0; +} + +int main(int argc, char **argv) { + struct ring_buffer *rb = NULL; + struct modify_random_bpf *skel; + int err; + + /* Parse command line arguments */ + err = argp_parse(&argp, argc, argv, 0, NULL, NULL); + if (err) + return err; + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + /* Set up libbpf errors and debug info callback */ + libbpf_set_print(libbpf_print_fn); + + /* Cleaner handling of Ctrl-C */ + signal(SIGINT, sig_handler); + signal(SIGTERM, sig_handler); + + /* Load and verify BPF application */ + skel = modify_random_bpf__open(); + if (!skel) { + fprintf(stderr, "Failed to open and load BPF skeleton\n"); + return 1; + } + + /* Parameterize BPF code with minimum duration parameter */ + // skel->rodata->min_duration_ns = env.min_duration_ms * 1000000ULL; + + /* Load & verify BPF programs */ + err = modify_random_bpf__load(skel); + if (err) { + fprintf(stderr, "Failed to load and verify BPF skeleton\n"); + goto cleanup; + } + + /* Attach tracepoints */ + err = modify_random_bpf__attach(skel); + if (err) { + fprintf(stderr, "Failed to attach BPF skeleton\n"); + goto cleanup; + } + + /* Set up ring buffer polling */ + rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), handle_event, NULL, NULL); + if (!rb) { + err = -1; + fprintf(stderr, "Failed to create ring buffer\n"); + goto cleanup; + } + + /* Process events */ + while (!exiting) { + err = ring_buffer__poll(rb, 100 /* timeout, ms */); + /* Ctrl-C will cause -EINTR */ + if (err == -EINTR) { + err = 0; + break; + } + if (err < 0) { + printf("Error polling perf buffer: %d\n", err); + break; + } + } + +cleanup: + /* Clean up */ + ring_buffer__free(rb); + modify_random_bpf__destroy(skel); + + return err < 0 ? -err : 0; +} diff --git a/bpf/src/modify_random.h b/bpf/src/modify_random.h new file mode 100644 index 0000000..b619224 --- /dev/null +++ b/bpf/src/modify_random.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +/* Copyright (c) 2020 Facebook */ +#ifndef __MODIFY_RANDOM_H +#define __MODIFY_RANDOM_H + +#define TASK_COMM_LEN 16 +#define MAX_FILENAME_LEN 127 + +struct event { + int pid; + int ppid; + unsigned exit_code; + unsigned long long duration_ns; + char comm[TASK_COMM_LEN]; + char filename[MAX_FILENAME_LEN]; + bool exit_event; +}; + +#endif /* __MODIFY_RANDOM_H */ diff --git a/start.sh b/start.sh index 644c949..7b9c470 100755 --- a/start.sh +++ b/start.sh @@ -8,7 +8,7 @@ fi PRELOAD_DIR_PATH="./preload/" PRELOAD_SO_NAME="modify_time.so" BPF_DIR_PATH="./bpf/src/" -BPF_Apps=('modify_time' 'modify_file_timestamp' 'modify_file_read' 'preload_filter') +BPF_Apps=('modify_time' 'modify_file_timestamp' 'modify_file_read' 'preload_filter' 'modify_file_name' 'modify_random') set -x diff --git a/stop.sh b/stop.sh index 12ebe6a..94a9903 100755 --- a/stop.sh +++ b/stop.sh @@ -9,7 +9,7 @@ PRELOAD_DIR_PATH="./preload/" set -x -BPF_Apps=('modify_time' 'modify_file_timestamp' 'modify_file_read' 'preload_filter') +BPF_Apps=('modify_time' 'modify_file_timestamp' 'modify_file_read' 'preload_filter' 'modify_file_name' 'modify_random') for app in "${BPF_Apps[@]}"; do echo "Killing $app" diff --git a/tests/cases/kernel/.gitignore b/tests/cases/kernel/.gitignore index b8fbd0b..dbba47a 100644 --- a/tests/cases/kernel/.gitignore +++ b/tests/cases/kernel/.gitignore @@ -1 +1 @@ -linux-*/ \ No newline at end of file +linux-* \ No newline at end of file diff --git a/tests/cases/kernel/init.sh b/tests/cases/kernel/init.sh index 8dcc654..c491a7b 100755 --- a/tests/cases/kernel/init.sh +++ b/tests/cases/kernel/init.sh @@ -32,32 +32,35 @@ else fi KERNEL_DIR=linux-$KERNEL_VERSION -KERNEL_SRC_FILE=$KERNEL_DIR.tar.xz -KERNEL_SIGN_FILE=$KERNEL_DIR.tar.sign +KERNEL_SRC_TAR_FILE=$KERNEL_DIR.tar +KERNEL_SRC_XZ_FILE=$KERNEL_SRC_TAR_FILE.xz +KERNEL_SIGN_FILE=$KERNEL_SRC_TAR_FILE.sign -set -x +set -eux -apt-get install -y git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison +apt-get install -y git fakeroot build-essential ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison gnupg2 if [ ! -d "$KERNEL_DIR" ]; then - echo "Downloading linux kernel $KERNEL_VERSION source from" $KERNEL_URL_PREFIX$KERNEL_SRC_FILE - if [ ! -f "$KERNEL_SRC_FILE" ]; then - wget $KERNEL_URL_PREFIX$KERNEL_SRC_FILE + echo "Downloading linux kernel $KERNEL_VERSION source from" $KERNEL_URL_PREFIX$KERNEL_SRC_XZ_FILE + if [ ! -f "$KERNEL_SRC_TAR_FILE" ]; then + if [ ! -f "$KERNEL_SRC_XZ_FILE" ]; then + wget -q $KERNEL_URL_PREFIX$KERNEL_SRC_XZ_FILE + fi + unxz -v $KERNEL_SRC_XZ_FILE fi - unxz -v $KERNEL_SRC_FILE if [ ! -f "$KERNEL_SIGN_FILE" ]; then - wget $KERNEL_URL_PREFIX$KERNEL_SIGN_FILE + wget -q $KERNEL_URL_PREFIX$KERNEL_SIGN_FILE fi gpg2 --locate-keys torvalds@kernel.org gregkh@kernel.org gpg --verify $KERNEL_SIGN_FILE - tar xvf $KERNEL_DIR.tar + tar -xf $KERNEL_SRC_TAR_FILE fi -cd linux-$KERNEL_VERSION +cd $KERNEL_DIR -if [ !(-f ".config") ]; then +if [ ! -f ".config" ]; then make defconfig fi set +x diff --git a/tests/cases/lto/.gitignore b/tests/cases/lto/.gitignore new file mode 100644 index 0000000..4897d09 --- /dev/null +++ b/tests/cases/lto/.gitignore @@ -0,0 +1,3 @@ +*.ltrace +*.strace +*.diff \ No newline at end of file diff --git a/tests/cases/lto/collect_trace.sh b/tests/cases/lto/collect_trace.sh new file mode 100755 index 0000000..595cd55 --- /dev/null +++ b/tests/cases/lto/collect_trace.sh @@ -0,0 +1,12 @@ +strace -f -e 'trace=!write' -o lto.strace g++ -flto -o f1.o -c f1.cpp +strace -f -e 'trace=!write' -o nolto.strace g++ -o f1.o -c f1.cpp + +diff <(awk '{$1 = ""; print $0}' nolto.strace | sed 's/0x[0-9a-f]\+/0x/g') \ + <(awk '{$1 = ""; print $0}' lto.strace | sed 's/0x[0-9a-f]\+/0x/g') > lto_strace.diff + +ltrace -f -o lto.ltrace g++ -flto -o f1.o -c f1.cpp +ltrace -f -o nolto.ltrace g++ -o f1.o -c f1.cpp + +diff <(awk '{$1 = ""; print $0}' nolto.ltrace | sed 's/0x[0-9a-f]\+/0x/g') \ + <(awk '{$1 = ""; print $0}' lto.ltrace | sed 's/0x[0-9a-f]\+/0x/g') > lto_ltrace.diff + diff --git a/tests/init.sh b/tests/init.sh old mode 100644 new mode 100755 index 13dfa8f..eb56018 --- a/tests/init.sh +++ b/tests/init.sh @@ -2,17 +2,17 @@ echo "Init cases" -set -x +set -ex cd cases ls . | parallel -j $(nproc) \ -"echo 'Initing' {}; -cd {}; \ -if [ -e init.sh ] -then - sh init.sh; -fi " +if [ -e {}/init.sh ] +then + echo 'Initing' {}; \ + cd {}; \ + bash init.sh || { err=$?; echo {} failed; exit $err; } +fi" cd ../