From 0b1a7742d64d8a563b686c05ff0bd8d9090ba4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= Date: Sun, 9 Apr 2023 22:46:42 +0200 Subject: [PATCH 1/6] buildkite: Add asan and force32bit variants to CI tests. --- .buildkite/lib/common.jl | 3 +++ .buildkite/lib/generate.jl | 14 ++++++++++++- .buildkite/lib/types.jl | 42 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/.buildkite/lib/common.jl b/.buildkite/lib/common.jl index 9285ac74d5a..3547cab18f2 100644 --- a/.buildkite/lib/common.jl +++ b/.buildkite/lib/common.jl @@ -4,5 +4,8 @@ function get_yaml_path(platform::Platform) lib_dir = @__DIR__ buildkite_dir = dirname(lib_dir) yaml_path = joinpath(buildkite_dir, "test-$(platform.arch).yml") + if platform.variant != "" + yaml_path = joinpath(buildkite_dir, "test-$(platform.arch)-$(platform.variant).yml") + end return yaml_path end diff --git a/.buildkite/lib/generate.jl b/.buildkite/lib/generate.jl index 0c12929d844..5db654a2d0c 100644 --- a/.buildkite/lib/generate.jl +++ b/.buildkite/lib/generate.jl @@ -3,6 +3,7 @@ import YAML include(joinpath(@__DIR__, "common.jl")) function generate(platform::Platform) + force32bit = occursin("force32bit", platform.variant) commands = """ echo "--- Print kernel information" uname -a @@ -18,12 +19,19 @@ function generate(platform::Platform) tar -C / -xf DebianGlibc.v2.33.0.aarch64-linux-gnu.tar.gz fi + if [[ "$(force32bit)" == "true" ]]; then + echo "--- Installing i386 packages" + dpkg --add-architecture i386 + apt update + apt install -y capnproto libcapnp-dev:i386 zlib1g-dev:i386 file + fi + echo "--- Generate build environment" cmake --version rm -rf obj mkdir obj cd obj - cmake .. + cmake $(platform.cmake_extra_arg) .. echo "--- Build" make --output-sync -j\$\${JULIA_CPU_THREADS:?} @@ -41,6 +49,10 @@ function generate(platform::Platform) """ job_label = "Test $(platform.arch)" job_key = "test-$(platform.arch)" + if platform.variant != "" + job_label = "Test $(platform.arch) $(platform.variant)" + job_key = "test-$(platform.arch)-$(platform.variant)" + end yaml = Dict( "steps" => [ Dict( diff --git a/.buildkite/lib/types.jl b/.buildkite/lib/types.jl index 5271abb6b10..640a90c1330 100644 --- a/.buildkite/lib/types.jl +++ b/.buildkite/lib/types.jl @@ -4,6 +4,8 @@ Base.@kwdef struct Platform rootfs_treehash::String allow_fail::Bool commit_status::Bool + cmake_extra_arg::String + variant::String end struct Platforms @@ -22,6 +24,26 @@ const platforms = Platforms( rootfs_treehash = "1cd67e278881dcfeed695282256b26fad603e15d", allow_fail = false, commit_status = true, + cmake_extra_arg = "", + variant = "", + ), + Platform(; + arch = "x86_64", + rootfs_tag = "v5.22", + rootfs_treehash = "1cd67e278881dcfeed695282256b26fad603e15d", + allow_fail = false, + commit_status = true, + cmake_extra_arg = "-Dasan=true", + variant = "asan", + ), + Platform(; + arch = "aarch64", + rootfs_tag = "v5.22", + rootfs_treehash = "7a63218e46996b36aa108b55746a3d94a3e312c1", + allow_fail = false, + commit_status = true, + cmake_extra_arg = "", + variant = "", ), Platform(; arch = "aarch64", @@ -29,6 +51,26 @@ const platforms = Platforms( rootfs_treehash = "7a63218e46996b36aa108b55746a3d94a3e312c1", allow_fail = false, commit_status = true, + cmake_extra_arg = "-Dasan=true", + variant = "asan", + ), + Platform(; + arch = "x86_64", + rootfs_tag = "v5.22", + rootfs_treehash = "1cd67e278881dcfeed695282256b26fad603e15d", + allow_fail = false, + commit_status = true, + cmake_extra_arg = "-Dforce32bit=true", + variant = "force32bit", + ), + Platform(; + arch = "x86_64", + rootfs_tag = "v5.22", + rootfs_treehash = "1cd67e278881dcfeed695282256b26fad603e15d", + allow_fail = false, + commit_status = true, + cmake_extra_arg = "-Dforce32bit=true -Dasan=true", + variant = "force32bit-asan", ), ] ) From ea7cb3c41fb79cce2bc2471431c41ec38dfe9d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= Date: Tue, 18 Apr 2023 16:45:06 +0200 Subject: [PATCH 2/6] buildkite: Add per variant timeout value and increase aarch asan and force32bit to 55 minutes. --- .buildkite/lib/generate.jl | 2 +- .buildkite/lib/types.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.buildkite/lib/generate.jl b/.buildkite/lib/generate.jl index 5db654a2d0c..6d32f5ccd50 100644 --- a/.buildkite/lib/generate.jl +++ b/.buildkite/lib/generate.jl @@ -58,7 +58,7 @@ function generate(platform::Platform) Dict( "label" => job_label, "key" => job_key, - "timeout_in_minutes" => 45, + "timeout_in_minutes" => platform.timeout, "agents" => Dict( "sandbox_capable" => "true", "queue" => "juliaecosystem", diff --git a/.buildkite/lib/types.jl b/.buildkite/lib/types.jl index 640a90c1330..773bd38908b 100644 --- a/.buildkite/lib/types.jl +++ b/.buildkite/lib/types.jl @@ -6,6 +6,7 @@ Base.@kwdef struct Platform commit_status::Bool cmake_extra_arg::String variant::String + timeout::Integer end struct Platforms @@ -26,6 +27,7 @@ const platforms = Platforms( commit_status = true, cmake_extra_arg = "", variant = "", + timeout = 45, ), Platform(; arch = "x86_64", @@ -35,6 +37,7 @@ const platforms = Platforms( commit_status = true, cmake_extra_arg = "-Dasan=true", variant = "asan", + timeout = 45, ), Platform(; arch = "aarch64", @@ -44,6 +47,7 @@ const platforms = Platforms( commit_status = true, cmake_extra_arg = "", variant = "", + timeout = 45, ), Platform(; arch = "aarch64", @@ -53,6 +57,7 @@ const platforms = Platforms( commit_status = true, cmake_extra_arg = "-Dasan=true", variant = "asan", + timeout = 55, ), Platform(; arch = "x86_64", @@ -62,6 +67,7 @@ const platforms = Platforms( commit_status = true, cmake_extra_arg = "-Dforce32bit=true", variant = "force32bit", + timeout = 55, ), Platform(; arch = "x86_64", @@ -71,6 +77,7 @@ const platforms = Platforms( commit_status = true, cmake_extra_arg = "-Dforce32bit=true -Dasan=true", variant = "force32bit-asan", + timeout = 55, ), ] ) From ac0227e81757fd98707f1948bca59826e58d4bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= Date: Fri, 21 Apr 2023 11:11:30 +0200 Subject: [PATCH 3/6] Attempt to avoid assert in force32bit. Error message: [FATAL src/MemoryRange.h:20:MemoryRange()] start_ <= end_ --- src/AddressSpace.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/AddressSpace.cc b/src/AddressSpace.cc index 9584c235fed..c3b05ca9cfb 100644 --- a/src/AddressSpace.cc +++ b/src/AddressSpace.cc @@ -2319,6 +2319,18 @@ remote_ptr AddressSpace::chaos_mode_find_free_memory(RecordTask* t, break; } addr = overlapping_range.end(); + +#ifdef __i386__ + // Moving forward and hitting the end + if (addr > addr + ceil_page_size(len)) { + /*remote_ptr end_ = addr + ceil_page_size(len); + if (addr > end_) { + fprintf(stderr, "addr[0x%x] > end_[0x%x], ceil_page_size(len)[0x%x], len[0x%x], start[0x%x], sizeof(uintptr_t)[%d], addr_space_start[0x%x], addr_space_end[0x%x], direction[%d]\n", + addr.as_int(), end_.as_int(), ceil_page_size(len), len, start.as_int(), sizeof(uintptr_t), addr_space_start.as_int(), addr_space_end.as_int(), direction); + }*/ + break; + } +#endif } } From 930ce5b3c475b0372747c314570d0337af618b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= Date: Sun, 9 Apr 2023 11:40:04 +0200 Subject: [PATCH 4/6] ASan: Increase timeout value. CI showed timeouts with test: madvise_fracture_flags (exceeded 120s, aarch64) nested_release (exceeded 120s) record_replay (exceeded 600s) --- CMakeLists.txt | 7 +++++++ src/test/record_replay.run | 1 + src/test/util.sh | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bae57cda646..7816ba78a61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,6 +629,11 @@ if (asan) set(RR_FLAGS "${ASAN_FLAGS} ${RR_FLAGS}") endif() +file(REMOVE "${CMAKE_BINARY_DIR}/src/test/util_env.sh") +if (asan) + file(WRITE "${CMAKE_BINARY_DIR}/src/test/util_env.sh" "export RR_ASAN=true\n") +endif() + set_source_files_properties(${RR_SOURCES} PROPERTIES COMPILE_FLAGS ${RR_FLAGS}) @@ -1797,6 +1802,8 @@ if(BUILD_TESTS) string(REGEX MATCH "^.*(Xeon Phi).*$" CPU_MODEL_PHI ${CPUINFO}) if(NOT "${CPU_MODEL_PHI}" STREQUAL "") set(TEST_MONITOR_DEFAULT_TIMEOUT 480) + elseif(asan) + set(TEST_MONITOR_DEFAULT_TIMEOUT 240) else() set(TEST_MONITOR_DEFAULT_TIMEOUT 120) endif() diff --git a/src/test/record_replay.run b/src/test/record_replay.run index b609f961de9..89be7983fe2 100644 --- a/src/test/record_replay.run +++ b/src/test/record_replay.run @@ -1,5 +1,6 @@ source `dirname $0`/util.sh if [ $TIMEOUT -lt 600 ]; then TIMEOUT=600; fi +if [ "$RR_ASAN" == "true" ]; then TIMEOUT=800; fi record record_replay_subject$bitness just_record rr "--suppress-environment-warnings replay -a $workdir/*-0" replay diff --git a/src/test/util.sh b/src/test/util.sh index 7bc0933c888..bb44115d6dc 100644 --- a/src/test/util.sh +++ b/src/test/util.sh @@ -107,6 +107,10 @@ if [[ "$NO_CHECK_CACHED_MMAP" == "" ]]; then GLOBAL_OPTIONS="${GLOBAL_OPTIONS} --check-cached-mmaps" fi +if [[ -f "${OBJDIR}/src/test/util_env.sh" ]]; then + source "${OBJDIR}/src/test/util_env.sh" +fi + SRCDIR=`dirname ${BASH_SOURCE[0]}`/../.. SRCDIR=`realpath $SRCDIR` @@ -135,6 +139,9 @@ OBJDIR=`realpath $OBJDIR` TIMEOUT=$4 if [[ "$TIMEOUT" == "" ]]; then TIMEOUT=120 + if [ "$RR_ASAN" == "true" ]; then + TIMEOUT=240 + fi fi # The temporary directory we create for this test run. From 720e8854f4cbdf4a882c100acff45a8202e07ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= Date: Sun, 16 Apr 2023 22:34:29 +0200 Subject: [PATCH 5/6] ASan: Remove tests failing with `executed strex instructions detected`. Remove those tests for now. No idea how that could be avoided? Error message: [FATAL src/PerfCounters.cc:785:read_ticks()] 1 (speculatively) executed strex instructions detected. Example: https://buildkite.com/julialang/rr/builds/1141 BUILDKITE_AGENT_NAME="default-armageddon.3" --- CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7816ba78a61..9b8605c1606 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1676,6 +1676,18 @@ set(TESTS_WITHOUT_PROGRAM when ) +if (asan) + if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64") + list(REMOVE_ITEM TESTS_WITH_PROGRAM "ignore_nested") + list(REMOVE_ITEM TESTS_WITHOUT_PROGRAM "nested_detach") + list(REMOVE_ITEM TESTS_WITHOUT_PROGRAM "nested_detach_kill") # depends on nested_detach_wait + list(REMOVE_ITEM TESTS_WITH_PROGRAM "nested_detach_kill_stuck") # depends on nested_detach_wait + list(REMOVE_ITEM TESTS_WITH_PROGRAM "nested_detach_wait") + list(REMOVE_ITEM TESTS_WITHOUT_PROGRAM "nested_release") + list(REMOVE_ITEM TESTS_WITHOUT_PROGRAM "record_replay") + endif() +endif() + if(BUILD_TESTS) # Part of the installable testsuite (test files). if(INSTALL_TESTSUITE) From cc0d479b79eff9d633ca42a242ae67e0b5ab5e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= Date: Sun, 2 Apr 2023 19:44:19 +0200 Subject: [PATCH 6/6] ASan: Run with verify_asan_link_order=0 Avoids error message: ASan runtime does not come first in initial library list; you should either link runtime to your application or manually preload it with LD_PRELOAD. Affected tests: nested_detach nested_detach_kill nested_detach_kill_stuck nested_detach_stop nested_detach_wait --- src/RecordSession.cc | 3 +++ src/util.cc | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/RecordSession.cc b/src/RecordSession.cc index c419c185d81..e97d8f3fbcc 100644 --- a/src/RecordSession.cc +++ b/src/RecordSession.cc @@ -2223,6 +2223,8 @@ static ExeInfo read_exe_info(const string& exe_file) { ElfFileReader reader(fd); ret.arch = reader.arch(); + // Don't try to modifiy LD_PRELOAD when rr itself is built with ASan +#ifndef __SANITIZE_ADDRESS__ DynamicSection dynamic = reader.read_dynamic(); for (auto& entry : dynamic.entries) { if (entry.tag == DT_NEEDED && entry.val < dynamic.strtab.size()) { @@ -2236,6 +2238,7 @@ static ExeInfo read_exe_info(const string& exe_file) { } } } +#endif auto syms = reader.read_symbols(".dynsym", ".dynstr"); for (size_t i = 0; i < syms.size(); ++i) { diff --git a/src/util.cc b/src/util.cc index 059fd660530..f52d548d56d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -2461,6 +2461,10 @@ int __lsan_is_turned_off(void) { return rr_lsan_is_turned_off; } +#include +const char *__asan_default_options() { + return "verify_asan_link_order=0"; +} #endif bool coredumping_signal_takes_down_entire_vm() {