Do not try to duplicate too big mappings, used by address sanitizer.

Visible in recording of nested_detach test with an asan enabled rr build.

1202: =================================================================
1202: ==901358==ERROR: AddressSanitizer: requested allocation size 0x20000000000 (0x20000001000 after adjustments for alignment, red zones etc.) exceeds maximum supported size of 0x10000000000 (thread T0)
1202:      0 0x7faf22267647 in operator new(unsigned long) ../../../../src/libsanitizer/asan/asan_new_delete.cpp:99
1202:      1 0x562cdcc49466 in __gnu_cxx::new_allocator<char>::allocate(unsigned long, void const*) /usr/include/c++/10/ext/new_allocator.h:115
1202:      2 0x562cdcc48ad5 in std::allocator_traits<std::allocator<char> >::allocate(std::allocator<char>&, unsigned long) /usr/include/c++/10/bits/alloc_traits.h:460
1202:      3 0x562cdcc497fd in std::_Vector_base<char, std::allocator<char> >::_M_allocate(unsigned long) /usr/include/c++/10/bits/stl_vector.h:346
1202:      4 0x562cdcc48f34 in std::vector<char, std::allocator<char> >::_M_default_append(unsigned long) /usr/include/c++/10/bits/vector.tcc:635
1202:      5 0x562cdcc488ae in std::vector<char, std::allocator<char> >::resize(unsigned long) /usr/include/c++/10/bits/stl_vector.h:940
1202:      6 0x562cdd087cb8 in copy_mem_mapping /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/Task.cc:3575
1202:      7 0x562cdd089c23 in rr::Task::dup_from(rr::Task*) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/Task.cc:3680
1202:      8 0x562cdce1597b in do_detach_teleport /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:3355
1202:      9 0x562cdce3c38c in rec_prepare_syscall_arch<rr::X64Arch> /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4796
1202:     10 0x562cdce1620e in operator() /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4935
1202:     11 0x562cdce49064 in with_converted_registers<rr::Switchable, rr::rec_prepare_syscall_internal(rr::RecordTask*, rr::TaskSyscallState&)::<lambda(const rr::Registers&)> > /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/Registers.h:604
1202:     12 0x562cdce163eb in rec_prepare_syscall_internal /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4933
1202:     13 0x562cdce16496 in rr::rec_prepare_syscall(rr::RecordTask*) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4944
1202:     14 0x562cdcdd7ea1 in rr::RecordSession::syscall_state_changed(rr::RecordTask*, rr::RecordSession::StepState*) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordSession.cc:1076
1202:     15 0x562cdcde5f8d in rr::RecordSession::record_step() /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordSession.cc:2401
1202:     16 0x562cdcdc7a48 in record /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordCommand.cc:656
1202:     17 0x562cdcdc935a in rr::RecordCommand::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordCommand.cc:791
1202:     18 0x562cdd13aa60 in main /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/main.cc:249
1202:     19 0x7faf21b97d09 in __libc_start_main ../csu/libc-start.c:308
1202:
1202: ==901358==HINT: if you don't care about these errors you may set allocator_may_return_null=1
1202: SUMMARY: AddressSanitizer: allocation-size-too-big ../../../../src/libsanitizer/asan/asan_new_delete.cpp:99 in operator new(unsigned long)
1202: ==901358==ABORTING

Author: bernhardu

src/Task.cc

@@ -3671,6 +3671,12 @@ void Task::dup_from(Task *other) {
       create_mapping(this, remote_this, km);
       LOG(debug) << "Copying mapping into " << tid;
       if (!(km.flags() & MAP_SHARED)) {
+#if defined(__x86_64__) && defined(__SANITIZE_ADDRESS__)
+        if (km.size() >= 0x100000000) {
+          LOG(warn) << "Not duplicating too big mapping of ASAN shadow or allocator reserve:" << km;
+          continue;
+        }
+#endif
         copy_mem_mapping(other, this, km);
       }
     }