[vm] Fix TSAN crashes due to TSAN's own overflow.

TEST=vm/cc/DartAPI_StackOverflow
Change-Id: If5456f4f0a194332b38901f9bd677f3cd9af561c
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/456421
Commit-Queue: Ryan Macnak <[email protected]>
Reviewed-by: Alexander Markov <[email protected]>

Author: rmacnak-google

runtime/vm/os_thread.cc

@@ -7,6 +7,7 @@
 #include "platform/address_sanitizer.h"
 #include "platform/atomic.h"
 #include "platform/memory_sanitizer.h"
+#include "platform/thread_sanitizer.h"
 #include "vm/lockers.h"
 #include "vm/log.h"
 #include "vm/thread_interrupter.h"
@@ -52,6 +53,20 @@ OSThread::OSThread()
 
   stack_headroom_ = CalculateHeadroom(stack_base_ - stack_limit_);
 
+#if defined(USING_THREAD_SANITIZER)
+  // The Mac default stack size of 8 MB results in TSAN's stack recording
+  // overflowing in some tests before Dart reaches regular stack overflow.
+  // Clamp the stack size used by Dart so we hit a Dart stack overflow
+  // exception before corrupting TSAN. This is only relevant in run_vm_tests,
+  // which runs Dart on the main thread. The standalone embedder runs even the
+  // main isolate on thread pool workers, which the VM created with a smaller
+  // stack size.
+  const size_t kMaxDartStackSize = GetMaxStackSize();
+  if (stack_base_ - stack_limit_ - stack_headroom_ > kMaxDartStackSize) {
+    stack_headroom_ = stack_base_ - stack_limit_ - kMaxDartStackSize;
+  }
+#endif
+
   ASSERT(stack_base_ != 0);
   ASSERT(stack_limit_ != 0);
   ASSERT(stack_base_ > stack_limit_);