[vm, gc] Allocate card tables up front.

This avoids data races when multiple mutators compete to install the card table, and allows removing the initialization check from the write barrier.

TEST=ci, tsan
Bug: #56895
Change-Id: I30fe4687d3ca771f6c00dfa59e8dadec23d557a3
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/390023
Reviewed-by: Siva Annamalai <[email protected]>
Commit-Queue: Ryan Macnak <[email protected]>

Author: rmacnak-google

runtime/vm/app_snapshot.cc

@@ -1414,6 +1414,7 @@ class CanonicalSetDeserializationCluster : public DeserializationCluster {
     if ((SetType::Storage::ArrayCid == kArrayCid) &&
         Array::UseCardMarkingForAllocation(length)) {
       table->untag()->SetCardRememberedBitUnsynchronized();
+      Page::Of(table)->AllocateCardTable();
     }
     InitTypeArgsOrNext(table);
     table->untag()->length_ = Smi::New(length);
@@ -6508,6 +6509,7 @@ class ArrayDeserializationCluster
                                      stamp_canonical);
       if (Array::UseCardMarkingForAllocation(length)) {
         array->untag()->SetCardRememberedBitUnsynchronized();
+        Page::Of(array)->AllocateCardTable();
       }
       array->untag()->type_arguments_ =
           static_cast<TypeArgumentsPtr>(d.ReadRef());

runtime/vm/compiler/runtime_offsets_extracted.h

@@ -1859,15 +1859,13 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ Ret();
   }
   if (cards) {
-    Label remember_card_slow, retry;
+    Label retry;
 
     // Get card table.
     __ Bind(&remember_card);
     __ AndImmediate(TMP, R1, target::kPageMask);  // Page.
     __ ldr(TMP,
            Address(TMP, target::Page::card_table_offset()));  // Card table.
-    __ cmp(TMP, Operand(0));
-    __ b(&remember_card_slow, EQ);
 
     // Atomically dirty the card.
     __ PushList((1 << R0) | (1 << R1) | (1 << R2));
@@ -1890,17 +1888,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ b(&retry, EQ);
     __ PopList((1 << R0) | (1 << R1) | (1 << R2));
     __ Ret();
-
-    // Card table not yet allocated.
-    __ Bind(&remember_card_slow);
-    {
-      LeafRuntimeScope rt(assembler, /*frame_size=*/0,
-                          /*preserve_registers=*/true);
-      __ mov(R0, Operand(R1));  // Arg0 = Object
-      __ mov(R1, Operand(R9));  // Arg1 = Slot
-      rt.Call(kRememberCardRuntimeEntry, 2);
-    }
-    __ Ret();
   }
 }
 

runtime/vm/compiler/stub_code_compiler_arm.cc

@@ -2242,14 +2242,13 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ ret();
   }
   if (cards) {
-    Label remember_card_slow, retry;
+    Label retry;
 
     // Get card table.
     __ Bind(&remember_card);
     __ AndImmediate(TMP, R1, target::kPageMask);  // Page.
     __ ldr(TMP2,
            Address(TMP, target::Page::card_table_offset()));  // Card table.
-    __ cbz(&remember_card_slow, TMP2);
 
     // Atomically dirty the card.
     __ sub(R25, R25, Operand(TMP));  // Offset in page.
@@ -2271,18 +2270,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
       __ PopRegister(R0);
     }
     __ ret();
-
-    // Card table not yet allocated.
-    __ Bind(&remember_card_slow);
-    {
-      LeafRuntimeScope rt(assembler,
-                          /*frame_size=*/0,
-                          /*preserve_registers=*/true);
-      __ mov(R0, R1);   // Arg0 = Object
-      __ mov(R1, R25);  // Arg1 = Slot
-      rt.Call(kRememberCardRuntimeEntry, /*argument_count=*/2);
-    }
-    __ ret();
   }
 }
 

runtime/vm/compiler/stub_code_compiler_arm64.cc

@@ -1675,14 +1675,10 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ ret();
   }
   if (cards) {
-    Label remember_card_slow;
-
     // Get card table.
     __ Bind(&remember_card);
     __ movl(EAX, EDX);                           // Object.
     __ andl(EAX, Immediate(target::kPageMask));  // Page.
-    __ cmpl(Address(EAX, target::Page::card_table_offset()), Immediate(0));
-    __ j(EQUAL, &remember_card_slow, Assembler::kNearJump);
 
     // Atomically dirty the card.
     __ pushl(EBX);
@@ -1702,21 +1698,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ popl(ECX);
     __ popl(EAX);
     __ ret();
-
-    // Card table not yet allocated.
-    __ Bind(&remember_card_slow);
-
-    {
-      LeafRuntimeScope rt(assembler,
-                          /*frame_size=*/2 * target::kWordSize,
-                          /*preserve_registers=*/true);
-      __ movl(Address(ESP, 0 * target::kWordSize), EDX);  // Object
-      __ movl(Address(ESP, 1 * target::kWordSize), EDI);  // Slot
-      rt.Call(kRememberCardRuntimeEntry, 2);
-    }
-    __ popl(ECX);
-    __ popl(EAX);
-    __ ret();
   }
 }
 

runtime/vm/compiler/stub_code_compiler_ia32.cc

@@ -1841,14 +1841,11 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ ret();
   }
   if (cards) {
-    Label remember_card_slow;
-
     // Get card table.
     __ Bind(&remember_card);
     __ AndImmediate(TMP, A0, target::kPageMask);  // Page.
     __ lx(TMP2,
           Address(TMP, target::Page::card_table_offset()));  // Card table.
-    __ beqz(TMP2, &remember_card_slow);
 
     // Atomically dirty the card.
     __ sub(A6, A6, TMP);                               // Offset in page.
@@ -1864,17 +1861,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ amoord(ZR, TMP, Address(TMP2, 0));
 #endif
     __ ret();
-
-    // Card table not yet allocated.
-    __ Bind(&remember_card_slow);
-    {
-      LeafRuntimeScope rt(assembler, /*frame_size=*/0,
-                          /*preserve_registers=*/true);
-      __ mv(A0, A0);  // Arg0 = Object
-      __ mv(A1, A6);  // Arg1 = Slot
-      rt.Call(kRememberCardRuntimeEntry, /*argument_count=*/2);
-    }
-    __ ret();
   }
 }
 

runtime/vm/compiler/stub_code_compiler_riscv.cc

@@ -2158,14 +2158,10 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
   }
 
   if (cards) {
-    Label remember_card_slow;
-
     // Get card table.
     __ Bind(&remember_card);
     __ movq(TMP, RDX);                           // Object.
     __ andq(TMP, Immediate(target::kPageMask));  // Page.
-    __ cmpq(Address(TMP, target::Page::card_table_offset()), Immediate(0));
-    __ j(EQUAL, &remember_card_slow, Assembler::kNearJump);
 
     // Atomically dirty the card.
     __ pushq(RAX);
@@ -2183,18 +2179,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
     __ popq(RCX);
     __ popq(RAX);
     __ ret();
-
-    // Card table not yet allocated.
-    __ Bind(&remember_card_slow);
-    {
-      LeafRuntimeScope rt(assembler,
-                          /*frame_size=*/0,
-                          /*preserve_registers=*/true);
-      __ movq(CallingConventions::kArg1Reg, RDX);
-      __ movq(CallingConventions::kArg2Reg, R13);
-      rt.Call(kRememberCardRuntimeEntry, 2);
-    }
-    __ ret();
   }
 }
 

runtime/vm/compiler/stub_code_compiler_x64.cc

@@ -271,32 +271,27 @@ class Page {
   bool IsSurvivor(uword raw_addr) const { return raw_addr < survivor_end_; }
   bool IsResolved() const { return top_ == resolved_top_; }
 
+  void AllocateCardTable() {
+    ASSERT(card_table_ == nullptr);
+    ASSERT(is_large());
+    size_t size_in_bits = card_table_size();
+    size_t size_in_bytes =
+        Utils::RoundUp(size_in_bits, kBitsPerWord) >> kBitsPerByteLog2;
+    card_table_ = reinterpret_cast<RelaxedAtomic<uword>*>(
+        calloc(size_in_bytes, sizeof(uint8_t)));
+  }
+
  private:
   void RememberCard(uword slot) {
     ASSERT(Contains(slot));
-    uword* card_table = card_table_.load();
-    if (card_table == nullptr) {
-      size_t size_in_bits = card_table_size();
-      size_t size_in_bytes =
-          Utils::RoundUp(size_in_bits, kBitsPerWord) >> kBitsPerByteLog2;
-      uword* new_card_table =
-          reinterpret_cast<uword*>(calloc(size_in_bytes, sizeof(uint8_t)));
-      if (card_table_.compare_exchange_strong(card_table, new_card_table)) {
-        card_table = new_card_table;
-      } else {
-        // Lost race.
-        ASSERT(card_table != nullptr);
-        free(new_card_table);
-      }
-    }
+    ASSERT(card_table_ != nullptr);
     intptr_t offset = slot - reinterpret_cast<uword>(this);
     intptr_t index = offset >> kBytesPerCardLog2;
     ASSERT((index >= 0) && (index < card_table_size()));
     intptr_t word_offset = index >> kBitsPerWordLog2;
     intptr_t bit_offset = index & (kBitsPerWord - 1);
     uword bit_mask = static_cast<uword>(1) << bit_offset;
-    reinterpret_cast<std::atomic<uword>*>(&card_table[word_offset])
-        ->fetch_or(bit_mask, std::memory_order_relaxed);
+    card_table_[word_offset].fetch_or(bit_mask);
   }
   bool IsCardRemembered(uword slot) {
     ASSERT(Contains(slot));
@@ -309,7 +304,7 @@ class Page {
     intptr_t word_offset = index >> kBitsPerWordLog2;
     intptr_t bit_offset = index & (kBitsPerWord - 1);
     uword bit_mask = static_cast<uword>(1) << bit_offset;
-    return (card_table_[word_offset] & bit_mask) != 0;
+    return (card_table_[word_offset].load() & bit_mask) != 0;
   }
 
   void set_object_end(uword value) {
@@ -328,7 +323,7 @@ class Page {
   VirtualMemory* memory_;
   Page* next_;
   ForwardingPage* forwarding_page_;
-  RelaxedAtomic<uword*> card_table_;  // Remembered set, not marking.
+  RelaxedAtomic<uword>* card_table_;  // Remembered set, not marking.
   RelaxedAtomic<intptr_t> progress_bar_;
 
   // The thread using this page for allocation, otherwise nullptr.

runtime/vm/heap/page.h

@@ -25248,6 +25248,7 @@ ArrayPtr Array::NewUninitialized(intptr_t class_id,
   if (UseCardMarkingForAllocation(len)) {
     ASSERT(raw->IsOldObject());
     raw->untag()->SetCardRememberedBitUnsynchronized();
+    Page::Of(raw)->AllocateCardTable();
   }
   return raw;
 }

runtime/vm/object.cc

@@ -1607,6 +1607,7 @@ class SlowObjectCopyBase : public ObjectCopyBase {
     if ((cid == kArrayCid || cid == kImmutableArrayCid) &&
         !IsAllocatableInNewSpace(size)) {
       to.untag()->SetCardRememberedBitUnsynchronized();
+      Page::Of(to)->AllocateCardTable();
     }
     if (IsExternalTypedDataClassId(cid)) {
       const auto& external_to = slow_forward_map_.AddExternalTypedData(