[vm, gc] Add write-write fence after initializing object headers, or defer marking new-space objects in an active TLAB.

rmacnak-google · Commit Queue · commit 949eb447a47b · 2024-10-28T22:41:39.000Z
On architectures with a weak memory model, there is the possibility that the concurrent marker can see the publishing store of a new object before it sees the initializing store of that object's header. On an M1, the barrier to prevent reordering these stores is fairly cheap, so we emit this barrier on Mac/iOS ARM64. Otherwise, this barrier is very expensive (or at least expensive for some hardware within the ABI), so instead we avoid the race by deferring marking of objects inside an active TLAB. Disable TSAN instrumentation on the marker setting the mark bit, as TSAN does not understand fences. TEST=ooo arm64 machines Bug: #56845 Change-Id: I0676661a7cf941fdc6b451e516d890c26826bb3b Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/389265 Commit-Queue: Ryan Macnak <rmacnak@google.com> Reviewed-by: Siva Annamalai <asiva@google.com>
diff --git a/runtime/vm/bitfield.h b/runtime/vm/bitfield.h
@@ -113,6 +113,13 @@ class AtomicBitFieldContainer {
     return TargetBitField::decode(old_field);
   }
 
+  template <class TargetBitField>
+  NO_SANITIZE_THREAD bool TryClearIgnoreRace() {
+    T mask = ~TargetBitField::encode(true);
+    T old_field = field_.fetch_and(mask, std::memory_order_relaxed);
+    return TargetBitField::decode(old_field);
+  }
+
  private:
   std::atomic<T> field_;
 };
diff --git a/runtime/vm/compiler/asm_intrinsifier_arm.cc b/runtime/vm/compiler/asm_intrinsifier_arm.cc
@@ -1558,7 +1558,7 @@ static void TryAllocateString(Assembler* assembler,
         target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
     __ LoadImmediate(TMP, tags);
     __ orr(R3, R3, Operand(TMP));
-    __ str(R3, FieldAddress(R0, target::Object::tags_offset()));  // Store tags.
+    __ InitializeHeader(R3, R0);
   }
 
   // Set the length field using the saved length (R8).
diff --git a/runtime/vm/compiler/asm_intrinsifier_arm64.cc b/runtime/vm/compiler/asm_intrinsifier_arm64.cc
@@ -1778,7 +1778,7 @@ static void TryAllocateString(Assembler* assembler,
         target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
     __ LoadImmediate(TMP, tags);
     __ orr(R2, R2, Operand(TMP));
-    __ str(R2, FieldAddress(R0, target::Object::tags_offset()));  // Store tags.
+    __ InitializeHeader(R2, R0);
   }
 
 #if DART_COMPRESSED_POINTERS
diff --git a/runtime/vm/compiler/asm_intrinsifier_ia32.cc b/runtime/vm/compiler/asm_intrinsifier_ia32.cc
@@ -1602,7 +1602,7 @@ static void TryAllocateString(Assembler* assembler,
     const uword tags =
         target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
     __ orl(EDI, Immediate(tags));
-    __ movl(FieldAddress(EAX, target::Object::tags_offset()), EDI);  // Tags.
+    __ InitializeHeader(EDI, EAX);
   }
 
   // Set the length field.
diff --git a/runtime/vm/compiler/asm_intrinsifier_riscv.cc b/runtime/vm/compiler/asm_intrinsifier_riscv.cc
@@ -1816,7 +1816,7 @@ static void TryAllocateString(Assembler* assembler,
     const uword tags =
         target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
     __ OrImmediate(A1, A1, tags);
-    __ sx(A1, FieldAddress(A0, target::Object::tags_offset()));  // Store tags.
+    __ InitializeHeader(A1, A0);
   }
 
   // Set the length field using the saved length (T0).
diff --git a/runtime/vm/compiler/asm_intrinsifier_x64.cc b/runtime/vm/compiler/asm_intrinsifier_x64.cc
@@ -1677,7 +1677,7 @@ static void TryAllocateString(Assembler* assembler,
     const uword tags =
         target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
     __ orq(RDI, Immediate(tags));
-    __ movq(FieldAddress(RAX, target::Object::tags_offset()), RDI);  // Tags.
+    __ InitializeHeader(RDI, RAX);
   }
 
   // Set the length field.
diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc
@@ -3628,7 +3628,7 @@ void Assembler::TryAllocateObject(intptr_t cid,
 
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
     LoadImmediate(temp_reg, tags);
-    str(temp_reg, FieldAddress(instance_reg, target::Object::tags_offset()));
+    InitializeHeader(temp_reg, instance_reg);
   } else {
     b(failure);
   }
@@ -3671,8 +3671,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
     // instance: new object start as a tagged pointer.
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
     LoadImmediate(temp2, tags);
-    str(temp2,
-        FieldAddress(instance, target::Object::tags_offset()));  // Store tags.
+    InitializeHeader(temp2, instance);
   } else {
     b(failure);
   }
diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h
@@ -1034,6 +1034,19 @@ class Assembler : public AssemblerBase {
   }
   void CompareObject(Register rn, const Object& object);
 
+  void InitializeHeader(Register tags, Register object) {
+    str(tags, FieldAddress(object, target::Object::tags_offset()));
+#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
+    dmb();
+#endif
+  }
+  void InitializeHeaderUntagged(Register tags, Register object) {
+    str(tags, Address(object, target::Object::tags_offset()));
+#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
+    dmb();
+#endif
+  }
+
   void StoreObjectIntoObjectNoBarrier(
       Register object,
       const Address& dest,
diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc
@@ -1969,7 +1969,7 @@ void Assembler::TryAllocateObject(intptr_t cid,
 
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
     LoadImmediate(temp_reg, tags);
-    Store(temp_reg, FieldAddress(instance_reg, target::Object::tags_offset()));
+    InitializeHeader(temp_reg, instance_reg);
   } else {
     b(failure);
   }
@@ -2011,7 +2011,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
     // instance: new object start as a tagged pointer.
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
     LoadImmediate(temp2, tags);
-    str(temp2, FieldAddress(instance, target::Object::tags_offset()));
+    InitializeHeader(temp2, instance);
   } else {
     b(failure);
   }
diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h
@@ -1232,6 +1232,8 @@ class Assembler : public AssemblerBase {
   // Breakpoint.
   void brk(uint16_t imm) { EmitExceptionGenOp(BRK, imm); }
 
+  void dmb() { Emit(kDataMemoryBarrier); }
+
   // Double floating point.
   bool fmovdi(VRegister vd, double immd) {
     int64_t imm64 = bit_cast<int64_t, double>(immd);
@@ -1948,6 +1950,19 @@ class Assembler : public AssemblerBase {
   void LoadCompressed(Register dest, const Address& slot) override;
 #endif
 
+  void InitializeHeader(Register header, Register object) {
+    str(header, FieldAddress(object, target::Object::tags_offset()));
+#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
+    dmb();
+#endif
+  }
+  void InitializeHeaderUntagged(Register header, Register object) {
+    str(header, Address(object, target::Object::tags_offset()));
+#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
+    dmb();
+#endif
+  }
+
   void StoreBarrier(Register object,
                     Register value,
                     CanBeSmi can_value_be_smi,
diff --git a/runtime/vm/compiler/assembler/assembler_ia32.cc b/runtime/vm/compiler/assembler/assembler_ia32.cc
@@ -2817,8 +2817,7 @@ void Assembler::TryAllocateObject(intptr_t cid,
     ASSERT(instance_size >= kHeapObjectTag);
     subl(instance_reg, Immediate(instance_size - kHeapObjectTag));
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
-    movl(FieldAddress(instance_reg, target::Object::tags_offset()),
-         Immediate(tags));
+    InitializeHeader(Immediate(tags), instance_reg);
   } else {
     jmp(failure);
   }
@@ -2859,8 +2858,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
 
     // Initialize the tags.
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
-    movl(FieldAddress(instance, target::Object::tags_offset()),
-         Immediate(tags));
+    InitializeHeader(Immediate(tags), instance);
   } else {
     jmp(failure);
   }
diff --git a/runtime/vm/compiler/assembler/assembler_ia32.h b/runtime/vm/compiler/assembler/assembler_ia32.h
@@ -820,6 +820,15 @@ class Assembler : public AssemblerBase {
   void PushObject(const Object& object);
   void CompareObject(Register reg, const Object& object);
 
+  void InitializeHeader(Register tags, Register object) {
+    movl(FieldAddress(object, target::Object::tags_offset()), tags);
+    // No fence: all stores are ordered on ia32.
+  }
+  void InitializeHeader(Immediate tags, Register object) {
+    movl(FieldAddress(object, target::Object::tags_offset()), tags);
+    // No fence: all stores are ordered on ia32.
+  }
+
   void StoreObjectIntoObjectNoBarrier(
       Register object,
       const Address& dest,
diff --git a/runtime/vm/compiler/assembler/assembler_riscv.cc b/runtime/vm/compiler/assembler/assembler_riscv.cc
@@ -4551,7 +4551,7 @@ void Assembler::TryAllocateObject(intptr_t cid,
 
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
     LoadImmediate(temp_reg, tags);
-    Store(temp_reg, FieldAddress(instance_reg, target::Object::tags_offset()));
+    InitializeHeader(temp_reg, instance_reg);
   } else {
     j(failure, distance);
   }
@@ -4592,7 +4592,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
     // instance: new object start as a tagged pointer.
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
     LoadImmediate(temp2, tags);
-    sx(temp2, FieldAddress(instance, target::Object::tags_offset()));
+    InitializeHeader(temp2, instance);
   } else {
     j(failure);
   }
diff --git a/runtime/vm/compiler/assembler/assembler_riscv.h b/runtime/vm/compiler/assembler/assembler_riscv.h
@@ -1193,6 +1193,19 @@ class Assembler : public MicroAssembler {
     UNREACHABLE();
   }
 
+  void InitializeHeader(Register tags, Register object) {
+    sx(tags, FieldAddress(object, target::Object::tags_offset()));
+#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
+    fence(kWrite, kWrite);
+#endif
+  }
+  void InitializeHeaderUntagged(Register tags, Register object) {
+    sx(tags, Address(object, target::Object::tags_offset()));
+#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
+    fence(kWrite, kWrite);
+#endif
+  }
+
   void StoreBarrier(Register object,
                     Register value,
                     CanBeSmi can_value_be_smi,
diff --git a/runtime/vm/compiler/assembler/assembler_x64.cc b/runtime/vm/compiler/assembler/assembler_x64.cc
@@ -2419,8 +2419,7 @@ void Assembler::TryAllocateObject(intptr_t cid,
     ASSERT(instance_size >= kHeapObjectTag);
     AddImmediate(instance_reg, Immediate(kHeapObjectTag - instance_size));
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
-    MoveImmediate(FieldAddress(instance_reg, target::Object::tags_offset()),
-                  Immediate(tags));
+    InitializeHeader(Immediate(tags), instance_reg);
   } else {
     jmp(failure);
   }
@@ -2461,8 +2460,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
     // Initialize the tags.
     // instance: new object start as a tagged pointer.
     const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
-    movq(FieldAddress(instance, target::Object::tags_offset()),
-         Immediate(tags));
+    InitializeHeader(Immediate(tags), instance);
   } else {
     jmp(failure);
   }
diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h
@@ -867,6 +867,19 @@ class Assembler : public AssemblerBase {
 
   void Call(Address target) { call(target); }
 
+  void InitializeHeader(Register tags, Register object) {
+    movq(FieldAddress(object, target::Object::tags_offset()), tags);
+    // No fence: all stores are ordered on x64.
+  }
+  void InitializeHeaderUntagged(Register tags, Register object) {
+    movq(Address(object, target::Object::tags_offset()), tags);
+    // No fence: all stores are ordered on x64.
+  }
+  void InitializeHeader(Immediate tags, Register object) {
+    movq(FieldAddress(object, target::Object::tags_offset()), tags);
+    // No fence: all stores are ordered on x64.
+  }
+
   // Unaware of write barrier (use StoreInto* methods for storing to objects).
   // TODO(koda): Add StackAddress/HeapAddress types to prevent misuse.
   void StoreObject(const Address& dst,
diff --git a/runtime/vm/compiler/assembler/disassembler_arm64.cc b/runtime/vm/compiler/assembler/disassembler_arm64.cc
@@ -994,6 +994,11 @@ void ARM64Decoder::DecodeSystem(Instr* instr) {
     return;
   }
 
+  if (instr->InstructionBits() == kDataMemoryBarrier) {
+    Format(instr, "dmb ish");
+    return;
+  }
+
   if ((instr->Bits(0, 8) == 0x1f) && (instr->Bits(12, 4) == 2) &&
       (instr->Bits(16, 3) == 3) && (instr->Bits(19, 2) == 0) &&
       (instr->Bit(21) == 0)) {
diff --git a/runtime/vm/compiler/stub_code_compiler.cc b/runtime/vm/compiler/stub_code_compiler.cc
@@ -1383,8 +1383,7 @@ void StubCodeCompiler::GenerateAllocateRecordStub() {
       __ Bind(&done);
       uword tags = target::MakeTagWordForNewSpaceObject(kRecordCid, 0);
       __ OrImmediate(temp_reg, tags);
-      __ StoreFieldToOffset(temp_reg, result_reg,
-                            target::Object::tags_offset());  // Tags.
+      __ InitializeHeader(temp_reg, result_reg);
     }
 
     __ StoreCompressedIntoObjectNoBarrier(
@@ -1863,8 +1862,7 @@ static void GenerateAllocateSuspendState(Assembler* assembler,
     __ Bind(&done);
     uword tags = target::MakeTagWordForNewSpaceObject(kSuspendStateCid, 0);
     __ OrImmediate(temp_reg, tags);
-    __ StoreFieldToOffset(temp_reg, result_reg,
-                          target::Object::tags_offset());  // Tags.
+    __ InitializeHeader(temp_reg, result_reg);
   }
 
   __ StoreFieldToOffset(frame_size_reg, result_reg,
diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc
@@ -1123,8 +1123,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub() {
           target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
       __ LoadImmediate(TMP, tags);
       __ orr(R8, R8, Operand(TMP));
-      __ str(R8, FieldAddress(AllocateArrayABI::kResultReg,
-                              target::Array::tags_offset()));  // Store tags.
+      __ InitializeHeader(R8, AllocateArrayABI::kResultReg);
     }
 
     // AllocateArrayABI::kResultReg: new object start as a tagged pointer.
@@ -1556,7 +1555,7 @@ static void GenerateAllocateContext(Assembler* assembler, Label* slow_case) {
 
   __ LoadImmediate(IP, tags);
   __ orr(R9, R9, Operand(IP));
-  __ str(R9, FieldAddress(R0, target::Object::tags_offset()));
+  __ InitializeHeader(R9, R0);
 
   // Setup up number of context variables field.
   // R0: new object.
@@ -1942,8 +1941,7 @@ static void GenerateAllocateObjectHelper(Assembler* assembler,
     }  //  kEndReg = R1, kInstanceSizeReg = R9
 
     // Tags.
-    __ str(kTagsReg, Address(AllocateObjectABI::kResultReg,
-                             target::Object::tags_offset()));
+    __ InitializeHeaderUntagged(kTagsReg, AllocateObjectABI::kResultReg);
 
     // Initialize the remaining words of the object.
     {
@@ -3548,7 +3546,7 @@ void StubCodeCompiler::GenerateAllocateTypedDataArrayStub(intptr_t cid) {
           target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
       __ LoadImmediate(TMP, tags);
       __ orr(R3, R3, Operand(TMP));
-      __ str(R3, FieldAddress(R0, target::Object::tags_offset())); /* Tags. */
+      __ InitializeHeader(R3, R0);
     }
     /* Set the length field. */
     /* R0: new object start as a tagged pointer. */
diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc
@@ -1444,8 +1444,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub() {
 
     __ LoadImmediate(TMP, tags);
     __ orr(R3, R3, Operand(TMP));
-    __ StoreFieldToOffset(R3, AllocateArrayABI::kResultReg,
-                          target::Array::tags_offset());
+    __ InitializeHeader(R3, AllocateArrayABI::kResultReg);
 
     // Store the type argument field.
     __ StoreCompressedIntoObjectOffsetNoBarrier(
@@ -1913,7 +1912,7 @@ static void GenerateAllocateContextSpaceStub(Assembler* assembler,
 
   __ LoadImmediate(TMP, tags);
   __ orr(R2, R2, Operand(TMP));
-  __ StoreFieldToOffset(R2, R0, target::Object::tags_offset());
+  __ InitializeHeader(R2, R0);
 
   // Setup up number of context variables field.
   // R0: new object.
@@ -2324,8 +2323,7 @@ static void GenerateAllocateObjectHelper(Assembler* assembler,
     }  // kInstanceSizeReg = R4, kEndReg = R5
 
     // Tags.
-    __ str(kTagsReg, Address(AllocateObjectABI::kResultReg,
-                             target::Object::tags_offset()));
+    __ InitializeHeaderUntagged(kTagsReg, AllocateObjectABI::kResultReg);
 
     // Initialize the remaining words of the object.
     {
@@ -3953,7 +3951,7 @@ void StubCodeCompiler::GenerateAllocateTypedDataArrayStub(intptr_t cid) {
           target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
       __ LoadImmediate(TMP, tags);
       __ orr(R2, R2, Operand(TMP));
-      __ str(R2, FieldAddress(R0, target::Object::tags_offset())); /* Tags. */
+      __ InitializeHeader(R2, R0);
     }
     /* Set the length field. */
     /* R0: new object start as a tagged pointer. */
diff --git a/runtime/vm/compiler/stub_code_compiler_ia32.cc b/runtime/vm/compiler/stub_code_compiler_ia32.cc
@@ -960,9 +960,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub() {
       // Get the class index and insert it into the tags.
       uword tags = target::MakeTagWordForNewSpaceObject(cid, 0);
       __ orl(EDI, Immediate(tags));
-      __ movl(FieldAddress(AllocateArrayABI::kResultReg,
-                           target::Object::tags_offset()),
-              EDI);  // Tags.
+      __ InitializeHeader(EDI, AllocateArrayABI::kResultReg);
     }
     // AllocateArrayABI::kResultReg: new object start as a tagged pointer.
     // EBX: allocation size.
@@ -1363,7 +1361,7 @@ static void GenerateAllocateContextSpaceStub(Assembler* assembler,
     // EBX: size and bit tags.
     uword tags = target::MakeTagWordForNewSpaceObject(kContextCid, 0);
     __ orl(EBX, Immediate(tags));
-    __ movl(FieldAddress(EAX, target::Object::tags_offset()), EBX);  // Tags.
+    __ InitializeHeader(EBX, EAX);
   }
 
   // Setup up number of context variables field.
diff --git a/runtime/vm/compiler/stub_code_compiler_riscv.cc b/runtime/vm/compiler/stub_code_compiler_riscv.cc
diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc
diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h
diff --git a/runtime/vm/globals.h b/runtime/vm/globals.h
diff --git a/runtime/vm/heap/marker.cc b/runtime/vm/heap/marker.cc
diff --git a/runtime/vm/heap/pages.h b/runtime/vm/heap/pages.h
diff --git a/runtime/vm/heap/scavenger.cc b/runtime/vm/heap/scavenger.cc
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc
diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h
diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc

Original file line number	Diff line number	Diff line change
`@@ -1558,7 +1558,7 @@ static void TryAllocateString(Assembler* assembler,`
`1558`	`1558`	`target::MakeTagWordForNewSpaceObject(cid, /instance_size=/0);`
`1559`	`1559`	`__ LoadImmediate(TMP, tags);`
`1560`	`1560`	`__ orr(R3, R3, Operand(TMP));`
`1561`		`- __ str(R3, FieldAddress(R0, target::Object::tags_offset())); // Store tags.`
	`1561`	`+ __ InitializeHeader(R3, R0);`
`1562`	`1562`	`}`
`1563`	`1563`
`1564`	`1564`	`// Set the length field using the saved length (R8).`
Original file line number	Diff line number	Diff line change
`@@ -1778,7 +1778,7 @@ static void TryAllocateString(Assembler* assembler,`
`1778`	`1778`	`target::MakeTagWordForNewSpaceObject(cid, /instance_size=/0);`
`1779`	`1779`	`__ LoadImmediate(TMP, tags);`
`1780`	`1780`	`__ orr(R2, R2, Operand(TMP));`
`1781`		`- __ str(R2, FieldAddress(R0, target::Object::tags_offset())); // Store tags.`
	`1781`	`+ __ InitializeHeader(R2, R0);`
`1782`	`1782`	`}`
`1783`	`1783`
`1784`	`1784`	`#if DART_COMPRESSED_POINTERS`
Original file line number	Diff line number	Diff line change
`@@ -1602,7 +1602,7 @@ static void TryAllocateString(Assembler* assembler,`
`1602`	`1602`	`const uword tags =`
`1603`	`1603`	`target::MakeTagWordForNewSpaceObject(cid, /instance_size=/0);`
`1604`	`1604`	`__ orl(EDI, Immediate(tags));`
`1605`		`- __ movl(FieldAddress(EAX, target::Object::tags_offset()), EDI); // Tags.`
	`1605`	`+ __ InitializeHeader(EDI, EAX);`
`1606`	`1606`	`}`
`1607`	`1607`
`1608`	`1608`	`// Set the length field.`
Original file line number	Diff line number	Diff line change
`@@ -1816,7 +1816,7 @@ static void TryAllocateString(Assembler* assembler,`
`1816`	`1816`	`const uword tags =`
`1817`	`1817`	`target::MakeTagWordForNewSpaceObject(cid, /instance_size=/0);`
`1818`	`1818`	`__ OrImmediate(A1, A1, tags);`
`1819`		`- __ sx(A1, FieldAddress(A0, target::Object::tags_offset())); // Store tags.`
	`1819`	`+ __ InitializeHeader(A1, A0);`
`1820`	`1820`	`}`
`1821`	`1821`
`1822`	`1822`	`// Set the length field using the saved length (T0).`
Original file line number	Diff line number	Diff line change
`@@ -1677,7 +1677,7 @@ static void TryAllocateString(Assembler* assembler,`
`1677`	`1677`	`const uword tags =`
`1678`	`1678`	`target::MakeTagWordForNewSpaceObject(cid, /instance_size=/0);`
`1679`	`1679`	`__ orq(RDI, Immediate(tags));`
`1680`		`- __ movq(FieldAddress(RAX, target::Object::tags_offset()), RDI); // Tags.`
	`1680`	`+ __ InitializeHeader(RDI, RAX);`
`1681`	`1681`	`}`
`1682`	`1682`
`1683`	`1683`	`// Set the length field.`