Skip to content

Commit dc97aa7

Browse files
rmacnak-googleCommit Queue
rmacnak-google
authored and
Commit Queue
committed
[vm, compiler] Update TSAN instrumentation.
TEST=tsan Cq-Include-Trybots: luci.dart.try:vm-tsan-linux-release-x64-try,vm-tsan-linux-release-arm64-try,iso-stress-linux-arm64-try,iso-stress-linux-x64-try Change-Id: I0c49b501c53f9a5117bb14800e342e6fe9b4eba6 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/442730 Reviewed-by: Alexander Aprelev <[email protected]> Commit-Queue: Ryan Macnak <[email protected]>
1 parent 8b97e6c commit dc97aa7

File tree

13 files changed

+551
-405
lines changed

13 files changed

+551
-405
lines changed

‎runtime/platform/thread_sanitizer.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,14 @@
1717

1818
#if defined(USING_THREAD_SANITIZER)
1919
#define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
20-
extern "C" void __tsan_acquire(void* addr);
21-
extern "C" void __tsan_release(void* addr);
20+
extern "C" void __tsan_atomic32_load(uint32_t* addr, int order);
21+
extern "C" void __tsan_atomic32_store(uint32_t* addr,
22+
uint32_t value,
23+
int order);
24+
extern "C" void __tsan_atomic64_load(uint64_t* addr, int order);
25+
extern "C" void __tsan_atomic64_store(uint64_t* addr,
26+
uint64_t value,
27+
int order);
2228
#else
2329
#define NO_SANITIZE_THREAD
2430
#endif

‎runtime/vm/compiler/assembler/assembler_arm64.cc‎

Lines changed: 60 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -286,16 +286,69 @@ void Assembler::Align(intptr_t alignment, intptr_t offset) {
286286
ASSERT(((offset + buffer_.GetPosition()) & (alignment - 1)) == 0);
287287
}
288288

289-
void Assembler::TsanLoadAcquire(Register addr) {
290-
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
289+
void Assembler::TsanLoadAcquire(Register dst, Register addr, OperandSize size) {
290+
RegisterSet registers(kDartVolatileCpuRegs & ~(1 << dst),
291+
kAllFpuRegistersList);
292+
293+
EnterFrame(0);
294+
PushRegisters(registers);
295+
ReserveAlignedFrameSpace(0);
296+
291297
MoveRegister(R0, addr);
292-
rt.Call(kTsanLoadAcquireRuntimeEntry, /*argument_count=*/1);
298+
LoadImmediate(R1, __ATOMIC_ACQUIRE);
299+
300+
mov(CSP, SP);
301+
switch (size) {
302+
case kEightBytes:
303+
ldr(TMP, compiler::Address(
304+
THR, kTsanAtomic64LoadRuntimeEntry.OffsetFromThread()));
305+
break;
306+
case kUnsignedFourBytes:
307+
ldr(TMP, compiler::Address(
308+
THR, kTsanAtomic32LoadRuntimeEntry.OffsetFromThread()));
309+
break;
310+
default:
311+
UNIMPLEMENTED();
312+
}
313+
str(TMP, compiler::Address(THR, target::Thread::vm_tag_offset()));
314+
blr(TMP);
315+
LoadImmediate(TMP, VMTag::kDartTagId);
316+
str(TMP, compiler::Address(THR, target::Thread::vm_tag_offset()));
317+
SetupCSPFromThread(THR);
318+
319+
MoveRegister(dst, R0);
320+
321+
AddImmediate(SP, FP, -registers.SpillSize());
322+
PopRegisters(registers);
323+
LeaveFrame();
293324
}
294325

295-
void Assembler::TsanStoreRelease(Register addr) {
326+
void Assembler::TsanStoreRelease(Register src,
327+
Register addr,
328+
OperandSize size) {
296329
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
297-
MoveRegister(R0, addr);
298-
rt.Call(kTsanStoreReleaseRuntimeEntry, /*argument_count=*/1);
330+
331+
if (src == R0) {
332+
MoveRegister(R1, src);
333+
MoveRegister(R0, addr);
334+
} else {
335+
MoveRegister(R0, addr);
336+
MoveRegister(R1, src);
337+
}
338+
LoadImmediate(R2, __ATOMIC_RELEASE);
339+
340+
switch (size) {
341+
case kEightBytes:
342+
rt.Call(kTsanAtomic64StoreRuntimeEntry, /*argument_count=*/3);
343+
break;
344+
case kFourBytes:
345+
case kUnsignedFourBytes:
346+
rt.Call(kTsanAtomic32StoreRuntimeEntry, /*argument_count=*/3);
347+
break;
348+
default:
349+
UNIMPLEMENTED();
350+
break;
351+
}
299352
}
300353

301354
static int CountLeadingZeros(uint64_t value, int width) {
@@ -1779,10 +1832,7 @@ LeafRuntimeScope::~LeafRuntimeScope() {
17791832
// SP might have been modified to reserve space for arguments
17801833
// and ensure proper alignment of the stack frame.
17811834
// We need to restore it before restoring registers.
1782-
const intptr_t kPushedRegistersSize =
1783-
kRuntimeCallSavedRegisters.CpuRegisterCount() * target::kWordSize +
1784-
kRuntimeCallSavedRegisters.FpuRegisterCount() * kFpuRegisterSize;
1785-
__ AddImmediate(SP, FP, -kPushedRegistersSize);
1835+
__ AddImmediate(SP, FP, -kRuntimeCallSavedRegisters.SpillSize());
17861836
__ PopRegisters(kRuntimeCallSavedRegisters);
17871837
}
17881838

‎runtime/vm/compiler/assembler/assembler_arm64.h‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -502,8 +502,8 @@ class Assembler : public AssemblerBase {
502502
StoreToOffset(src, base, offset, kEightBytes);
503503
}
504504

505-
void TsanLoadAcquire(Register addr);
506-
void TsanStoreRelease(Register addr);
505+
void TsanLoadAcquire(Register dst, Register addr, OperandSize size);
506+
void TsanStoreRelease(Register src, Register addr, OperandSize size);
507507

508508
void LoadAcquire(Register dst,
509509
const Address& address,
@@ -515,9 +515,10 @@ class Assembler : public AssemblerBase {
515515
AddImmediate(TMP2, src, address.offset());
516516
src = TMP2;
517517
}
518-
ldar(dst, src, size);
519518
if (FLAG_target_thread_sanitizer) {
520-
TsanLoadAcquire(src);
519+
TsanLoadAcquire(dst, src, size);
520+
} else {
521+
ldar(dst, src, size);
521522
}
522523
}
523524

@@ -538,9 +539,10 @@ class Assembler : public AssemblerBase {
538539
AddImmediate(TMP2, dst, address.offset());
539540
dst = TMP2;
540541
}
541-
stlr(src, dst, size);
542542
if (FLAG_target_thread_sanitizer) {
543-
TsanStoreRelease(dst);
543+
TsanStoreRelease(src, dst, size);
544+
} else {
545+
stlr(src, dst, size);
544546
}
545547
}
546548

‎runtime/vm/compiler/assembler/assembler_riscv.cc‎

Lines changed: 98 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2880,8 +2880,7 @@ void Assembler::PushRegisters(const RegisterSet& regs) {
28802880
// The order in which the registers are pushed must match the order
28812881
// in which the registers are encoded in the safepoint's stack map.
28822882

2883-
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
2884-
(regs.FpuRegisterCount() * kFpuRegisterSize);
2883+
intptr_t size = regs.SpillSize();
28852884
if (size == 0) {
28862885
return; // Skip no-op SP update.
28872886
}
@@ -2909,8 +2908,7 @@ void Assembler::PopRegisters(const RegisterSet& regs) {
29092908
// The order in which the registers are pushed must match the order
29102909
// in which the registers are encoded in the safepoint's stack map.
29112910

2912-
intptr_t size = (regs.CpuRegisterCount() * target::kWordSize) +
2913-
(regs.FpuRegisterCount() * kFpuRegisterSize);
2911+
intptr_t size = regs.SpillSize();
29142912
if (size == 0) {
29152913
return; // Skip no-op SP update.
29162914
}
@@ -3084,22 +3082,104 @@ void Assembler::Jump(const Address& address) {
30843082
jr(TMP2);
30853083
}
30863084

3087-
void Assembler::TsanLoadAcquire(Register addr) {
3088-
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
3089-
MoveRegister(A0, addr);
3090-
rt.Call(kTsanLoadAcquireRuntimeEntry, /*argument_count=*/1);
3085+
void Assembler::TsanLoadAcquire(Register dst,
3086+
const Address& addr,
3087+
OperandSize size) {
3088+
ASSERT(addr.base() != SP);
3089+
ASSERT(addr.base() != FP);
3090+
ASSERT(dst != SP);
3091+
ASSERT(dst != FP);
3092+
3093+
RegisterSet registers(kDartVolatileCpuRegs & ~(1 << dst),
3094+
kAbiVolatileFpuRegs);
3095+
3096+
subi(SP, SP, 4 * target::kWordSize);
3097+
sx(RA, Address(SP, 3 * target::kWordSize));
3098+
sx(FP, Address(SP, 2 * target::kWordSize));
3099+
sx(CODE_REG, Address(SP, 1 * target::kWordSize));
3100+
sx(PP, Address(SP, 0 * target::kWordSize));
3101+
addi(FP, SP, 4 * target::kWordSize);
3102+
3103+
PushRegisters(registers);
3104+
ReserveAlignedFrameSpace(0);
3105+
3106+
AddImmediate(A0, addr.base(), addr.offset());
3107+
LoadImmediate(A1, __ATOMIC_ACQUIRE);
3108+
3109+
switch (size) {
3110+
case kEightBytes:
3111+
lx(TMP2, compiler::Address(
3112+
THR, kTsanAtomic64LoadRuntimeEntry.OffsetFromThread()));
3113+
break;
3114+
case kUnsignedFourBytes:
3115+
lx(TMP2, compiler::Address(
3116+
THR, kTsanAtomic32LoadRuntimeEntry.OffsetFromThread()));
3117+
break;
3118+
default:
3119+
UNIMPLEMENTED();
3120+
break;
3121+
}
3122+
sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
3123+
jalr(TMP2);
3124+
LoadImmediate(TMP2, VMTag::kDartTagId);
3125+
sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
3126+
3127+
MoveRegister(dst, A0);
3128+
3129+
subi(SP, FP, registers.SpillSize() + 4 * target::kWordSize);
3130+
PopRegisters(registers);
3131+
3132+
subi(SP, FP, 4 * target::kWordSize);
3133+
lx(PP, Address(SP, 0 * target::kWordSize));
3134+
lx(CODE_REG, Address(SP, 1 * target::kWordSize));
3135+
lx(FP, Address(SP, 2 * target::kWordSize));
3136+
lx(RA, Address(SP, 3 * target::kWordSize));
3137+
addi(SP, SP, 4 * target::kWordSize);
30913138
}
3092-
void Assembler::TsanStoreRelease(Register addr) {
3139+
3140+
void Assembler::TsanStoreRelease(Register src,
3141+
const Address& addr,
3142+
OperandSize size) {
3143+
ASSERT(addr.base() != SP);
3144+
ASSERT(addr.base() != FP);
3145+
ASSERT(src != SP);
3146+
ASSERT(src != FP);
3147+
30933148
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
3094-
MoveRegister(A0, addr);
3095-
rt.Call(kTsanStoreReleaseRuntimeEntry, /*argument_count=*/1);
3149+
3150+
if (src == A0) {
3151+
MoveRegister(A1, src);
3152+
AddImmediate(A0, addr.base(), addr.offset());
3153+
} else {
3154+
AddImmediate(A0, addr.base(), addr.offset());
3155+
MoveRegister(A1, src);
3156+
}
3157+
LoadImmediate(A2, __ATOMIC_RELEASE);
3158+
3159+
switch (size) {
3160+
case kEightBytes:
3161+
rt.Call(kTsanAtomic64StoreRuntimeEntry, /*argument_count=*/3);
3162+
break;
3163+
case kFourBytes:
3164+
case kUnsignedFourBytes:
3165+
rt.Call(kTsanAtomic32StoreRuntimeEntry, /*argument_count=*/3);
3166+
break;
3167+
default:
3168+
UNIMPLEMENTED();
3169+
break;
3170+
}
30963171
}
30973172

30983173
void Assembler::LoadAcquire(Register dst,
30993174
const Address& address,
31003175
OperandSize size) {
31013176
ASSERT(dst != address.base());
31023177

3178+
if (FLAG_target_thread_sanitizer) {
3179+
TsanLoadAcquire(dst, address, size);
3180+
return;
3181+
}
3182+
31033183
if (Supports(RV_Zalasr)) {
31043184
Address addr = PrepareAtomicOffset(address.base(), address.offset());
31053185
switch (size) {
@@ -3124,20 +3204,16 @@ void Assembler::LoadAcquire(Register dst,
31243204
Load(dst, address, size);
31253205
fence(HartEffects::kRead, HartEffects::kMemory);
31263206
}
3127-
3128-
if (FLAG_target_thread_sanitizer) {
3129-
if (address.offset() == 0) {
3130-
TsanLoadAcquire(address.base());
3131-
} else {
3132-
AddImmediate(TMP2, address.base(), address.offset());
3133-
TsanLoadAcquire(TMP2);
3134-
}
3135-
}
31363207
}
31373208

31383209
void Assembler::StoreRelease(Register src,
31393210
const Address& address,
31403211
OperandSize size) {
3212+
if (FLAG_target_thread_sanitizer) {
3213+
TsanStoreRelease(src, address, size);
3214+
return;
3215+
}
3216+
31413217
if (Supports(RV_Zalasr)) {
31423218
Address addr = PrepareAtomicOffset(address.base(), address.offset());
31433219
switch (size) {
@@ -4781,13 +4857,8 @@ void LeafRuntimeScope::Call(const RuntimeEntry& entry,
47814857

47824858
LeafRuntimeScope::~LeafRuntimeScope() {
47834859
if (preserve_registers_) {
4784-
const intptr_t kSavedRegistersSize =
4785-
kRuntimeCallSavedRegisters.CpuRegisterCount() * target::kWordSize +
4786-
kRuntimeCallSavedRegisters.FpuRegisterCount() * kFpuRegisterSize +
4787-
4 * target::kWordSize;
4788-
4789-
__ subi(SP, FP, kSavedRegistersSize);
4790-
4860+
__ subi(SP, FP,
4861+
kRuntimeCallSavedRegisters.SpillSize() + 4 * target::kWordSize);
47914862
__ PopRegisters(kRuntimeCallSavedRegisters);
47924863
}
47934864

‎runtime/vm/compiler/assembler/assembler_riscv.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1007,8 +1007,8 @@ class Assembler : public MicroAssembler {
10071007
StoreToOffset(src, base, offset, kWordBytes);
10081008
}
10091009

1010-
void TsanLoadAcquire(Register addr);
1011-
void TsanStoreRelease(Register addr);
1010+
void TsanLoadAcquire(Register dst, const Address& address, OperandSize size);
1011+
void TsanStoreRelease(Register src, const Address& address, OperandSize size);
10121012

10131013
void LoadAcquire(Register dst,
10141014
const Address& address,

0 commit comments

Comments
 (0)