Skip to content

Commit 9c90cad

Browse files
zhaomaosuzhaomaosu
authored
[DevTSAN] Implement tsan_[read|write] for different address space (#19127)
By doing this, we can reduce redundant conditional branch for better performance.
1 parent e868ecc commit 9c90cad

File tree

6 files changed

+127
-107
lines changed

6 files changed

+127
-107
lines changed

‎libdevice/sanitizer/asan_rtl.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ inline void ConvertGenericPointer(uptr &addr, uint32_t &as) {
7878
// FIXME: I'm not sure if we need to check ADDRESS_SPACE_CONSTANT,
7979
// but this can really simplify the generic pointer conversion logic
8080
as = ADDRESS_SPACE_GLOBAL;
81-
addr = old;
81+
addr = (uptr)ToGlobal((void *)old);
8282
}
8383
ASAN_DEBUG(__spirv_ocl_printf(__generic_to, old, addr, as));
8484
}

‎libdevice/sanitizer/msan_rtl.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ inline void ConvertGenericPointer(uptr &addr, uint32_t &as) {
7474
// FIXME: I'm not sure if we need to check ADDRESS_SPACE_CONSTANT,
7575
// but this can really simplify the generic pointer conversion logic
7676
as = ADDRESS_SPACE_GLOBAL;
77-
addr = old;
77+
addr = (uptr)ToGlobal((void *)old);
7878
}
7979
MSAN_DEBUG(__spirv_ocl_printf(__msan_print_generic_to, old, addr, as));
8080
}

‎libdevice/sanitizer/tsan_rtl.cpp‎

Lines changed: 47 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ inline void ConvertGenericPointer(uptr &addr, uint32_t &as) {
6767
// FIXME: I'm not sure if we need to check ADDRESS_SPACE_CONSTANT,
6868
// but this can really simplify the generic pointer conversion logic
6969
as = ADDRESS_SPACE_GLOBAL;
70-
addr = old;
70+
addr = (uptr)ToGlobal((void *)old);
7171
}
7272
TSAN_DEBUG(__spirv_ocl_printf(__tsan_print_generic_to, old, addr, as));
7373
}
@@ -317,10 +317,10 @@ inline bool ContainsSameAccess(__SYCL_GLOBAL__ RawShadow *s, Shadow cur,
317317

318318
} // namespace
319319

320-
#define TSAN_CHECK(type, is_write, size) \
321-
DEVICE_EXTERN_C_NOINLINE void __tsan_##type##size( \
322-
uptr addr, uint32_t as, const char __SYCL_CONSTANT__ *file, \
323-
uint32_t line, const char __SYCL_CONSTANT__ *func) { \
320+
#define TSAN_CHECK_BASE(type, is_write, size, as) \
321+
DEVICE_EXTERN_C_NOINLINE void __tsan_##type##size##_p##as( \
322+
uptr addr, const char __SYCL_CONSTANT__ *file, uint32_t line, \
323+
const char __SYCL_CONSTANT__ *func) { \
324324
__SYCL_GLOBAL__ RawShadow *shadow_mem = MemToShadow(addr, as); \
325325
if (!shadow_mem) \
326326
return; \
@@ -341,6 +341,11 @@ inline bool ContainsSameAccess(__SYCL_GLOBAL__ RawShadow *s, Shadow cur,
341341
CheckRace(shadow_mem, cur, type, addr, size, as, file, line, func); \
342342
}
343343

344+
#define TSAN_CHECK(type, is_write, size) \
345+
TSAN_CHECK_BASE(type, is_write, size, 1) \
346+
TSAN_CHECK_BASE(type, is_write, size, 3) \
347+
TSAN_CHECK_BASE(type, is_write, size, 4)
348+
344349
TSAN_CHECK(read, false, 1)
345350
TSAN_CHECK(read, false, 2)
346351
TSAN_CHECK(read, false, 4)
@@ -350,24 +355,26 @@ TSAN_CHECK(write, true, 2)
350355
TSAN_CHECK(write, true, 4)
351356
TSAN_CHECK(write, true, 8)
352357

353-
DEVICE_EXTERN_C_NOINLINE void
354-
__tsan_write16(uptr addr, uint32_t as, const char __SYCL_CONSTANT__ *file,
355-
uint32_t line, const char __SYCL_CONSTANT__ *func) {
356-
__tsan_write8(addr, as, file, line, func);
357-
__tsan_write8(addr + 8, as, file, line, func);
358-
}
358+
#define TSAN_CHECK16_BASE(type, as) \
359+
DEVICE_EXTERN_C_NOINLINE void __tsan_##type##16_p##as( \
360+
uptr addr, const char __SYCL_CONSTANT__ *file, uint32_t line, \
361+
const char __SYCL_CONSTANT__ *func) { \
362+
__tsan_##type##8_p##as(addr, file, line, func); \
363+
__tsan_##type##8_p##as(addr + 8, file, line, func); \
364+
}
359365

360-
DEVICE_EXTERN_C_NOINLINE void
361-
__tsan_read16(uptr addr, uint32_t as, const char __SYCL_CONSTANT__ *file,
362-
uint32_t line, const char __SYCL_CONSTANT__ *func) {
363-
__tsan_read8(addr, as, file, line, func);
364-
__tsan_read8(addr + 8, as, file, line, func);
365-
}
366+
#define TSAN_CHECK16(type) \
367+
TSAN_CHECK16_BASE(type, 1) \
368+
TSAN_CHECK16_BASE(type, 3) \
369+
TSAN_CHECK16_BASE(type, 4)
366370

367-
#define TSAN_UNALIGNED_CHECK(type, is_write, size) \
368-
DEVICE_EXTERN_C_NOINLINE void __tsan_unaligned_##type##size( \
369-
uptr addr, uint32_t as, const char __SYCL_CONSTANT__ *file, \
370-
uint32_t line, const char __SYCL_CONSTANT__ *func) { \
371+
TSAN_CHECK16(read)
372+
TSAN_CHECK16(write)
373+
374+
#define TSAN_UNALIGNED_CHECK_BASE(type, is_write, size, as) \
375+
DEVICE_EXTERN_C_NOINLINE void __tsan_unaligned_##type##size##_p##as( \
376+
uptr addr, const char __SYCL_CONSTANT__ *file, uint32_t line, \
377+
const char __SYCL_CONSTANT__ *func) { \
371378
__SYCL_GLOBAL__ RawShadow *shadow_mem = MemToShadow(addr, as); \
372379
if (!shadow_mem) \
373380
return; \
@@ -410,6 +417,11 @@ __tsan_read16(uptr addr, uint32_t as, const char __SYCL_CONSTANT__ *file,
410417
} \
411418
}
412419

420+
#define TSAN_UNALIGNED_CHECK(type, is_write, size) \
421+
TSAN_UNALIGNED_CHECK_BASE(type, is_write, size, 1) \
422+
TSAN_UNALIGNED_CHECK_BASE(type, is_write, size, 3) \
423+
TSAN_UNALIGNED_CHECK_BASE(type, is_write, size, 4)
424+
413425
TSAN_UNALIGNED_CHECK(read, false, 1)
414426
TSAN_UNALIGNED_CHECK(read, false, 2)
415427
TSAN_UNALIGNED_CHECK(read, false, 4)
@@ -419,21 +431,21 @@ TSAN_UNALIGNED_CHECK(write, true, 2)
419431
TSAN_UNALIGNED_CHECK(write, true, 4)
420432
TSAN_UNALIGNED_CHECK(write, true, 8)
421433

422-
DEVICE_EXTERN_C_NOINLINE void
423-
__tsan_unaligned_write16(uptr addr, uint32_t as,
424-
const char __SYCL_CONSTANT__ *file, uint32_t line,
425-
const char __SYCL_CONSTANT__ *func) {
426-
__tsan_unaligned_write8(addr, as, file, line, func);
427-
__tsan_unaligned_write8(addr + 8, as, file, line, func);
428-
}
434+
#define TSAN_UNALIGNED_CHECK16_BASE(type, as) \
435+
DEVICE_EXTERN_C_NOINLINE void __tsan_unaligned_##type##16_p##as( \
436+
uptr addr, const char __SYCL_CONSTANT__ *file, uint32_t line, \
437+
const char __SYCL_CONSTANT__ *func) { \
438+
__tsan_unaligned_##type##8_p##as(addr, file, line, func); \
439+
__tsan_unaligned_##type##8_p##as(addr + 8, file, line, func); \
440+
}
429441

430-
DEVICE_EXTERN_C_NOINLINE void
431-
__tsan_unaligned_read16(uptr addr, uint32_t as,
432-
const char __SYCL_CONSTANT__ *file, uint32_t line,
433-
const char __SYCL_CONSTANT__ *func) {
434-
__tsan_unaligned_read8(addr, as, file, line, func);
435-
__tsan_unaligned_read8(addr + 8, as, file, line, func);
436-
}
442+
#define TSAN_UNALIGNED_CHECK16(type) \
443+
TSAN_UNALIGNED_CHECK16_BASE(type, 1) \
444+
TSAN_UNALIGNED_CHECK16_BASE(type, 3) \
445+
TSAN_UNALIGNED_CHECK16_BASE(type, 4)
446+
447+
TSAN_UNALIGNED_CHECK16(read)
448+
TSAN_UNALIGNED_CHECK16(write)
437449

438450
static inline void __tsan_cleanup_private_cpu_impl(uptr addr, uint32_t size) {
439451
if (size) {

‎llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp‎

Lines changed: 46 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -158,15 +158,17 @@ struct ThreadSanitizerOnSpirv {
158158

159159
// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
160160
static const size_t kNumberOfAccessSizes = 5;
161+
static const size_t kNumberOfAddressSpace = 5;
161162
FunctionCallee TsanCleanupPrivate;
162163
FunctionCallee TsanCleanupStaticLocal;
163164
FunctionCallee TsanCleanupDynamicLocal;
164165
FunctionCallee TsanDeviceBarrier;
165166
FunctionCallee TsanGroupBarrier;
166-
FunctionCallee TsanRead[kNumberOfAccessSizes];
167-
FunctionCallee TsanWrite[kNumberOfAccessSizes];
168-
FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes];
169-
FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes];
167+
FunctionCallee TsanRead[kNumberOfAccessSizes][kNumberOfAddressSpace];
168+
FunctionCallee TsanWrite[kNumberOfAccessSizes][kNumberOfAddressSpace];
169+
FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes][kNumberOfAddressSpace];
170+
FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes]
171+
[kNumberOfAddressSpace];
170172

171173
friend struct ThreadSanitizer;
172174
};
@@ -313,34 +315,40 @@ void ThreadSanitizerOnSpirv::initialize() {
313315
"__tsan_group_barrier", Attr.addFnAttribute(C, Attribute::Convergent),
314316
IRB.getVoidTy());
315317

316-
for (size_t i = 0; i < kNumberOfAccessSizes; ++i) {
317-
const unsigned ByteSize = 1U << i;
318-
std::string ByteSizeStr = utostr(ByteSize);
319-
// __tsan_readX/__tsan_writeX(
320-
// ...
321-
// char* file,
322-
// unsigned int line,
323-
// char* func
324-
// )
325-
SmallString<32> ReadName("__tsan_read" + ByteSizeStr);
326-
TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(),
327-
IntptrTy, IRB.getInt32Ty(), Int8PtrTy,
328-
IRB.getInt32Ty(), Int8PtrTy);
329-
330-
SmallString<32> WriteName("__tsan_write" + ByteSizeStr);
331-
TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(),
332-
IntptrTy, IRB.getInt32Ty(), Int8PtrTy,
333-
IRB.getInt32Ty(), Int8PtrTy);
334-
335-
SmallString<32> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr);
336-
TsanUnalignedRead[i] = M.getOrInsertFunction(
337-
UnalignedReadName, Attr, IRB.getVoidTy(), IntptrTy, IRB.getInt32Ty(),
338-
Int8PtrTy, IRB.getInt32Ty(), Int8PtrTy);
339-
340-
SmallString<32> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr);
341-
TsanUnalignedWrite[i] = M.getOrInsertFunction(
342-
UnalignedWriteName, Attr, IRB.getVoidTy(), IntptrTy, IRB.getInt32Ty(),
343-
Int8PtrTy, IRB.getInt32Ty(), Int8PtrTy);
318+
for (size_t AddressSpaceIndex = 0; AddressSpaceIndex < kNumberOfAddressSpace;
319+
AddressSpaceIndex++) {
320+
for (size_t i = 0; i < kNumberOfAccessSizes; ++i) {
321+
const unsigned ByteSize = 1U << i;
322+
std::string ByteSizeStr = utostr(ByteSize);
323+
std::string Suffix = "_p" + itostr(AddressSpaceIndex);
324+
// __tsan_readX/__tsan_writeX(
325+
// ...
326+
// char* file,
327+
// unsigned int line,
328+
// char* func
329+
// )
330+
SmallString<32> ReadName("__tsan_read" + ByteSizeStr + Suffix);
331+
TsanRead[i][AddressSpaceIndex] =
332+
M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(), IntptrTy,
333+
Int8PtrTy, IRB.getInt32Ty(), Int8PtrTy);
334+
335+
SmallString<32> WriteName("__tsan_write" + ByteSizeStr + Suffix);
336+
TsanWrite[i][AddressSpaceIndex] =
337+
M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(), IntptrTy,
338+
Int8PtrTy, IRB.getInt32Ty(), Int8PtrTy);
339+
340+
SmallString<32> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr +
341+
Suffix);
342+
TsanUnalignedRead[i][AddressSpaceIndex] = M.getOrInsertFunction(
343+
UnalignedReadName, Attr, IRB.getVoidTy(), IntptrTy, Int8PtrTy,
344+
IRB.getInt32Ty(), Int8PtrTy);
345+
346+
SmallString<32> UnalignedWriteName("__tsan_unaligned_write" +
347+
ByteSizeStr + Suffix);
348+
TsanUnalignedWrite[i][AddressSpaceIndex] = M.getOrInsertFunction(
349+
UnalignedWriteName, Attr, IRB.getVoidTy(), IntptrTy, Int8PtrTy,
350+
IRB.getInt32Ty(), Int8PtrTy);
351+
}
344352
}
345353
}
346354

@@ -1213,14 +1221,17 @@ bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
12131221
assert((!IsVolatile || !IsCompoundRW) && "Compound volatile invalid!");
12141222

12151223
const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
1224+
const unsigned int AS = cast<PointerType>(Addr->getType()->getScalarType())
1225+
->getPointerAddressSpace();
12161226
FunctionCallee OnAccessFunc = nullptr;
12171227
if (Alignment >= Align(8) || (Alignment.value() % (TypeSize / 8)) == 0) {
12181228
if (IsCompoundRW)
12191229
OnAccessFunc = TsanCompoundRW[Idx];
12201230
else if (IsVolatile)
12211231
OnAccessFunc = IsWrite ? TsanVolatileWrite[Idx] : TsanVolatileRead[Idx];
12221232
else if (Spirv)
1223-
OnAccessFunc = IsWrite ? Spirv->TsanWrite[Idx] : Spirv->TsanRead[Idx];
1233+
OnAccessFunc =
1234+
IsWrite ? Spirv->TsanWrite[Idx][AS] : Spirv->TsanRead[Idx][AS];
12241235
else
12251236
OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx];
12261237
} else {
@@ -1230,17 +1241,14 @@ bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
12301241
OnAccessFunc = IsWrite ? TsanUnalignedVolatileWrite[Idx]
12311242
: TsanUnalignedVolatileRead[Idx];
12321243
else if (Spirv)
1233-
OnAccessFunc = IsWrite ? Spirv->TsanUnalignedWrite[Idx]
1234-
: Spirv->TsanUnalignedRead[Idx];
1244+
OnAccessFunc = IsWrite ? Spirv->TsanUnalignedWrite[Idx][AS]
1245+
: Spirv->TsanUnalignedRead[Idx][AS];
12351246
else
12361247
OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx];
12371248
}
12381249
if (Spirv) {
12391250
SmallVector<Value *, 5> Args;
12401251
Args.push_back(IRB.CreatePointerCast(Addr, IntptrTy));
1241-
unsigned int AS = cast<PointerType>(Addr->getType()->getScalarType())
1242-
->getPointerAddressSpace();
1243-
Args.push_back(ConstantInt::get(IRB.getInt32Ty(), AS));
12441252
Spirv->appendDebugInfoToArgs(II.Inst, Args);
12451253
IRB.CreateCall(OnAccessFunc, Args);
12461254
} else

0 commit comments

Comments
 (0)