Skip to content

Commit 3582368

Browse files
zhaomaosuzhaomaosu
authored
[DevMSAN] Fix missed symbols __msan_memcpy & __msan_memmove (#16535)
1 parent c0e2fd1 commit 3582368

File tree

4 files changed

+145
-29
lines changed

4 files changed

+145
-29
lines changed

‎libdevice/sanitizer/msan_rtl.cpp‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,18 +221,82 @@ DEVICE_EXTERN_C_NOINLINE uptr __msan_get_shadow(uptr addr, uint32_t as) {
221221
}
222222

223223
#define MSAN_MEMSET(as) \
224-
DEVICE_EXTERN_C_NOINLINE void __msan_memset_p##as( \
224+
DEVICE_EXTERN_C_NOINLINE \
225+
__attribute__((address_space(as))) void *__msan_memset_p##as( \
225226
__attribute__((address_space(as))) char *dest, int val, size_t size) { \
226227
uptr shadow = __msan_get_shadow((uptr)dest, as); \
227228
for (size_t i = 0; i < size; i++) { \
228229
dest[i] = val; \
229230
((__SYCL_GLOBAL__ char *)shadow)[i] = 0; \
230231
} \
232+
return dest; \
231233
}
232234

233235
MSAN_MEMSET(0)
234236
MSAN_MEMSET(1)
235237
MSAN_MEMSET(3)
236238
MSAN_MEMSET(4)
237239

240+
#define MSAN_MEMMOVE_BASE(dst_as, src_as) \
241+
DEVICE_EXTERN_C_NOINLINE __attribute__((address_space(dst_as))) void \
242+
*__msan_memmove_p##dst_as##_p##src_as( \
243+
__attribute__((address_space(dst_as))) char *dest, \
244+
__attribute__((address_space(src_as))) char *src, size_t size) { \
245+
uptr dest_shadow = __msan_get_shadow((uptr)dest, dst_as); \
246+
uptr src_shadow = __msan_get_shadow((uptr)src, src_as); \
247+
if ((uptr)dest > (uptr)src) { \
248+
for (size_t i = size - 1; i < size; i--) { \
249+
dest[i] = src[i]; \
250+
((__SYCL_GLOBAL__ char *)dest_shadow)[i] = \
251+
((__SYCL_GLOBAL__ char *)src_shadow)[i]; \
252+
} \
253+
} else { \
254+
for (size_t i = 0; i < size; i++) { \
255+
dest[i] = src[i]; \
256+
((__SYCL_GLOBAL__ char *)dest_shadow)[i] = \
257+
((__SYCL_GLOBAL__ char *)src_shadow)[i]; \
258+
} \
259+
} \
260+
return dest; \
261+
}
262+
263+
#define MSAN_MEMMOVE(dst_as) \
264+
MSAN_MEMMOVE_BASE(dst_as, 0) \
265+
MSAN_MEMMOVE_BASE(dst_as, 1) \
266+
MSAN_MEMMOVE_BASE(dst_as, 2) \
267+
MSAN_MEMMOVE_BASE(dst_as, 3) \
268+
MSAN_MEMMOVE_BASE(dst_as, 4)
269+
270+
MSAN_MEMMOVE(0)
271+
MSAN_MEMMOVE(1)
272+
MSAN_MEMMOVE(3)
273+
MSAN_MEMMOVE(4)
274+
275+
#define MSAN_MEMCPY_BASE(dst_as, src_as) \
276+
DEVICE_EXTERN_C_NOINLINE __attribute__((address_space(dst_as))) void \
277+
*__msan_memcpy_p##dst_as##_p##src_as( \
278+
__attribute__((address_space(dst_as))) char *dest, \
279+
__attribute__((address_space(src_as))) char *src, size_t size) { \
280+
uptr dest_shadow = __msan_get_shadow((uptr)dest, dst_as); \
281+
uptr src_shadow = __msan_get_shadow((uptr)src, src_as); \
282+
for (size_t i = 0; i < size; i++) { \
283+
dest[i] = src[i]; \
284+
((__SYCL_GLOBAL__ char *)dest_shadow)[i] = \
285+
((__SYCL_GLOBAL__ char *)src_shadow)[i]; \
286+
} \
287+
return dest; \
288+
}
289+
290+
#define MSAN_MEMCPY(dst_as) \
291+
MSAN_MEMCPY_BASE(dst_as, 0) \
292+
MSAN_MEMCPY_BASE(dst_as, 1) \
293+
MSAN_MEMCPY_BASE(dst_as, 2) \
294+
MSAN_MEMCPY_BASE(dst_as, 3) \
295+
MSAN_MEMCPY_BASE(dst_as, 4)
296+
297+
MSAN_MEMCPY(0)
298+
MSAN_MEMCPY(1)
299+
MSAN_MEMCPY(3)
300+
MSAN_MEMCPY(4)
301+
238302
#endif // __SPIR__ || __SPIRV__

‎llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,9 @@ class MemorySanitizer {
681681
FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
682682

683683
/// MSan runtime replacements for memset with address space.
684-
FunctionCallee MemsetOffloadFn[kNumberOfAddressSpace];
684+
FunctionCallee MemmoveOffloadFn[kNumberOfAddressSpace][kNumberOfAddressSpace],
685+
MemcpyOffloadFn[kNumberOfAddressSpace][kNumberOfAddressSpace],
686+
MemsetOffloadFn[kNumberOfAddressSpace];
685687

686688
/// KMSAN callback for task-local function argument shadow.
687689
StructType *MsanContextStateTy;
@@ -948,7 +950,7 @@ static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
948950
// FIXME: spirv target doesn't support TLS, need to handle it later.
949951
if (Triple(M.getTargetTriple()).isSPIROrSPIRV()) {
950952
return M.getOrInsertGlobal(Name, Ty, [&] {
951-
return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
953+
return new GlobalVariable(M, Ty, false, GlobalVariable::InternalLinkage,
952954
Constant::getNullValue(Ty), Name, nullptr,
953955
GlobalVariable::NotThreadLocal,
954956
kSpirOffloadGlobalAS);
@@ -1088,11 +1090,23 @@ void MemorySanitizer::initializeCallbacks(Module &M,
10881090
} else {
10891091
for (unsigned FirstArgAS = 0; FirstArgAS < kNumberOfAddressSpace;
10901092
FirstArgAS++) {
1091-
const std::string Suffix = "_p" + itostr(FirstArgAS);
1093+
const std::string Suffix1 = "_p" + itostr(FirstArgAS);
10921094
PointerType *FirstArgPtrTy = IRB.getPtrTy(FirstArgAS);
10931095
MemsetOffloadFn[FirstArgAS] = M.getOrInsertFunction(
1094-
"__msan_memset" + Suffix, TLI.getAttrList(C, {1}, /*Signed=*/true),
1096+
"__msan_memset" + Suffix1, TLI.getAttrList(C, {1}, /*Signed=*/true),
10951097
FirstArgPtrTy, FirstArgPtrTy, IRB.getInt32Ty(), IntptrTy);
1098+
1099+
for (unsigned SecondArgAS = 0; SecondArgAS < kNumberOfAddressSpace;
1100+
SecondArgAS++) {
1101+
const std::string Suffix2 = Suffix1 + "_p" + itostr(SecondArgAS);
1102+
PointerType *SecondArgPtrTy = IRB.getPtrTy(SecondArgAS);
1103+
MemmoveOffloadFn[FirstArgAS][SecondArgAS] =
1104+
M.getOrInsertFunction("__msan_memmove" + Suffix2, FirstArgPtrTy,
1105+
FirstArgPtrTy, SecondArgPtrTy, IntptrTy);
1106+
MemcpyOffloadFn[FirstArgAS][SecondArgAS] =
1107+
M.getOrInsertFunction("__msan_memcpy" + Suffix2, FirstArgPtrTy,
1108+
FirstArgPtrTy, SecondArgPtrTy, IntptrTy);
1109+
}
10961110
}
10971111
}
10981112

@@ -1320,8 +1334,6 @@ static void setNoSanitizedMetadataSPIR(Instruction &I) {
13201334
Addr = XCHG->getPointerOperand();
13211335
else if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(&I))
13221336
Addr = ASC->getPointerOperand();
1323-
else if (isa<MemCpyInst>(&I))
1324-
I.setNoSanitizeMetadata();
13251337
else if (isa<AllocaInst>(&I))
13261338
I.setNoSanitizeMetadata();
13271339
else if (const auto *CI = dyn_cast<CallInst>(&I)) {
@@ -3180,9 +3192,19 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
31803192
///
31813193
/// Similar situation exists for memcpy and memset.
31823194
void visitMemMoveInst(MemMoveInst &I) {
3195+
if (SpirOrSpirv && ((isa<Instruction>(I.getArgOperand(0)) &&
3196+
cast<Instruction>(I.getArgOperand(0))
3197+
->getMetadata(LLVMContext::MD_nosanitize)) ||
3198+
(isa<Instruction>(I.getArgOperand(1)) &&
3199+
cast<Instruction>(I.getArgOperand(1))
3200+
->getMetadata(LLVMContext::MD_nosanitize))))
3201+
return;
3202+
31833203
getShadow(I.getArgOperand(1)); // Ensure shadow initialized
31843204
IRBuilder<> IRB(&I);
3185-
IRB.CreateCall(MS.MemmoveFn,
3205+
IRB.CreateCall(SpirOrSpirv ? MS.MemmoveOffloadFn[I.getDestAddressSpace()]
3206+
[I.getSourceAddressSpace()]
3207+
: MS.MemmoveFn,
31863208
{I.getArgOperand(0), I.getArgOperand(1),
31873209
IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
31883210
I.eraseFromParent();
@@ -3203,9 +3225,19 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
32033225
/// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
32043226
/// itself, instrumentation should be disabled with the no_sanitize attribute.
32053227
void visitMemCpyInst(MemCpyInst &I) {
3228+
if (SpirOrSpirv && ((isa<Instruction>(I.getArgOperand(0)) &&
3229+
cast<Instruction>(I.getArgOperand(0))
3230+
->getMetadata(LLVMContext::MD_nosanitize)) ||
3231+
(isa<Instruction>(I.getArgOperand(1)) &&
3232+
cast<Instruction>(I.getArgOperand(1))
3233+
->getMetadata(LLVMContext::MD_nosanitize))))
3234+
return;
3235+
32063236
getShadow(I.getArgOperand(1)); // Ensure shadow initialized
32073237
IRBuilder<> IRB(&I);
3208-
IRB.CreateCall(MS.MemcpyFn,
3238+
IRB.CreateCall(SpirOrSpirv ? MS.MemcpyOffloadFn[I.getDestAddressSpace()]
3239+
[I.getSourceAddressSpace()]
3240+
: MS.MemcpyFn,
32093241
{I.getArgOperand(0), I.getArgOperand(1),
32103242
IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
32113243
I.eraseFromParent();
@@ -3220,10 +3252,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
32203252

32213253
IRBuilder<> IRB(&I);
32223254
IRB.CreateCall(
3223-
SpirOrSpirv ? MS.MemsetOffloadFn[cast<PointerType>(
3224-
I.getArgOperand(0)->getType())
3225-
->getAddressSpace()]
3226-
: MS.MemsetFn,
3255+
SpirOrSpirv ? MS.MemsetOffloadFn[I.getDestAddressSpace()] : MS.MemsetFn,
32273256
{I.getArgOperand(0),
32283257
IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
32293258
IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});

‎llvm/test/Instrumentation/MemorySanitizer/SPIRV/check_memfunc.ll‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
; RUN: opt < %s -passes=msan -msan-instrumentation-with-call-threshold=0 -msan-eager-checks=1 -S | FileCheck %s
2+
target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64-G1"
3+
target triple = "spir64-unknown-unknown"
4+
5+
define spir_kernel void @MyKernelMemset(ptr %offset.i) {
6+
; CHECK-LABEL: @MyKernelMemset
7+
entry:
8+
call void @llvm.memset.p0.i64(ptr %offset.i, i8 0, i64 0, i1 false)
9+
; CHECK: call ptr @__msan_memset_p0
10+
ret void
11+
}
12+
13+
define spir_kernel void @MyKernelMemmove(ptr %x, ptr %y) {
14+
; CHECK-LABEL: @MyKernelMemmove
15+
entry:
16+
tail call void @llvm.memmove.p0.p0.i64(ptr %x, ptr %y, i64 0, i1 false)
17+
; CHECK: call ptr @__msan_memmove_p0_p0
18+
ret void
19+
}
20+
21+
define spir_kernel void @MyKernelMemcpy(ptr %x, ptr %y) {
22+
; CHECK-LABEL: @MyKernelMemcpy
23+
entry:
24+
tail call void @llvm.memcpy.p0.p0.i64(ptr %x, ptr %y, i64 0, i1 false)
25+
; CHECK: call ptr @__msan_memcpy_p0_p0
26+
ret void
27+
}
28+
29+
; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write)
30+
declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #0
31+
32+
; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
33+
declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg) #1
34+
35+
; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
36+
declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg) #1
37+
38+
attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: write) }
39+
attributes #1 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }

‎llvm/test/Instrumentation/MemorySanitizer/SPIRV/check_memset.ll‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

0 commit comments

Comments
 (0)