diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp index b202b20291aff..e3c39a1b8dda5 100644 --- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp +++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp @@ -26,6 +26,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicsBPF.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" @@ -478,9 +479,95 @@ static void aspaceWrapOperand(DenseMap &Cache, Instruction *I, } } +static Value *wrapPtrIfASNotZero(DenseMap &Cache, + CallInst *CI, Value *P) { + if (auto *PTy = dyn_cast(P->getType())) { + if (PTy->getAddressSpace() == 0) + return P; + } + return aspaceWrapValue(Cache, CI->getFunction(), P); +} + +static Instruction *aspaceMemSet(Intrinsic::ID ID, + DenseMap &Cache, + CallInst *CI) { + auto *MI = cast(CI); + IRBuilder<> B(CI); + + Value *OldDst = CI->getArgOperand(0); + Value *NewDst = wrapPtrIfASNotZero(Cache, CI, OldDst); + if (OldDst == NewDst) + return nullptr; + + // memset(new_dst, val, len, align, isvolatile, md) + Value *Val = CI->getArgOperand(1); + Value *Len = CI->getArgOperand(2); + + auto *MS = cast(CI); + MaybeAlign Align = MS->getDestAlign(); + bool IsVolatile = MS->isVolatile(); + + if (ID == Intrinsic::memset) + return B.CreateMemSet(NewDst, Val, Len, Align, IsVolatile, + MI->getAAMetadata()); + else + return B.CreateMemSetInline(NewDst, Align, Val, Len, IsVolatile, + MI->getAAMetadata()); +} + +static Instruction *aspaceMemCpy(Intrinsic::ID ID, + DenseMap &Cache, + CallInst *CI) { + auto *MI = cast(CI); + IRBuilder<> B(CI); + + Value *OldDst = CI->getArgOperand(0); + Value *OldSrc = CI->getArgOperand(1); + Value *NewDst = wrapPtrIfASNotZero(Cache, CI, OldDst); + Value *NewSrc = wrapPtrIfASNotZero(Cache, CI, OldSrc); + if (OldDst == NewDst && OldSrc == NewSrc) + return nullptr; + + // memcpy(new_dst, dst_align, new_src, src_align, len, isvolatile, md) + Value *Len = CI->getArgOperand(2); + + auto *MT = cast(CI); + MaybeAlign DstAlign = MT->getDestAlign(); + MaybeAlign SrcAlign = MT->getSourceAlign(); + bool IsVolatile = MT->isVolatile(); + + return B.CreateMemTransferInst(ID, NewDst, DstAlign, NewSrc, SrcAlign, Len, + IsVolatile, MI->getAAMetadata()); +} + +static Instruction *aspaceMemMove(DenseMap &Cache, + CallInst *CI) { + auto *MI = cast(CI); + IRBuilder<> B(CI); + + Value *OldDst = CI->getArgOperand(0); + Value *OldSrc = CI->getArgOperand(1); + Value *NewDst = wrapPtrIfASNotZero(Cache, CI, OldDst); + Value *NewSrc = wrapPtrIfASNotZero(Cache, CI, OldSrc); + if (OldDst == NewDst && OldSrc == NewSrc) + return nullptr; + + // memmove(new_dst, dst_align, new_src, src_align, len, isvolatile, md) + Value *Len = CI->getArgOperand(2); + + auto *MT = cast(CI); + MaybeAlign DstAlign = MT->getDestAlign(); + MaybeAlign SrcAlign = MT->getSourceAlign(); + bool IsVolatile = MT->isVolatile(); + + return B.CreateMemMove(NewDst, DstAlign, NewSrc, SrcAlign, Len, IsVolatile, + MI->getAAMetadata()); +} + // Support for BPF address spaces: // - for each function in the module M, update pointer operand of // each memory access instruction (load/store/cmpxchg/atomicrmw) +// or intrinsic call insns (memset/memcpy/memmove) // by casting it from non-zero address space to zero address space, e.g: // // (load (ptr addrspace (N) %p) ...) @@ -493,21 +580,60 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) { for (Function &F : M) { DenseMap CastsCache; for (BasicBlock &BB : F) { - for (Instruction &I : BB) { + for (Instruction &I : llvm::make_early_inc_range(BB)) { unsigned PtrOpNum; - if (auto *LD = dyn_cast(&I)) + if (auto *LD = dyn_cast(&I)) { PtrOpNum = LD->getPointerOperandIndex(); - else if (auto *ST = dyn_cast(&I)) + aspaceWrapOperand(CastsCache, &I, PtrOpNum); + continue; + } + if (auto *ST = dyn_cast(&I)) { PtrOpNum = ST->getPointerOperandIndex(); - else if (auto *CmpXchg = dyn_cast(&I)) + aspaceWrapOperand(CastsCache, &I, PtrOpNum); + continue; + } + if (auto *CmpXchg = dyn_cast(&I)) { PtrOpNum = CmpXchg->getPointerOperandIndex(); - else if (auto *RMW = dyn_cast(&I)) + aspaceWrapOperand(CastsCache, &I, PtrOpNum); + continue; + } + if (auto *RMW = dyn_cast(&I)) { PtrOpNum = RMW->getPointerOperandIndex(); + aspaceWrapOperand(CastsCache, &I, PtrOpNum); + continue; + } + + auto *CI = dyn_cast(&I); + if (!CI) + continue; + + Function *Callee = CI->getCalledFunction(); + if (!Callee || !Callee->isIntrinsic()) + continue; + + // Check memset/memcpy/memmove + Intrinsic::ID ID = Callee->getIntrinsicID(); + bool IsSet = ID == Intrinsic::memset || ID == Intrinsic::memset_inline; + bool IsCpy = ID == Intrinsic::memcpy || ID == Intrinsic::memcpy_inline; + bool IsMove = ID == Intrinsic::memmove; + if (!IsSet && !IsCpy && !IsMove) + continue; + + Instruction *New; + if (IsSet) + New = aspaceMemSet(ID, CastsCache, CI); + else if (IsCpy) + New = aspaceMemCpy(ID, CastsCache, CI); else + New = aspaceMemMove(CastsCache, CI); + + if (!New) continue; - aspaceWrapOperand(CastsCache, &I, PtrOpNum); + I.replaceAllUsesWith(New); + New->takeName(&I); + I.eraseFromParent(); } } Changed |= !CastsCache.empty(); diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll new file mode 100644 index 0000000000000..1db83915b00a3 --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll @@ -0,0 +1,60 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s + +@page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8 +@page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8 + +define dso_local void @test_memset() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memset() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false) + ret void +} + +declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg) + +define dso_local void @test_memcpy() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memcpy() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false) + ret void +} + +declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg) + +define dso_local void @test_memmove() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memmove() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), i64 16, i1 false) + ret void +} + +declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) writeonly captures(none), ptr addrspace(1) readonly captures(none), i64, i1 immarg) + +define dso_local void @test_memset_inline() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memset_inline() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memset.inline.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memset.inline.p1.i64(ptr addrspace(1) nonnull align 8 getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false) + ret void +} + +declare void @llvm.memset.inline.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg) + +define dso_local void @test_memcpy_inline() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memcpy_inline() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) nonnull align 8 getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) nonnull align 8 getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false) + ret void +} + +declare void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg) diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll new file mode 100644 index 0000000000000..62fa2e4d4491f --- /dev/null +++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll @@ -0,0 +1,49 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s + +@page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8 +@page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8 + +define dso_local void @test_memset() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memset() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i8 0, i64 16, i1 false) + ret void +} + +declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg) + +define dso_local void @test_memcpy() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memcpy() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page2, ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i64 16, i1 false) + ret void +} + +declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg) + +define dso_local void @test_memset_inline() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memset_inline() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memset.inline.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memset.inline.p1.i64(ptr addrspace(1) align 8 @page1, i8 0, i64 16, i1 false) + ret void +} + +declare void @llvm.memset.inline.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg) + +define dso_local void @test_memcpy_inline() local_unnamed_addr { +; CHECK-LABEL: define dso_local void @test_memcpy_inline() local_unnamed_addr { +; CHECK-NEXT: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false) +; CHECK-NEXT: ret void +; + tail call void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) align 8 @page2, ptr addrspace(1) align 8 @page1, i64 16, i1 false) + ret void +} + +declare void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)