Skip to content

[NVTPX] Copy kernel arguments as byte array #110356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 14, 2024
Merged

[NVTPX] Copy kernel arguments as byte array #110356

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fc56a3c
[NVTPX] Copy kernel arguments as byte array
mkuron Sep 28, 2024
35bcad6
[NVPTX] Coalesce kernel argument copies
mkuron Sep 29, 2024
88a395e
[NVTPX] Copy kernel arguments via memcpy intrinsic
mkuron Oct 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,13 +623,33 @@ void NVPTXLowerArgs::handleByValParam(const NVPTXTargetMachine &TM,
Value *ArgInParam = new AddrSpaceCastInst(
Arg, PointerType::get(Arg->getContext(), ADDRESS_SPACE_PARAM),
Arg->getName(), FirstInst);
// Create an opaque type of same size as StructType but without padding
// holes as this could have been a union.
const auto StructBytes = *AllocA->getAllocationSize(DL);
SmallVector<Type *, 5> ChunkTypes;
if (StructBytes >= 16) {
Type *IntType = Type::getInt64Ty(Func->getContext());
Type *ChunkType = VectorType::get(IntType, 2, false);
Type *OpaqueType = StructBytes < 32 ? ChunkType :
ArrayType::get(ChunkType, StructBytes / 16);
ChunkTypes.push_back(OpaqueType);
}
for (const auto ChunkBytes: {8, 4, 2, 1}) {
if (StructBytes & ChunkBytes) {
Type *ChunkType = Type::getIntNTy(Func->getContext(), 8 * ChunkBytes);
ChunkTypes.push_back(ChunkType);
}
}
Type * OpaqueType = ChunkTypes.size() == 1 ? ChunkTypes[0] :
StructType::create(ChunkTypes);
// Be sure to propagate alignment to this load; LLVM doesn't know that NVPTX
// addrspacecast preserves alignment. Since params are constant, this load
// is definitely not volatile.
LoadInst *LI =
new LoadInst(StructType, ArgInParam, Arg->getName(),
new LoadInst(OpaqueType, ArgInParam, Arg->getName(),
/*isVolatile=*/false, AllocA->getAlign(), FirstInst);
new StoreInst(LI, AllocA, FirstInst);
new StoreInst(LI, AllocA,
/*isVolatile=*/false, AllocA->getAlign(), FirstInst);
}
}

Expand Down
3 changes: 2 additions & 1 deletion llvm/test/CodeGen/NVPTX/lower-args.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@
target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
target triple = "nvptx64-nvidia-cuda"

; IR: [[OPAQUE_OUTER:.*]] = type { <2 x i64>, i64 }
%class.outer = type <{ %class.inner, i32, [4 x i8] }>
%class.inner = type { ptr, ptr }

; Check that nvptx-lower-args preserves arg alignment
; COMMON-LABEL: load_alignment
define void @load_alignment(ptr nocapture readonly byval(%class.outer) align 8 %arg) {
entry:
; IR: load %class.outer, ptr addrspace(101)
; IR: load [[OPAQUE_OUTER]], ptr addrspace(101)
; IR-SAME: align 8
; PTX: ld.param.u64
; PTX-NOT: ld.param.u8
Expand Down
88 changes: 64 additions & 24 deletions llvm/test/CodeGen/NVPTX/lower-byval-args.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ source_filename = "<stdin>"
target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
target triple = "nvptx64-nvidia-cuda"

; COMMON: [[OPAQUE_C:.*]] = type { [2 x <2 x i64>], i64, i32, i8 }
%struct.S = type { i32, i32 }

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
Expand Down Expand Up @@ -88,8 +89,8 @@ define dso_local void @read_only_gep_asc0(ptr nocapture noundef writeonly %out,
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
; COMMON-NEXT: [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
; COMMON-NEXT: [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
Expand All @@ -115,8 +116,8 @@ define dso_local void @escape_ptr(ptr nocapture noundef readnone %out, ptr nound
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
; COMMON-NEXT: [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
; COMMON-NEXT: call void @_Z6escapePv(ptr noundef nonnull [[S3]])
Expand All @@ -134,8 +135,8 @@ define dso_local void @escape_ptr_gep(ptr nocapture noundef readnone %out, ptr n
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
; COMMON-NEXT: [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
; COMMON-NEXT: [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
Expand All @@ -155,8 +156,8 @@ define dso_local void @escape_ptr_store(ptr nocapture noundef writeonly %out, pt
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
; COMMON-NEXT: [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
; COMMON-NEXT: store ptr [[S3]], ptr [[OUT2]], align 8
Expand All @@ -174,8 +175,8 @@ define dso_local void @escape_ptr_gep_store(ptr nocapture noundef writeonly %out
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
; COMMON-NEXT: [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
; COMMON-NEXT: [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S3]], i64 4
Expand All @@ -195,8 +196,8 @@ define dso_local void @escape_ptrtoint(ptr nocapture noundef writeonly %out, ptr
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[OUT1:%.*]] = addrspacecast ptr [[OUT]] to ptr addrspace(1)
; COMMON-NEXT: [[OUT2:%.*]] = addrspacecast ptr addrspace(1) [[OUT1]] to ptr
; COMMON-NEXT: [[I:%.*]] = ptrtoint ptr [[S3]] to i64
Expand Down Expand Up @@ -232,8 +233,8 @@ define dso_local void @memcpy_to_param(ptr nocapture noundef readonly %in, ptr n
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
; COMMON-NEXT: [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
; COMMON-NEXT: tail call void @llvm.memcpy.p0.p0.i64(ptr [[S3]], ptr [[IN2]], i64 16, i1 true)
Expand All @@ -251,8 +252,8 @@ define dso_local void @copy_on_store(ptr nocapture noundef readonly %in, ptr noc
; COMMON-NEXT: [[BB:.*:]]
; COMMON-NEXT: [[S3:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[S4:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S5:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[S5:%.*]] = load i64, ptr addrspace(101) [[S4]], align 4
; COMMON-NEXT: store i64 [[S5]], ptr [[S3]], align 4
; COMMON-NEXT: [[IN1:%.*]] = addrspacecast ptr [[IN]] to ptr addrspace(1)
; COMMON-NEXT: [[IN2:%.*]] = addrspacecast ptr addrspace(1) [[IN1]] to ptr
; COMMON-NEXT: [[I:%.*]] = load i32, ptr [[IN2]], align 4
Expand Down Expand Up @@ -337,12 +338,12 @@ define void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval(%struct.S)
; SM_60-NEXT: [[INOUT8:%.*]] = addrspacecast ptr addrspace(1) [[INOUT7]] to ptr
; SM_60-NEXT: [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
; SM_60-NEXT: [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
; SM_60-NEXT: [[INPUT26:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT25]], align 8
; SM_60-NEXT: store [[STRUCT_S]] [[INPUT26]], ptr [[INPUT24]], align 4
; SM_60-NEXT: [[INPUT26:%.*]] = load i64, ptr addrspace(101) [[INPUT25]], align 8
; SM_60-NEXT: store i64 [[INPUT26]], ptr [[INPUT24]], align 8
; SM_60-NEXT: [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
; SM_60-NEXT: [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
; SM_60-NEXT: [[INPUT13:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT12]], align 4
; SM_60-NEXT: store [[STRUCT_S]] [[INPUT13]], ptr [[INPUT11]], align 4
; SM_60-NEXT: [[INPUT13:%.*]] = load i64, ptr addrspace(101) [[INPUT12]], align 4
; SM_60-NEXT: store i64 [[INPUT13]], ptr [[INPUT11]], align 4
; SM_60-NEXT: br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
; SM_60: [[FIRST]]:
; SM_60-NEXT: [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT11]], i32 0, i32 0
Expand Down Expand Up @@ -402,12 +403,12 @@ define void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr byval(%str
; COMMON-NEXT: [[BB:.*:]]
; COMMON-NEXT: [[INPUT24:%.*]] = alloca [[STRUCT_S]], align 8
; COMMON-NEXT: [[INPUT25:%.*]] = addrspacecast ptr [[INPUT2]] to ptr addrspace(101)
; COMMON-NEXT: [[INPUT26:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT25]], align 8
; COMMON-NEXT: store [[STRUCT_S]] [[INPUT26]], ptr [[INPUT24]], align 4
; COMMON-NEXT: [[INPUT26:%.*]] = load i64, ptr addrspace(101) [[INPUT25]], align 8
; COMMON-NEXT: store i64 [[INPUT26]], ptr [[INPUT24]], align 8
; COMMON-NEXT: [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
; COMMON-NEXT: [[INPUT12:%.*]] = addrspacecast ptr [[INPUT1]] to ptr addrspace(101)
; COMMON-NEXT: [[INPUT13:%.*]] = load [[STRUCT_S]], ptr addrspace(101) [[INPUT12]], align 4
; COMMON-NEXT: store [[STRUCT_S]] [[INPUT13]], ptr [[INPUT11]], align 4
; COMMON-NEXT: [[INPUT13:%.*]] = load i64, ptr addrspace(101) [[INPUT12]], align 4
; COMMON-NEXT: store i64 [[INPUT13]], ptr [[INPUT11]], align 4
; COMMON-NEXT: br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
; COMMON: [[FIRST]]:
; COMMON-NEXT: [[PTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT11]], i32 0, i32 0
Expand Down Expand Up @@ -437,6 +438,45 @@ merge: ; preds = %second, %first
ret void
}

%union.U = type { %struct.P }
%struct.P = type { i8, i32 }

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
define dso_local void @padding(ptr nocapture noundef readnone %out, ptr noundef byval(%union.U) align 4 %s) local_unnamed_addr #0 {
; COMMON-LABEL: define dso_local void @padding(
; COMMON-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[UNION_U:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S1:%.*]] = alloca [[UNION_U]], align 4
; COMMON-NEXT: [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S3:%.*]] = load i64, ptr addrspace(101) [[S2]], align 4
; COMMON-NEXT: store i64 [[S3]], ptr [[S1]], align 4
; COMMON-NEXT: call void @_Z6escapePv(ptr noundef nonnull [[S1]])
; COMMON-NEXT: ret void
;
entry:
call void @_Z6escapePv(ptr noundef nonnull %s) #0
ret void
}

%struct.C = type { [45 x i8] }

; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite)
define dso_local void @coalescing(ptr nocapture noundef readnone %out, ptr noundef byval(%struct.C) align 4 %s) local_unnamed_addr #0 {
; COMMON-LABEL: define dso_local void @coalescing(
; COMMON-SAME: ptr nocapture noundef readnone [[OUT:%.*]], ptr noundef byval([[STRUCT_C:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
; COMMON-NEXT: [[ENTRY:.*:]]
; COMMON-NEXT: [[S1:%.*]] = alloca [[STRUCT_C]], align 4
; COMMON-NEXT: [[S2:%.*]] = addrspacecast ptr [[S]] to ptr addrspace(101)
; COMMON-NEXT: [[S3:%.*]] = load [[OPAQUE_C]], ptr addrspace(101) [[S2]], align 4
; COMMON-NEXT: store [[OPAQUE_C]] [[S3]], ptr [[S1]], align 4
; COMMON-NEXT: call void @_Z6escapePv(ptr noundef nonnull [[S1]])
; COMMON-NEXT: ret void
;
entry:
call void @_Z6escapePv(ptr noundef nonnull %s) #0
ret void
}

attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) "no-trapping-math"="true" "target-cpu"="sm_60" "target-features"="+ptx78,+sm_60" "uniform-work-group-size"="true" }
attributes #1 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: write) }
Expand Down
16 changes: 8 additions & 8 deletions llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct
; CHECK-NEXT: cvta.local.u32 %SP, %SPL;
; CHECK-NEXT: ld.param.u32 %r1, [caller_St8x4_param_1];
; CHECK-NEXT: add.u32 %r3, %SPL, 0;
; CHECK-NEXT: ld.param.u64 %rd1, [caller_St8x4_param_0+24];
; CHECK-NEXT: ld.param.u64 %rd2, [caller_St8x4_param_0+16];
; CHECK-NEXT: ld.param.u64 %rd3, [caller_St8x4_param_0+8];
; CHECK-NEXT: ld.param.u64 %rd4, [caller_St8x4_param_0];
; CHECK-NEXT: st.local.u64 [%r3], %rd4;
; CHECK-NEXT: st.local.u64 [%r3+8], %rd3;
; CHECK-NEXT: st.local.u64 [%r3+16], %rd2;
; CHECK-NEXT: st.local.u64 [%r3+24], %rd1;
; CHECK-NEXT: ld.param.u64 %rd1, [caller_St8x4_param_0+16];
; CHECK-NEXT: ld.param.u64 %rd2, [caller_St8x4_param_0+24];
; CHECK-NEXT: ld.param.u64 %rd3, [caller_St8x4_param_0];
; CHECK-NEXT: ld.param.u64 %rd4, [caller_St8x4_param_0+8];
; CHECK-NEXT: st.local.u64 [%r3+8], %rd4;
; CHECK-NEXT: st.local.u64 [%r3], %rd3;
; CHECK-NEXT: st.local.u64 [%r3+24], %rd2;
; CHECK-NEXT: st.local.u64 [%r3+16], %rd1;
; CHECK-NEXT: ld.u64 %rd5, [%SP+8];
; CHECK-NEXT: ld.u64 %rd6, [%SP+0];
; CHECK-NEXT: ld.u64 %rd7, [%SP+24];
Expand Down
Loading