Skip to content

Commit 27f6562

Browse files
clementvalGeorgeARM
clementval
authored and
GeorgeARM
committed
[flang][cuda] Use a reference for asyncObject (llvm#138186)
Switch from `int64_t` to `int64_t*` to fit with the rest of the implementation. New tentative with some fix. The previous was reverted yesterday.
1 parent 03437f1 commit 27f6562

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+169
-171
lines changed

‎flang-rt/include/flang-rt/runtime/allocator-registry.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
namespace Fortran::runtime {
2121

22-
using AllocFct = void *(*)(std::size_t, std::int64_t);
22+
using AllocFct = void *(*)(std::size_t, std::int64_t *);
2323
using FreeFct = void (*)(void *);
2424

2525
typedef struct Allocator_t {
@@ -28,7 +28,7 @@ typedef struct Allocator_t {
2828
} Allocator_t;
2929

3030
static RT_API_ATTRS void *MallocWrapper(
31-
std::size_t size, [[maybe_unused]] std::int64_t) {
31+
std::size_t size, [[maybe_unused]] std::int64_t *) {
3232
return std::malloc(size);
3333
}
3434
#ifdef RT_DEVICE_COMPILATION

‎flang-rt/include/flang-rt/runtime/descriptor.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
#include <cstdio>
3030
#include <cstring>
3131

32-
/// Value used for asyncId when no specific stream is specified.
33-
static constexpr std::int64_t kNoAsyncId = -1;
32+
/// Value used for asyncObject when no specific stream is specified.
33+
static constexpr std::int64_t *kNoAsyncObject = nullptr;
3434

3535
namespace Fortran::runtime {
3636

@@ -372,7 +372,7 @@ class Descriptor {
372372
// before calling. It (re)computes the byte strides after
373373
// allocation. Does not allocate automatic components or
374374
// perform default component initialization.
375-
RT_API_ATTRS int Allocate(std::int64_t asyncId);
375+
RT_API_ATTRS int Allocate(std::int64_t *asyncObject);
376376
RT_API_ATTRS void SetByteStrides();
377377

378378
// Deallocates storage; does not call FINAL subroutines or

‎flang-rt/include/flang-rt/runtime/reduction-templates.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ inline RT_API_ATTRS void DoMaxMinNorm2(Descriptor &result, const Descriptor &x,
347347
// as the element size of the source.
348348
result.Establish(x.type(), x.ElementBytes(), nullptr, 0, nullptr,
349349
CFI_attribute_allocatable);
350-
if (int stat{result.Allocate(kNoAsyncId)}) {
350+
if (int stat{result.Allocate(kNoAsyncObject)}) {
351351
terminator.Crash(
352352
"%s: could not allocate memory for result; STAT=%d", intrinsic, stat);
353353
}

‎flang-rt/lib/cuda/allocatable.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ namespace Fortran::runtime::cuda {
2323
extern "C" {
2424
RT_EXT_API_GROUP_BEGIN
2525

26-
int RTDEF(CUFAllocatableAllocateSync)(Descriptor &desc, int64_t stream,
26+
int RTDEF(CUFAllocatableAllocateSync)(Descriptor &desc, int64_t *stream,
2727
bool *pinned, bool hasStat, const Descriptor *errMsg,
2828
const char *sourceFile, int sourceLine) {
2929
int stat{RTNAME(CUFAllocatableAllocate)(
@@ -41,7 +41,7 @@ int RTDEF(CUFAllocatableAllocateSync)(Descriptor &desc, int64_t stream,
4141
return stat;
4242
}
4343

44-
int RTDEF(CUFAllocatableAllocate)(Descriptor &desc, int64_t stream,
44+
int RTDEF(CUFAllocatableAllocate)(Descriptor &desc, int64_t *stream,
4545
bool *pinned, bool hasStat, const Descriptor *errMsg,
4646
const char *sourceFile, int sourceLine) {
4747
if (desc.HasAddendum()) {
@@ -63,7 +63,7 @@ int RTDEF(CUFAllocatableAllocate)(Descriptor &desc, int64_t stream,
6363
}
6464

6565
int RTDEF(CUFAllocatableAllocateSource)(Descriptor &alloc,
66-
const Descriptor &source, int64_t stream, bool *pinned, bool hasStat,
66+
const Descriptor &source, int64_t *stream, bool *pinned, bool hasStat,
6767
const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
6868
int stat{RTNAME(CUFAllocatableAllocate)(
6969
alloc, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};
@@ -76,7 +76,7 @@ int RTDEF(CUFAllocatableAllocateSource)(Descriptor &alloc,
7676
}
7777

7878
int RTDEF(CUFAllocatableAllocateSourceSync)(Descriptor &alloc,
79-
const Descriptor &source, int64_t stream, bool *pinned, bool hasStat,
79+
const Descriptor &source, int64_t *stream, bool *pinned, bool hasStat,
8080
const Descriptor *errMsg, const char *sourceFile, int sourceLine) {
8181
int stat{RTNAME(CUFAllocatableAllocateSync)(
8282
alloc, stream, pinned, hasStat, errMsg, sourceFile, sourceLine)};

‎flang-rt/lib/cuda/allocator.cpp

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -98,15 +98,15 @@ static unsigned findAllocation(void *ptr) {
9898
return allocNotFound;
9999
}
100100

101-
static void insertAllocation(void *ptr, std::size_t size, std::int64_t stream) {
101+
static void insertAllocation(void *ptr, std::size_t size, cudaStream_t stream) {
102102
CriticalSection critical{lock};
103103
initAllocations();
104104
if (numDeviceAllocations >= maxDeviceAllocations) {
105105
doubleAllocationArray();
106106
}
107107
deviceAllocations[numDeviceAllocations].ptr = ptr;
108108
deviceAllocations[numDeviceAllocations].size = size;
109-
deviceAllocations[numDeviceAllocations].stream = (cudaStream_t)stream;
109+
deviceAllocations[numDeviceAllocations].stream = stream;
110110
++numDeviceAllocations;
111111
qsort(deviceAllocations, numDeviceAllocations, sizeof(DeviceAllocation),
112112
compareDeviceAlloc);
@@ -136,26 +136,26 @@ void RTDEF(CUFRegisterAllocator)() {
136136
}
137137

138138
void *CUFAllocPinned(
139-
std::size_t sizeInBytes, [[maybe_unused]] std::int64_t asyncId) {
139+
std::size_t sizeInBytes, [[maybe_unused]] std::int64_t *asyncObject) {
140140
void *p;
141141
CUDA_REPORT_IF_ERROR(cudaMallocHost((void **)&p, sizeInBytes));
142142
return p;
143143
}
144144

145145
void CUFFreePinned(void *p) { CUDA_REPORT_IF_ERROR(cudaFreeHost(p)); }
146146

147-
void *CUFAllocDevice(std::size_t sizeInBytes, std::int64_t asyncId) {
147+
void *CUFAllocDevice(std::size_t sizeInBytes, std::int64_t *asyncObject) {
148148
void *p;
149149
if (Fortran::runtime::executionEnvironment.cudaDeviceIsManaged) {
150150
CUDA_REPORT_IF_ERROR(
151151
cudaMallocManaged((void **)&p, sizeInBytes, cudaMemAttachGlobal));
152152
} else {
153-
if (asyncId == kNoAsyncId) {
153+
if (asyncObject == kNoAsyncObject) {
154154
CUDA_REPORT_IF_ERROR(cudaMalloc(&p, sizeInBytes));
155155
} else {
156156
CUDA_REPORT_IF_ERROR(
157-
cudaMallocAsync(&p, sizeInBytes, (cudaStream_t)asyncId));
158-
insertAllocation(p, sizeInBytes, asyncId);
157+
cudaMallocAsync(&p, sizeInBytes, (cudaStream_t)*asyncObject));
158+
insertAllocation(p, sizeInBytes, (cudaStream_t)*asyncObject);
159159
}
160160
}
161161
return p;
@@ -174,7 +174,7 @@ void CUFFreeDevice(void *p) {
174174
}
175175

176176
void *CUFAllocManaged(
177-
std::size_t sizeInBytes, [[maybe_unused]] std::int64_t asyncId) {
177+
std::size_t sizeInBytes, [[maybe_unused]] std::int64_t *asyncObject) {
178178
void *p;
179179
CUDA_REPORT_IF_ERROR(
180180
cudaMallocManaged((void **)&p, sizeInBytes, cudaMemAttachGlobal));
@@ -184,9 +184,9 @@ void *CUFAllocManaged(
184184
void CUFFreeManaged(void *p) { CUDA_REPORT_IF_ERROR(cudaFree(p)); }
185185

186186
void *CUFAllocUnified(
187-
std::size_t sizeInBytes, [[maybe_unused]] std::int64_t asyncId) {
187+
std::size_t sizeInBytes, [[maybe_unused]] std::int64_t *asyncObject) {
188188
// Call alloc managed for the time being.
189-
return CUFAllocManaged(sizeInBytes, asyncId);
189+
return CUFAllocManaged(sizeInBytes, asyncObject);
190190
}
191191

192192
void CUFFreeUnified(void *p) {

‎flang-rt/lib/cuda/descriptor.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ RT_EXT_API_GROUP_BEGIN
2121
Descriptor *RTDEF(CUFAllocDescriptor)(
2222
std::size_t sizeInBytes, const char *sourceFile, int sourceLine) {
2323
return reinterpret_cast<Descriptor *>(
24-
CUFAllocManaged(sizeInBytes, /*asyncId*/ -1));
24+
CUFAllocManaged(sizeInBytes, /*asyncObject=*/nullptr));
2525
}
2626

2727
void RTDEF(CUFFreeDescriptor)(

‎flang-rt/lib/runtime/allocatable.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,17 +133,17 @@ void RTDEF(AllocatableApplyMold)(
133133
}
134134
}
135135

136-
int RTDEF(AllocatableAllocate)(Descriptor &descriptor, std::int64_t asyncId,
137-
bool hasStat, const Descriptor *errMsg, const char *sourceFile,
138-
int sourceLine) {
136+
int RTDEF(AllocatableAllocate)(Descriptor &descriptor,
137+
std::int64_t *asyncObject, bool hasStat, const Descriptor *errMsg,
138+
const char *sourceFile, int sourceLine) {
139139
Terminator terminator{sourceFile, sourceLine};
140140
if (!descriptor.IsAllocatable()) {
141141
return ReturnError(terminator, StatInvalidDescriptor, errMsg, hasStat);
142142
} else if (descriptor.IsAllocated()) {
143143
return ReturnError(terminator, StatBaseNotNull, errMsg, hasStat);
144144
} else {
145-
int stat{
146-
ReturnError(terminator, descriptor.Allocate(asyncId), errMsg, hasStat)};
145+
int stat{ReturnError(
146+
terminator, descriptor.Allocate(asyncObject), errMsg, hasStat)};
147147
if (stat == StatOk) {
148148
if (const DescriptorAddendum * addendum{descriptor.Addendum()}) {
149149
if (const auto *derived{addendum->derivedType()}) {
@@ -162,7 +162,7 @@ int RTDEF(AllocatableAllocateSource)(Descriptor &alloc,
162162
const Descriptor &source, bool hasStat, const Descriptor *errMsg,
163163
const char *sourceFile, int sourceLine) {
164164
int stat{RTNAME(AllocatableAllocate)(
165-
alloc, /*asyncId=*/-1, hasStat, errMsg, sourceFile, sourceLine)};
165+
alloc, /*asyncObject=*/nullptr, hasStat, errMsg, sourceFile, sourceLine)};
166166
if (stat == StatOk) {
167167
Terminator terminator{sourceFile, sourceLine};
168168
DoFromSourceAssign(alloc, source, terminator);

‎flang-rt/lib/runtime/array-constructor.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ static RT_API_ATTRS void AllocateOrReallocateVectorIfNeeded(
5050
initialAllocationSize(fromElements, to.ElementBytes())};
5151
to.GetDimension(0).SetBounds(1, allocationSize);
5252
RTNAME(AllocatableAllocate)
53-
(to, /*asyncId=*/-1, /*hasStat=*/false, /*errMsg=*/nullptr,
53+
(to, /*asyncObject=*/nullptr, /*hasStat=*/false, /*errMsg=*/nullptr,
5454
vector.sourceFile, vector.sourceLine);
5555
to.GetDimension(0).SetBounds(1, fromElements);
5656
vector.actualAllocationSize = allocationSize;
@@ -59,7 +59,7 @@ static RT_API_ATTRS void AllocateOrReallocateVectorIfNeeded(
5959
// first value: there should be no reallocation.
6060
RUNTIME_CHECK(terminator, previousToElements >= fromElements);
6161
RTNAME(AllocatableAllocate)
62-
(to, /*asyncId=*/-1, /*hasStat=*/false, /*errMsg=*/nullptr,
62+
(to, /*asyncObject=*/nullptr, /*hasStat=*/false, /*errMsg=*/nullptr,
6363
vector.sourceFile, vector.sourceLine);
6464
vector.actualAllocationSize = previousToElements;
6565
}

‎flang-rt/lib/runtime/assign.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ static RT_API_ATTRS int AllocateAssignmentLHS(
9999
toDim.SetByteStride(stride);
100100
stride *= toDim.Extent();
101101
}
102-
int result{ReturnError(terminator, to.Allocate(kNoAsyncId))};
102+
int result{ReturnError(terminator, to.Allocate(kNoAsyncObject))};
103103
if (result == StatOk && derived && !derived->noInitializationNeeded()) {
104104
result = ReturnError(terminator, Initialize(to, *derived, terminator));
105105
}
@@ -277,7 +277,7 @@ RT_API_ATTRS void Assign(Descriptor &to, const Descriptor &from,
277277
// entity, otherwise, the Deallocate() below will not
278278
// free the descriptor memory.
279279
newFrom.raw().attribute = CFI_attribute_allocatable;
280-
auto stat{ReturnError(terminator, newFrom.Allocate(kNoAsyncId))};
280+
auto stat{ReturnError(terminator, newFrom.Allocate(kNoAsyncObject))};
281281
if (stat == StatOk) {
282282
if (HasDynamicComponent(from)) {
283283
// If 'from' has allocatable/automatic component, we cannot

‎flang-rt/lib/runtime/character.cpp

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ static RT_API_ATTRS void Compare(Descriptor &result, const Descriptor &x,
118118
for (int j{0}; j < rank; ++j) {
119119
result.GetDimension(j).SetBounds(1, ub[j]);
120120
}
121-
if (result.Allocate(kNoAsyncId) != CFI_SUCCESS) {
121+
if (result.Allocate(kNoAsyncObject) != CFI_SUCCESS) {
122122
terminator.Crash("Compare: could not allocate storage for result");
123123
}
124124
std::size_t xChars{x.ElementBytes() >> shift<CHAR>};
@@ -173,7 +173,7 @@ static RT_API_ATTRS void AdjustLRHelper(Descriptor &result,
173173
for (int j{0}; j < rank; ++j) {
174174
result.GetDimension(j).SetBounds(1, ub[j]);
175175
}
176-
if (result.Allocate(kNoAsyncId) != CFI_SUCCESS) {
176+
if (result.Allocate(kNoAsyncObject) != CFI_SUCCESS) {
177177
terminator.Crash("ADJUSTL/R: could not allocate storage for result");
178178
}
179179
for (SubscriptValue resultAt{0}; elements-- > 0;
@@ -227,7 +227,7 @@ static RT_API_ATTRS void LenTrim(Descriptor &result, const Descriptor &string,
227227
for (int j{0}; j < rank; ++j) {
228228
result.GetDimension(j).SetBounds(1, ub[j]);
229229
}
230-
if (result.Allocate(kNoAsyncId) != CFI_SUCCESS) {
230+
if (result.Allocate(kNoAsyncObject) != CFI_SUCCESS) {
231231
terminator.Crash("LEN_TRIM: could not allocate storage for result");
232232
}
233233
std::size_t stringElementChars{string.ElementBytes() >> shift<CHAR>};
@@ -427,7 +427,7 @@ static RT_API_ATTRS void GeneralCharFunc(Descriptor &result,
427427
for (int j{0}; j < rank; ++j) {
428428
result.GetDimension(j).SetBounds(1, ub[j]);
429429
}
430-
if (result.Allocate(kNoAsyncId) != CFI_SUCCESS) {
430+
if (result.Allocate(kNoAsyncObject) != CFI_SUCCESS) {
431431
terminator.Crash("SCAN/VERIFY: could not allocate storage for result");
432432
}
433433
std::size_t stringElementChars{string.ElementBytes() >> shift<CHAR>};
@@ -530,7 +530,8 @@ static RT_API_ATTRS void MaxMinHelper(Descriptor &accumulator,
530530
for (int j{0}; j < rank; ++j) {
531531
accumulator.GetDimension(j).SetBounds(1, ub[j]);
532532
}
533-
RUNTIME_CHECK(terminator, accumulator.Allocate(kNoAsyncId) == CFI_SUCCESS);
533+
RUNTIME_CHECK(
534+
terminator, accumulator.Allocate(kNoAsyncObject) == CFI_SUCCESS);
534535
}
535536
for (CHAR *result{accumulator.OffsetElement<CHAR>()}; elements-- > 0;
536537
accumData += accumChars, result += chars, x.IncrementSubscripts(xAt)) {
@@ -606,7 +607,7 @@ void RTDEF(CharacterConcatenate)(Descriptor &accumulator,
606607
for (int j{0}; j < rank; ++j) {
607608
accumulator.GetDimension(j).SetBounds(1, ub[j]);
608609
}
609-
if (accumulator.Allocate(kNoAsyncId) != CFI_SUCCESS) {
610+
if (accumulator.Allocate(kNoAsyncObject) != CFI_SUCCESS) {
610611
terminator.Crash(
611612
"CharacterConcatenate: could not allocate storage for result");
612613
}
@@ -629,7 +630,8 @@ void RTDEF(CharacterConcatenateScalar1)(
629630
accumulator.set_base_addr(nullptr);
630631
std::size_t oldLen{accumulator.ElementBytes()};
631632
accumulator.raw().elem_len += chars;
632-
RUNTIME_CHECK(terminator, accumulator.Allocate(kNoAsyncId) == CFI_SUCCESS);
633+
RUNTIME_CHECK(
634+
terminator, accumulator.Allocate(kNoAsyncObject) == CFI_SUCCESS);
633635
std::memcpy(accumulator.OffsetElement<char>(oldLen), from, chars);
634636
FreeMemory(old);
635637
}
@@ -831,7 +833,7 @@ void RTDEF(Repeat)(Descriptor &result, const Descriptor &string,
831833
std::size_t origBytes{string.ElementBytes()};
832834
result.Establish(string.type(), origBytes * ncopies, nullptr, 0, nullptr,
833835
CFI_attribute_allocatable);
834-
if (result.Allocate(kNoAsyncId) != CFI_SUCCESS) {
836+
if (result.Allocate(kNoAsyncObject) != CFI_SUCCESS) {
835837
terminator.Crash("REPEAT could not allocate storage for result");
836838
}
837839
const char *from{string.OffsetElement()};
@@ -865,7 +867,7 @@ void RTDEF(Trim)(Descriptor &result, const Descriptor &string,
865867
}
866868
result.Establish(string.type(), resultBytes, nullptr, 0, nullptr,
867869
CFI_attribute_allocatable);
868-
RUNTIME_CHECK(terminator, result.Allocate(kNoAsyncId) == CFI_SUCCESS);
870+
RUNTIME_CHECK(terminator, result.Allocate(kNoAsyncObject) == CFI_SUCCESS);
869871
std::memcpy(result.OffsetElement(), string.OffsetElement(), resultBytes);
870872
}
871873

0 commit comments

Comments
 (0)