Skip to content

Commit ef82092

Browse files
ehellbarehellbar
authored
Merge de31829 into sapling-pr-archive-ehellbar
2 parents 45498d3 + de31829 commit ef82092

37 files changed

+913
-548
lines changed

‎Common/Utils/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,6 @@ o2_add_executable(treemergertool
9191
COMPONENT_NAME CommonUtils
9292
SOURCES src/TreeMergerTool.cxx
9393
PUBLIC_LINK_LIBRARIES O2::CommonUtils Boost::program_options ROOT::Core)
94+
95+
add_library(fpu_support OBJECT src/fpu.cxx)
96+
add_library(O2::fpu_support ALIAS fpu_support)

‎Detectors/AOD/src/AODProducerWorkflowSpec.cxx‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1126,7 +1126,7 @@ void AODProducerWorkflowDPL::fillMCTrackLabelsTable(MCTrackLabelCursorType& mcTr
11261126
if (!needToStore(mGIDToTableID)) {
11271127
continue;
11281128
}
1129-
if (mcTruth.isValid()) { // if not set, -1 will be stored
1129+
if (mcTruth.isValid()) { // if not set, -1 will be stored
11301130
labelHolder.labelID = (mToStore[mcTruth.getSourceID()][mcTruth.getEventID()])[mcTruth.getTrackID()]; // defined by TPC if it contributes, otherwise: by ITS
11311131
if (mcTruth.isFake()) {
11321132
labelHolder.labelMask |= (0x1 << 15);
@@ -1139,6 +1139,21 @@ void AODProducerWorkflowDPL::fillMCTrackLabelsTable(MCTrackLabelCursorType& mcTr
11391139
}
11401140
}
11411141
}
1142+
if (trackIndex.includesDet(DetID::ITS)) {
1143+
auto itsGID = data.getITSContributorGID(trackIndex);
1144+
auto itsSource = itsGID.getSource();
1145+
if (itsSource == GIndex::ITS) {
1146+
auto& itsTrack = data.getITSTrack(itsGID);
1147+
for (unsigned int iL = 0; iL < 7; ++iL) {
1148+
if (itsTrack.isFakeOnLayer(iL)) {
1149+
labelHolder.labelMask |= (0x1 << iL);
1150+
}
1151+
}
1152+
} else if (itsSource == GIndex::ITSAB) {
1153+
labelHolder.labelMask |= (data.getTrackMCLabel(itsGID).isFake() << 12);
1154+
}
1155+
}
1156+
11421157
} else if (mcTruth.isNoise()) {
11431158
labelHolder.labelMask |= (0x1 << 14);
11441159
}

‎Detectors/ITSMFT/ITS/tracking/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,5 @@ o2_target_root_dictionary(ITStracking
5656
if(CUDA_ENABLED OR HIP_ENABLED)
5757
add_subdirectory(GPU)
5858
endif()
59+
60+
add_subdirectory(test)

‎Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu‎

Lines changed: 77 additions & 65 deletions
Large diffs are not rendered by default.

‎Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,14 @@ void TrackerTraitsGPU<nLayers>::initialiseTimeFrame(const int iteration)
3333
mTimeFrameGPU->loadVertices(iteration);
3434
mTimeFrameGPU->loadIndexTableUtils(iteration);
3535
mTimeFrameGPU->loadMultiplicityCutMask(iteration);
36+
// pinned on host
3637
mTimeFrameGPU->createUsedClustersDeviceArray(iteration);
3738
mTimeFrameGPU->createClustersDeviceArray(iteration);
3839
mTimeFrameGPU->createUnsortedClustersDeviceArray(iteration);
3940
mTimeFrameGPU->createClustersIndexTablesArray(iteration);
4041
mTimeFrameGPU->createTrackingFrameInfoDeviceArray(iteration);
4142
mTimeFrameGPU->createROFrameClustersDeviceArray(iteration);
43+
// device array
4244
mTimeFrameGPU->createTrackletsLUTDeviceArray(iteration);
4345
mTimeFrameGPU->createTrackletsBuffersArray(iteration);
4446
mTimeFrameGPU->createCellsBuffersArray(iteration);
@@ -106,7 +108,7 @@ void TrackerTraitsGPU<nLayers>::computeLayerTracklets(const int iteration, int i
106108
mTimeFrameGPU->getPositionResolutions(),
107109
this->mTrkParams[iteration].LayerRadii,
108110
mTimeFrameGPU->getMSangles(),
109-
mTimeFrameGPU->getExternalAllocator(),
111+
mTimeFrameGPU->getExternalDeviceAllocator(),
110112
conf.nBlocksLayerTracklets[iteration],
111113
conf.nThreadsLayerTracklets[iteration],
112114
mTimeFrameGPU->getStreams());
@@ -144,7 +146,7 @@ void TrackerTraitsGPU<nLayers>::computeLayerTracklets(const int iteration, int i
144146
mTimeFrameGPU->getPositionResolutions(),
145147
this->mTrkParams[iteration].LayerRadii,
146148
mTimeFrameGPU->getMSangles(),
147-
mTimeFrameGPU->getExternalAllocator(),
149+
mTimeFrameGPU->getExternalDeviceAllocator(),
148150
conf.nBlocksLayerTracklets[iteration],
149151
conf.nThreadsLayerTracklets[iteration],
150152
mTimeFrameGPU->getStreams());
@@ -195,7 +197,7 @@ void TrackerTraitsGPU<nLayers>::computeLayerCells(const int iteration)
195197
this->mTrkParams[iteration].MaxChi2ClusterAttachment,
196198
this->mTrkParams[iteration].CellDeltaTanLambdaSigma,
197199
this->mTrkParams[iteration].NSigmaCut,
198-
mTimeFrameGPU->getExternalAllocator(),
200+
mTimeFrameGPU->getExternalDeviceAllocator(),
199201
conf.nBlocksLayerCells[iteration],
200202
conf.nThreadsLayerCells[iteration],
201203
mTimeFrameGPU->getStreams());
@@ -251,7 +253,7 @@ void TrackerTraitsGPU<nLayers>::findCellsNeighbours(const int iteration)
251253
currentLayerCellsNum,
252254
nextLayerCellsNum,
253255
1e2,
254-
mTimeFrameGPU->getExternalAllocator(),
256+
mTimeFrameGPU->getExternalDeviceAllocator(),
255257
conf.nBlocksFindNeighbours[iteration],
256258
conf.nThreadsFindNeighbours[iteration],
257259
mTimeFrameGPU->getStream(iLayer));
@@ -279,7 +281,7 @@ void TrackerTraitsGPU<nLayers>::findCellsNeighbours(const int iteration)
279281
mTimeFrameGPU->getDeviceNeighbours(iLayer),
280282
mTimeFrameGPU->getArrayNNeighbours()[iLayer],
281283
mTimeFrameGPU->getStream(iLayer),
282-
mTimeFrameGPU->getExternalAllocator());
284+
mTimeFrameGPU->getExternalDeviceAllocator());
283285
}
284286
mTimeFrameGPU->syncStreams(false);
285287
}
@@ -310,7 +312,7 @@ void TrackerTraitsGPU<nLayers>::findRoads(const int iteration)
310312
this->mTrkParams[0].MaxChi2NDF,
311313
mTimeFrameGPU->getDevicePropagator(),
312314
this->mTrkParams[0].CorrType,
313-
mTimeFrameGPU->getExternalAllocator(),
315+
mTimeFrameGPU->getExternalDeviceAllocator(),
314316
conf.nBlocksProcessNeighbours[iteration],
315317
conf.nThreadsProcessNeighbours[iteration]);
316318
}

‎Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h‎

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#include <new>
2323
#include <vector>
2424

25+
#include "ITStracking/ExternalAllocator.h"
26+
2527
#include "GPUCommonLogger.h"
2628

2729
namespace o2::its
@@ -56,6 +58,7 @@ class BoundedMemoryResource final : public std::pmr::memory_resource
5658

5759
BoundedMemoryResource(size_t maxBytes = std::numeric_limits<size_t>::max(), std::pmr::memory_resource* upstream = std::pmr::get_default_resource())
5860
: mMaxMemory(maxBytes), mUpstream(upstream) {}
61+
BoundedMemoryResource(ExternalAllocator* alloc) : mAdaptor(std::make_unique<ExternalAllocatorAdaptor>(alloc)), mUpstream(mAdaptor.get()) {}
5962

6063
void* do_allocate(size_t bytes, size_t alignment) final
6164
{
@@ -69,7 +72,14 @@ class BoundedMemoryResource final : public std::pmr::memory_resource
6972
} while (!mUsedMemory.compare_exchange_weak(current_used, new_used,
7073
std::memory_order_acq_rel,
7174
std::memory_order_relaxed));
72-
return mUpstream->allocate(bytes, alignment);
75+
void* p{nullptr};
76+
try {
77+
p = mUpstream->allocate(bytes, alignment);
78+
} catch (...) {
79+
mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed);
80+
throw;
81+
}
82+
return p;
7383
}
7484

7585
void do_deallocate(void* p, size_t bytes, size_t alignment) final
@@ -87,11 +97,12 @@ class BoundedMemoryResource final : public std::pmr::memory_resource
8797
size_t getMaxMemory() const noexcept { return mMaxMemory; }
8898
void setMaxMemory(size_t max)
8999
{
90-
if (mUsedMemory > max) {
100+
size_t used = mUsedMemory.load(std::memory_order_acquire);
101+
if (used > max) {
91102
++mCountThrow;
92-
throw MemoryLimitExceeded(0, mUsedMemory, max);
103+
throw MemoryLimitExceeded(0, used, max);
93104
}
94-
mMaxMemory = max;
105+
mMaxMemory.store(max, std::memory_order_release);
95106
}
96107

97108
void print() const
@@ -106,76 +117,74 @@ class BoundedMemoryResource final : public std::pmr::memory_resource
106117
}
107118

108119
private:
109-
size_t mMaxMemory{std::numeric_limits<size_t>::max()};
120+
std::atomic<size_t> mMaxMemory{std::numeric_limits<size_t>::max()};
110121
std::atomic<size_t> mCountThrow{0};
111122
std::atomic<size_t> mUsedMemory{0};
112-
std::pmr::memory_resource* mUpstream;
123+
std::unique_ptr<ExternalAllocatorAdaptor> mAdaptor{nullptr};
124+
std::pmr::memory_resource* mUpstream{nullptr};
113125
};
114126

115127
template <typename T>
116128
using bounded_vector = std::pmr::vector<T>;
117129

118130
template <typename T>
119-
void deepVectorClear(std::vector<T>& vec)
131+
inline void deepVectorClear(std::vector<T>& vec)
120132
{
121133
std::vector<T>().swap(vec);
122134
}
123135

124136
template <typename T>
125-
inline void deepVectorClear(bounded_vector<T>& vec, BoundedMemoryResource* bmr = nullptr)
137+
inline void deepVectorClear(bounded_vector<T>& vec, std::pmr::memory_resource* mr = nullptr)
126138
{
139+
std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource();
127140
vec.~bounded_vector<T>();
128-
if (bmr == nullptr) {
129-
auto alloc = vec.get_allocator().resource();
130-
new (&vec) bounded_vector<T>(alloc);
131-
} else {
132-
new (&vec) bounded_vector<T>(bmr);
133-
}
141+
new (&vec) bounded_vector<T>(std::pmr::polymorphic_allocator<T>{tmr});
134142
}
135143

136144
template <typename T>
137-
void deepVectorClear(std::vector<bounded_vector<T>>& vec, BoundedMemoryResource* bmr = nullptr)
145+
inline void deepVectorClear(std::vector<bounded_vector<T>>& vec, std::pmr::memory_resource* mr = nullptr)
138146
{
139147
for (auto& v : vec) {
140-
deepVectorClear(v, bmr);
148+
deepVectorClear(v, mr);
141149
}
142150
}
143151

144152
template <typename T, size_t S>
145-
void deepVectorClear(std::array<bounded_vector<T>, S>& arr, BoundedMemoryResource* bmr = nullptr)
153+
inline void deepVectorClear(std::array<bounded_vector<T>, S>& arr, std::pmr::memory_resource* mr = nullptr)
146154
{
147155
for (size_t i{0}; i < S; ++i) {
148-
deepVectorClear(arr[i], bmr);
156+
deepVectorClear(arr[i], mr);
149157
}
150158
}
151159

152160
template <typename T>
153-
void clearResizeBoundedVector(bounded_vector<T>& vec, size_t size, BoundedMemoryResource* bmr, T def = T())
161+
inline void clearResizeBoundedVector(bounded_vector<T>& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T())
154162
{
163+
std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource();
155164
vec.~bounded_vector<T>();
156-
new (&vec) bounded_vector<T>(size, def, bmr);
165+
new (&vec) bounded_vector<T>(sz, def, std::pmr::polymorphic_allocator<T>{tmr});
157166
}
158167

159168
template <typename T>
160-
void clearResizeBoundedVector(std::vector<bounded_vector<T>>& vec, size_t size, BoundedMemoryResource* bmr)
169+
inline void clearResizeBoundedVector(std::vector<bounded_vector<T>>& vec, size_t size, std::pmr::memory_resource* mr)
161170
{
162171
vec.clear();
163172
vec.reserve(size);
164-
for (size_t i{0}; i < size; ++i) {
165-
vec.emplace_back(bmr);
173+
for (size_t i = 0; i < size; ++i) {
174+
vec.emplace_back(std::pmr::polymorphic_allocator<bounded_vector<T>>{mr});
166175
}
167176
}
168177

169178
template <typename T, size_t S>
170-
void clearResizeBoundedArray(std::array<bounded_vector<T>, S>& arr, size_t size, BoundedMemoryResource* bmr, T def = T())
179+
inline void clearResizeBoundedArray(std::array<bounded_vector<T>, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T())
171180
{
172181
for (size_t i{0}; i < S; ++i) {
173-
clearResizeBoundedVector(arr[i], size, bmr, def);
182+
clearResizeBoundedVector(arr[i], size, mr, def);
174183
}
175184
}
176185

177186
template <typename T>
178-
std::vector<T> toSTDVector(const bounded_vector<T>& b)
187+
inline std::vector<T> toSTDVector(const bounded_vector<T>& b)
179188
{
180189
std::vector<T> t(b.size());
181190
std::copy(b.cbegin(), b.cend(), t.begin());

‎Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@
1616
#ifndef TRACKINGITSU_INCLUDE_CACLUSTER_H_
1717
#define TRACKINGITSU_INCLUDE_CACLUSTER_H_
1818

19+
#include <array>
20+
#include "ITStracking/Constants.h"
1921
#include "GPUCommonRtypes.h"
20-
#include "GPUCommonArray.h"
2122

2223
namespace o2::its
2324
{
@@ -47,8 +48,8 @@ struct Cluster final {
4748
float zCoordinate{-999.f};
4849
float phi{-999.f};
4950
float radius{-999.f};
50-
int clusterId{-1};
51-
int indexTableBinIndex{-1};
51+
int clusterId{constants::UnusedIndex};
52+
int indexTableBinIndex{constants::UnusedIndex};
5253

5354
ClassDefNV(Cluster, 1);
5455
};
@@ -70,7 +71,7 @@ struct TrackingFrameInfo final {
7071
float zCoordinate{-999.f};
7172
float xTrackingFrame{-999.f};
7273
float alphaTrackingFrame{-999.f};
73-
std::array<float, 2> positionTrackingFrame = {-1., -1.};
74+
std::array<float, 2> positionTrackingFrame = {constants::UnusedIndex, constants::UnusedIndex};
7475
std::array<float, 3> covarianceTrackingFrame = {999., 999., 999.};
7576

7677
ClassDefNV(TrackingFrameInfo, 1);

‎Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ struct TrackingParameters {
5151
float Diamond[3] = {0.f, 0.f, 0.f};
5252

5353
/// General parameters
54+
bool AllowSharingFirstCluster = false;
5455
int ClusterSharing = 0;
5556
int MinTrackLength = 7;
5657
float NSigmaCut = 5;

‎Detectors/ITSMFT/ITS/tracking/include/ITStracking/ExternalAllocator.h‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
#ifndef TRACKINGITSU_INCLUDE_EXTERNALALLOCATOR_H_
1717
#define TRACKINGITSU_INCLUDE_EXTERNALALLOCATOR_H_
1818

19+
#include <memory_resource>
20+
1921
namespace o2::its
2022
{
2123

@@ -25,6 +27,36 @@ class ExternalAllocator
2527
virtual void* allocate(size_t) = 0;
2628
virtual void deallocate(char*, size_t) = 0;
2729
};
30+
31+
class ExternalAllocatorAdaptor final : public std::pmr::memory_resource
32+
{
33+
public:
34+
explicit ExternalAllocatorAdaptor(ExternalAllocator* alloc) : mAlloc(alloc) {}
35+
36+
protected:
37+
void* do_allocate(size_t bytes, size_t alignment) override
38+
{
39+
void* p = mAlloc->allocate(bytes);
40+
if (!p) {
41+
throw std::bad_alloc();
42+
}
43+
return p;
44+
}
45+
46+
void do_deallocate(void* p, size_t bytes, size_t) override
47+
{
48+
mAlloc->deallocate(static_cast<char*>(p), bytes);
49+
}
50+
51+
bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override
52+
{
53+
return this == &other;
54+
}
55+
56+
private:
57+
ExternalAllocator* mAlloc;
58+
};
59+
2860
} // namespace o2::its
2961

3062
#endif

0 commit comments

Comments
 (0)