Skip to content

Commit 16308d5

Browse files
authored
Merge pull request #41287 from PixelTracksAlpaka/alpaka_port_13_1_vertex
Pixel Alpaka Migration: Vertexing [V]
2 parents 4aa04be + 3f668a2 commit 16308d5

24 files changed

+2234
-26
lines changed

RecoTracker/Configuration/python/RecoPixelVertexing_cff.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,33 @@
9898
pixelVerticesTask.copy()
9999
))
100100

101+
## pixel vertex reconstruction with Alpaka
102+
103+
# pixel vertex SoA producer with alpaka on the device
104+
from RecoTracker.PixelVertexFinding.pixelVertexProducerAlpakaPhase1_cfi import pixelVertexProducerAlpakaPhase1 as _pixelVerticesAlpakaPhase1
105+
from RecoTracker.PixelVertexFinding.pixelVertexProducerAlpakaPhase2_cfi import pixelVertexProducerAlpakaPhase2 as _pixelVerticesAlpakaPhase2
106+
pixelVerticesAlpaka = _pixelVerticesAlpakaPhase1.clone()
107+
phase2_tracker.toReplaceWith(pixelVerticesAlpaka,_pixelVerticesAlpakaPhase2.clone())
108+
109+
from RecoTracker.PixelVertexFinding.pixelVertexFromSoAAlpaka_cfi import pixelVertexFromSoAAlpaka as _pixelVertexFromSoAAlpaka
110+
alpaka.toReplaceWith(pixelVertices, _pixelVertexFromSoAAlpaka.clone())
111+
112+
# pixel vertex SoA producer with alpaka on the cpu, for validation
113+
pixelVerticesAlpakaSerial = pixelVerticesAlpaka.clone(
114+
pixelTrackSrc = 'pixelTracksAlpakaSerial',
115+
alpaka = None
116+
)
117+
pixelVerticesAlpakaSerial._TypedParameterizable__type = 'alpaka_serial_sync' + pixelVerticesAlpaka._TypedParameterizable__type.removesuffix('@alpaka')
118+
119+
alpaka.toReplaceWith(pixelVerticesTask, cms.Task(
120+
# Build the pixel vertices in SoA format with alpaka on the device
121+
pixelVerticesAlpaka,
122+
# Build the pixel vertices in SoA format with alpaka on the cpu (if requested by the validation)
123+
pixelVerticesAlpakaSerial,
124+
# Convert the pixel vertices from SoA format (on the host) to the legacy format
125+
pixelVertices
126+
))
127+
101128
# Tasks and Sequences
102129
recopixelvertexingTask = cms.Task(
103130
pixelTracksTask,
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1+
<use name="alpaka"/>
2+
<use name="rootmath"/>
13
<use name="CommonTools/Clustering1D"/>
24
<use name="DataFormats/GeometryCommonDetAlgo"/>
35
<use name="DataFormats/Math"/>
46
<use name="DataFormats/TrackReco"/>
57
<use name="DataFormats/VertexReco"/>
68
<use name="FWCore/MessageLogger"/>
79
<use name="FWCore/Utilities"/>
8-
<use name="rootmath"/>
910
<export>
1011
<lib name="1"/>
1112
</export>
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#ifndef RecoTracker_PixelVertexFinding_interface_PixelVertexWorkSpaceLayout_h
2+
#define RecoTracker_PixelVertexFinding_interface_PixelVertexWorkSpaceLayout_h
3+
4+
#include <alpaka/alpaka.hpp>
5+
6+
#include "DataFormats/SoATemplate/interface/SoALayout.h"
7+
8+
// Intermediate data used in the vertex reco algos
9+
// For internal use only
10+
namespace vertexFinder {
11+
12+
GENERATE_SOA_LAYOUT(PixelVertexWSSoALayout,
13+
SOA_COLUMN(uint16_t, itrk), // index of original track
14+
SOA_COLUMN(float, zt), // input track z at bs
15+
SOA_COLUMN(float, ezt2), // input error^2 on the above
16+
SOA_COLUMN(float, ptt2), // input pt^2 on the above
17+
SOA_COLUMN(uint8_t, izt), // interized z-position of input tracks
18+
SOA_COLUMN(int32_t, iv), // vertex index for each associated track
19+
SOA_SCALAR(uint32_t, ntrks), // number of "selected tracks"
20+
SOA_SCALAR(uint32_t, nvIntermediate)) // the number of vertices after splitting pruning etc.
21+
22+
using PixelVertexWorkSpaceSoALayout = PixelVertexWSSoALayout<>;
23+
using PixelVertexWorkSpaceSoAView = PixelVertexWSSoALayout<>::View;
24+
using PixelVertexWorkSpaceSoAConstView = PixelVertexWSSoALayout<>::ConstView;
25+
26+
ALPAKA_FN_HOST_ACC ALPAKA_FN_INLINE void init(PixelVertexWorkSpaceSoAView& workspace_view) {
27+
workspace_view.ntrks() = 0;
28+
workspace_view.nvIntermediate() = 0;
29+
}
30+
31+
} // namespace vertexFinder
32+
33+
#endif // RecoTracker_PixelVertexFinding_interface_PixelVertexWorkSpaceLayout_h

RecoTracker/PixelVertexFinding/plugins/BuildFile.xml

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
<use name="CUDADataFormats/Vertex"/>
2-
<use name="CUDADataFormats/Track"/>
31
<use name="CommonTools/Clustering1D"/>
42
<use name="DataFormats/BeamSpot"/>
53
<use name="DataFormats/GeometryCommonDetAlgo"/>
@@ -17,18 +15,33 @@
1715
<use name="FWCore/Utilities"/>
1816
<use name="Geometry/Records"/>
1917
<use name="Geometry/TrackerGeometryBuilder"/>
20-
<use name="HeterogeneousCore/CUDACore"/>
21-
<use name="HeterogeneousCore/CUDAUtilities"/>
2218
<use name="RecoLocalTracker/ClusterParameterEstimator"/>
2319
<use name="RecoLocalTracker/Records"/>
2420
<use name="RecoTracker/PixelVertexFinding"/>
2521
<use name="SimDataFormats/PileupSummaryInfo"/>
22+
2623
<iftool name="cuda-gcc-support">
2724
<use name="cuda"/>
2825
<set name="cuda_src" value="*.cu"/>
2926
<else/>
3027
<set name="cuda_src" value=""/>
3128
</iftool>
29+
3230
<library file="*.cc ${cuda_src}" name="RecoPixelVertexingPixelVertexFindingPlugins">
31+
<use name="CUDADataFormats/Vertex"/>
32+
<use name="CUDADataFormats/Track"/>
33+
<use name="HeterogeneousCore/CUDACore"/>
34+
<use name="HeterogeneousCore/CUDAUtilities"/>
35+
<flags EDM_PLUGIN="1"/>
36+
</library>
37+
38+
<library file="alpaka/*.cc" name="RecoPixelVertexingPixelVertexFindingPluginsPortable">
39+
<use name="alpaka"/>
40+
<use name="DataFormats/Portable"/>
41+
<use name="DataFormats/TrackSoA"/>
42+
<use name="DataFormats/VertexSoA"/>
43+
<use name="HeterogeneousCore/AlpakaCore"/>
44+
<use name="HeterogeneousCore/AlpakaInterface"/>
45+
<flags ALPAKA_BACKENDS="1"/>
3346
<flags EDM_PLUGIN="1"/>
3447
</library>
Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
#include "DataFormats/BeamSpot/interface/BeamSpot.h"
2+
#include "DataFormats/Common/interface/OrphanHandle.h"
3+
#include "DataFormats/TrackReco/interface/Track.h"
4+
#include "DataFormats/TrackReco/interface/TrackExtra.h"
5+
#include "DataFormats/TrackReco/interface/TrackFwd.h"
6+
#include "DataFormats/VertexReco/interface/Vertex.h"
7+
#include "DataFormats/VertexReco/interface/VertexFwd.h"
8+
#include "DataFormats/VertexSoA/interface/ZVertexHost.h"
9+
#include "FWCore/Framework/interface/Event.h"
10+
#include "FWCore/Framework/interface/EventSetup.h"
11+
#include "FWCore/Framework/interface/MakerMacros.h"
12+
#include "FWCore/Framework/interface/global/EDProducer.h"
13+
#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
14+
#include "FWCore/ParameterSet/interface/ParameterSet.h"
15+
#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
16+
#include "FWCore/PluginManager/interface/ModuleDef.h"
17+
#include "FWCore/Utilities/interface/EDGetToken.h"
18+
#include "FWCore/Utilities/interface/InputTag.h"
19+
#include "Geometry/Records/interface/TrackerTopologyRcd.h"
20+
#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h"
21+
22+
class PixelVertexProducerFromSoAAlpaka : public edm::global::EDProducer<> {
23+
public:
24+
using IndToEdm = std::vector<uint32_t>;
25+
26+
explicit PixelVertexProducerFromSoAAlpaka(const edm::ParameterSet &iConfig);
27+
~PixelVertexProducerFromSoAAlpaka() override = default;
28+
29+
static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
30+
31+
private:
32+
void produce(edm::StreamID streamID, edm::Event &iEvent, const edm::EventSetup &iSetup) const override;
33+
34+
edm::EDGetTokenT<ZVertexHost> tokenVertex_;
35+
edm::EDGetTokenT<reco::BeamSpot> tokenBeamSpot_;
36+
edm::EDGetTokenT<reco::TrackCollection> tokenTracks_;
37+
edm::EDGetTokenT<IndToEdm> tokenIndToEdm_;
38+
};
39+
40+
PixelVertexProducerFromSoAAlpaka::PixelVertexProducerFromSoAAlpaka(const edm::ParameterSet &conf)
41+
: tokenVertex_(consumes(conf.getParameter<edm::InputTag>("src"))),
42+
tokenBeamSpot_(consumes(conf.getParameter<edm::InputTag>("beamSpot"))),
43+
tokenTracks_(consumes(conf.getParameter<edm::InputTag>("TrackCollection"))),
44+
tokenIndToEdm_(consumes(conf.getParameter<edm::InputTag>("TrackCollection"))) {
45+
produces<reco::VertexCollection>();
46+
}
47+
48+
void PixelVertexProducerFromSoAAlpaka::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
49+
edm::ParameterSetDescription desc;
50+
51+
desc.add<edm::InputTag>("TrackCollection", edm::InputTag("pixelTracks"));
52+
desc.add<edm::InputTag>("beamSpot", edm::InputTag("offlineBeamSpot"));
53+
desc.add<edm::InputTag>("src", edm::InputTag("pixelVerticesAlpaka"));
54+
55+
descriptions.add("pixelVertexFromSoAAlpaka", desc);
56+
}
57+
58+
void PixelVertexProducerFromSoAAlpaka::produce(edm::StreamID streamID,
59+
edm::Event &iEvent,
60+
const edm::EventSetup &) const {
61+
auto vertexes = std::make_unique<reco::VertexCollection>();
62+
63+
auto tracksHandle = iEvent.getHandle(tokenTracks_);
64+
auto tracksSize = tracksHandle->size();
65+
auto const &indToEdm = iEvent.get(tokenIndToEdm_);
66+
auto bsHandle = iEvent.getHandle(tokenBeamSpot_);
67+
68+
float x0 = 0, y0 = 0, z0 = 0, dxdz = 0, dydz = 0;
69+
std::vector<int32_t> itrk;
70+
itrk.reserve(64); // avoid first relocations
71+
if (!bsHandle.isValid()) {
72+
edm::LogWarning("PixelVertexProducer") << "No beamspot found. returning vertexes with (0,0,Z) ";
73+
} else {
74+
const reco::BeamSpot &bs = *bsHandle;
75+
x0 = bs.x0();
76+
y0 = bs.y0();
77+
z0 = bs.z0();
78+
dxdz = bs.dxdz();
79+
dydz = bs.dydz();
80+
}
81+
82+
auto const &soa = iEvent.get(tokenVertex_);
83+
84+
int nv = soa.view().nvFinal();
85+
86+
#ifdef PIXVERTEX_DEBUG_PRODUCE
87+
std::cout << "converting " << nv << " vertices "
88+
<< " from " << indToEdm.size() << " tracks" << std::endl;
89+
#endif // PIXVERTEX_DEBUG_PRODUCE
90+
91+
std::set<uint32_t> uind; // for verifing index consistency
92+
for (int j = nv - 1; j >= 0; --j) {
93+
auto i = soa.view()[j].sortInd(); // on gpu sorted in ascending order....
94+
assert(i < nv);
95+
uind.insert(i);
96+
assert(itrk.empty());
97+
auto z = soa.view()[i].zv();
98+
auto x = x0 + dxdz * z;
99+
auto y = y0 + dydz * z;
100+
z += z0;
101+
reco::Vertex::Error err;
102+
err(2, 2) = 1.f / soa.view()[i].wv();
103+
err(2, 2) *= 2.; // artifically inflate error
104+
//Copy also the tracks (no intention to be efficient....)
105+
for (auto k = 0U; k < indToEdm.size(); ++k) {
106+
if (soa.view()[k].idv() == int16_t(i))
107+
itrk.push_back(k);
108+
}
109+
auto nt = itrk.size();
110+
if (nt == 0) {
111+
#ifdef PIXVERTEX_DEBUG_PRODUCE
112+
std::cout << "vertex " << i << " with no tracks..." << std::endl;
113+
#endif // PIXVERTEX_DEBUG_PRODUCE
114+
continue;
115+
}
116+
if (nt < 2) {
117+
itrk.clear();
118+
continue;
119+
} // remove outliers
120+
(*vertexes).emplace_back(reco::Vertex::Point(x, y, z), err, soa.view()[i].chi2(), soa.view()[i].ndof(), nt);
121+
auto &v = (*vertexes).back();
122+
v.reserve(itrk.size());
123+
for (auto it : itrk) {
124+
assert(it < int(indToEdm.size()));
125+
auto k = indToEdm[it];
126+
if (k > tracksSize) {
127+
edm::LogWarning("PixelVertexProducer") << "oops track " << it << " does not exists on CPU " << k;
128+
continue;
129+
}
130+
auto tk = reco::TrackRef(tracksHandle, k);
131+
v.add(tk);
132+
}
133+
itrk.clear();
134+
}
135+
136+
LogDebug("PixelVertexProducer") << ": Found " << vertexes->size() << " vertexes\n";
137+
for (unsigned int i = 0; i < vertexes->size(); ++i) {
138+
LogDebug("PixelVertexProducer") << "Vertex number " << i << " has " << (*vertexes)[i].tracksSize()
139+
<< " tracks with a position of " << (*vertexes)[i].z() << " +- "
140+
<< std::sqrt((*vertexes)[i].covariance(2, 2));
141+
}
142+
143+
// legacy logic....
144+
if (vertexes->empty() && bsHandle.isValid()) {
145+
const reco::BeamSpot &bs = *bsHandle;
146+
147+
GlobalError bse(bs.rotatedCovariance3D());
148+
if ((bse.cxx() <= 0.) || (bse.cyy() <= 0.) || (bse.czz() <= 0.)) {
149+
AlgebraicSymMatrix33 we;
150+
we(0, 0) = 10000;
151+
we(1, 1) = 10000;
152+
we(2, 2) = 10000;
153+
vertexes->push_back(reco::Vertex(bs.position(), we, 0., 0., 0));
154+
155+
edm::LogInfo("PixelVertexProducer") << "No vertices found. Beamspot with invalid errors " << bse.matrix()
156+
<< "\nWill put Vertex derived from dummy-fake BeamSpot into Event.\n"
157+
<< (*vertexes)[0].x() << "\n"
158+
<< (*vertexes)[0].y() << "\n"
159+
<< (*vertexes)[0].z() << "\n";
160+
} else {
161+
vertexes->push_back(reco::Vertex(bs.position(), bs.rotatedCovariance3D(), 0., 0., 0));
162+
163+
edm::LogInfo("PixelVertexProducer") << "No vertices found. Will put Vertex derived from BeamSpot into Event:\n"
164+
<< (*vertexes)[0].x() << "\n"
165+
<< (*vertexes)[0].y() << "\n"
166+
<< (*vertexes)[0].z() << "\n";
167+
}
168+
} else if (vertexes->empty() && !bsHandle.isValid()) {
169+
edm::LogWarning("PixelVertexProducer") << "No beamspot and no vertex found. No vertex returned.";
170+
}
171+
172+
iEvent.put(std::move(vertexes));
173+
}
174+
175+
DEFINE_FWK_MODULE(PixelVertexProducerFromSoAAlpaka);

RecoTracker/PixelVertexFinding/plugins/PixelVertexWorkSpaceSoADevice.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
template <int32_t S>
99
class PixelVertexWorkSpaceSoADevice : public cms::cuda::PortableDeviceCollection<PixelVertexWSSoALayout<>> {
1010
public:
11-
PixelVertexWorkSpaceSoADevice() = default;
11+
explicit PixelVertexWorkSpaceSoADevice() = default;
1212

1313
// Constructor which specifies the SoA size and CUDA stream
1414
explicit PixelVertexWorkSpaceSoADevice(cudaStream_t stream)

RecoTracker/PixelVertexFinding/plugins/PixelVertexWorkSpaceSoAHost.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ template <int32_t S>
99
class PixelVertexWorkSpaceSoAHost : public cms::cuda::PortableHostCollection<PixelVertexWSSoALayout<>> {
1010
public:
1111
explicit PixelVertexWorkSpaceSoAHost() : PortableHostCollection<PixelVertexWSSoALayout<>>(S) {}
12+
1213
// Constructor which specifies the SoA size and CUDA stream
1314
explicit PixelVertexWorkSpaceSoAHost(cudaStream_t stream)
1415
: PortableHostCollection<PixelVertexWSSoALayout<>>(S, stream) {}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#ifndef RecoTracker_PixelVertexFinding_plugins_PixelVertexWorkSpaceSoAHostAlpaka_h
2+
#define RecoTracker_PixelVertexFinding_plugins_PixelVertexWorkSpaceSoAHostAlpaka_h
3+
4+
#include <alpaka/alpaka.hpp>
5+
6+
#include "DataFormats/Portable/interface/PortableHostCollection.h"
7+
#include "RecoTracker/PixelVertexFinding/interface/PixelVertexWorkSpaceLayout.h"
8+
9+
namespace vertexFinder {
10+
11+
using PixelVertexWorkSpaceSoAHost = PortableHostCollection<PixelVertexWSSoALayout<>>;
12+
13+
} // namespace vertexFinder
14+
15+
#endif // RecoTracker_PixelVertexFinding_plugins_PixelVertexWorkSpaceSoAHostAlpaka_h

0 commit comments

Comments
 (0)