Skip to content

Commit 6b83282

Browse files
davschnellerdavschneller
committed
Begin adding peer access methods
1 parent 772be05 commit 6b83282

File tree

6 files changed

+178
-0
lines changed

6 files changed

+178
-0
lines changed

‎AbstractAPI.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,15 @@ struct AbstractAPI {
114114
virtual void putProfilingMark(const std::string& name, ProfilingColors color) = 0;
115115
virtual void popLastProfilingMark() = 0;
116116

117+
virtual bool canAccessPeer(int otherDeviceId) = 0;
118+
virtual void switchPeerAccess(int otherDeviceId, bool enable) = 0;
119+
120+
virtual std::vector<uint8_t> makeIpcMemHandle(void* ptr) = 0;
121+
virtual void* openIpcMemHandle(const std::vector<uint8_t>& handle) = 0;
122+
virtual void closeIpcMemHandle(void* ptr) = 0;
123+
virtual std::vector<uint8_t> makeIpcEventHandle(void* event) = 0;
124+
virtual void* openIpcEventHandle(const std::vector<uint8_t>& handle) = 0;
125+
117126
virtual void setupPrinting(int rank) = 0;
118127

119128
bool hasFinalized() { return m_isFinalized; }

‎interfaces/cuda/CudaWrappedAPI.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <vector>
2121

2222
namespace device {
23+
2324
class ConcreteAPI : public AbstractAPI {
2425
public:
2526
ConcreteAPI();
@@ -111,6 +112,15 @@ class ConcreteAPI : public AbstractAPI {
111112

112113
bool isUnifiedMemoryDefault() override;
113114

115+
bool canAccessPeer(int otherDeviceId) override;
116+
void switchPeerAccess(int otherDeviceId, bool enable) override;
117+
118+
std::vector<uint8_t> makeIpcMemHandle(void* ptr) override;
119+
void* openIpcMemHandle(const std::vector<uint8_t>& handle) override;
120+
void closeIpcMemHandle(void* ptr) override;
121+
std::vector<uint8_t> makeIpcEventHandle(void* event) override;
122+
void* openIpcEventHandle(const std::vector<uint8_t>& handle) override;
123+
114124
void setupPrinting(int rank) override;
115125

116126
private:

‎interfaces/cuda/Interop.cu‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// SPDX-FileCopyrightText: 2026 SeisSol Group
2+
//
3+
// SPDX-License-Identifier: BSD-3-Clause
4+
5+
#include "AbstractAPI.h"
6+
#include "CudaWrappedAPI.h"
7+
#include "Internals.h"
8+
#include "utils/logger.h"
9+
10+
using namespace device;
11+
12+
namespace {
13+
14+
template <typename T>
15+
std::vector<uint8_t> serialize(const T& data) {
16+
std::vector<uint8_t> ser(sizeof(T));
17+
memcpy(ser.data(), &data, sizeof(T));
18+
return ser;
19+
}
20+
21+
template <typename T>
22+
T deserialize(const std::vector<uint8_t>& ser) {
23+
T data{};
24+
memcpy(&data, ser.data(), sizeof(T));
25+
return data;
26+
}
27+
28+
} // namespace
29+
30+
bool ConcreteAPI::canAccessPeer(int otherDeviceId) {
31+
int result = 0;
32+
APIWRAP(cudaDeviceCanAccessPeer(&result, getDeviceId(), otherDeviceId));
33+
return result != 0;
34+
}
35+
36+
void ConcreteAPI::switchPeerAccess(int otherDeviceId, bool enable) {
37+
if (enable) {
38+
APIWRAP(cudaDeviceEnablePeerAccess(otherDeviceId, 0));
39+
} else {
40+
APIWRAP(cudaDeviceDisablePeerAccess(otherDeviceId));
41+
}
42+
}
43+
44+
std::vector<uint8_t> ConcreteAPI::makeIpcMemHandle(void* ptr) {
45+
cudaIpcMemHandle_t handle{};
46+
APIWRAP(cudaIpcGetMemHandle(&handle, ptr));
47+
return serialize(handle);
48+
}
49+
50+
void* ConcreteAPI::openIpcMemHandle(const std::vector<uint8_t>& handle) {
51+
void* ptr = nullptr;
52+
APIWRAP(cudaIpcOpenMemHandle(
53+
&ptr, deserialize<cudaIpcMemHandle_t>(handle), cudaIpcMemLazyEnablePeerAccess));
54+
return ptr;
55+
}
56+
57+
void ConcreteAPI::closeIpcMemHandle(void* ptr) { APIWRAP(cudaIpcCloseMemHandle(ptr)); }
58+
59+
std::vector<uint8_t> ConcreteAPI::makeIpcEventHandle(void* event) {
60+
cudaIpcEventHandle_t handle{};
61+
APIWRAP(cudaIpcGetEventHandle(&handle, static_cast<cudaEvent_t>(event)));
62+
return serialize(handle);
63+
}
64+
65+
void* ConcreteAPI::openIpcEventHandle(const std::vector<uint8_t>& handle) {
66+
cudaEvent_t event{};
67+
APIWRAP(cudaIpcOpenEventHandle(&event, deserialize<cudaIpcEventHandle_t>(handle)));
68+
return static_cast<void*>(event);
69+
}

‎interfaces/hip/HipWrappedAPI.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,15 @@ class ConcreteAPI : public AbstractAPI {
110110

111111
bool isUnifiedMemoryDefault() override;
112112

113+
bool canAccessPeer(int otherDeviceId) override;
114+
void switchPeerAccess(int otherDeviceId, bool enable) override;
115+
116+
std::vector<uint8_t> makeIpcMemHandle(void* ptr) override;
117+
void* openIpcMemHandle(const std::vector<uint8_t>& handle) override;
118+
void closeIpcMemHandle(void* ptr) override;
119+
std::vector<uint8_t> makeIpcEventHandle(void* event) override;
120+
void* openIpcEventHandle(const std::vector<uint8_t>& handle) override;
121+
113122
void setupPrinting(int rank) override;
114123

115124
private:

‎interfaces/hip/Interop.cpp‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// SPDX-FileCopyrightText: 2026 SeisSol Group
2+
//
3+
// SPDX-License-Identifier: BSD-3-Clause
4+
5+
#include "AbstractAPI.h"
6+
#include "HipWrappedAPI.h"
7+
#include "Internals.h"
8+
#include "utils/logger.h"
9+
10+
#include <cstdint>
11+
#include <vector>
12+
13+
using namespace device;
14+
15+
namespace {
16+
17+
template <typename T>
18+
std::vector<uint8_t> serialize(const T& data) {
19+
std::vector<uint8_t> ser(sizeof(T));
20+
memcpy(ser.data(), &data, sizeof(T));
21+
return ser;
22+
}
23+
24+
template <typename T>
25+
T deserialize(const std::vector<uint8_t>& ser) {
26+
T data{};
27+
memcpy(&data, ser.data(), sizeof(T));
28+
return data;
29+
}
30+
31+
} // namespace
32+
33+
bool ConcreteAPI::canAccessPeer(int otherDeviceId) {
34+
int result = 0;
35+
APIWRAP(cudaDeviceCanAccessPeer(&result, getDeviceId(), otherDeviceId));
36+
return result != 0;
37+
}
38+
39+
void ConcreteAPI::switchPeerAccess(int otherDeviceId, bool enable) {
40+
if (enable) {
41+
APIWRAP(cudaDeviceEnablePeerAccess(otherDeviceId, 0));
42+
} else {
43+
APIWRAP(cudaDeviceDisablePeerAccess(otherDeviceId));
44+
}
45+
}
46+
47+
std::vector<uint8_t> ConcreteAPI::makeIpcMemHandle(void* ptr) {
48+
cudaIpcMemHandle_t handle{};
49+
APIWRAP(cudaIpcGetMemHandle(&handle, ptr));
50+
return serialize(handle);
51+
}
52+
53+
void* ConcreteAPI::openIpcMemHandle(const std::vector<uint8_t>& handle) {
54+
void* ptr = nullptr;
55+
APIWRAP(cudaIpcOpenMemHandle(
56+
&ptr, deserialize<cudaIpcMemHandle_t>(handle), cudaIpcMemLazyEnablePeerAccess));
57+
return ptr;
58+
}
59+
60+
void ConcreteAPI::closeIpcMemHandle(void* ptr) { APIWRAP(cudaIpcCloseMemHandle(ptr)); }
61+
62+
std::vector<uint8_t> ConcreteAPI::makeIpcEventHandle(void* event) {
63+
cudaIpcEventHandle_t handle{};
64+
APIWRAP(cudaIpcGetEventHandle(&handle, static_cast<cudaEvent_t>(event)));
65+
return serialize(handle);
66+
}
67+
68+
void* ConcreteAPI::openIpcEventHandle(const std::vector<uint8_t>& handle) {
69+
cudaEvent_t event{};
70+
APIWRAP(cudaIpcOpenEventHandle(&event, deserialize<cudaIpcEventHandle_t>(handle)));
71+
return static_cast<void*>(event);
72+
}

‎interfaces/sycl/SyclWrappedAPI.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,15 @@ class ConcreteAPI : public AbstractAPI {
146146

147147
bool isUnifiedMemoryDefault() override;
148148

149+
bool canAccessPeer(int otherDeviceId) override;
150+
void switchPeerAccess(int otherDeviceId, bool enable) override;
151+
152+
std::vector<uint8_t> makeIpcMemHandle(void* ptr) override;
153+
void* openIpcMemHandle(const std::vector<uint8_t>& handle) override;
154+
void closeIpcMemHandle(void* ptr) override;
155+
std::vector<uint8_t> makeIpcEventHandle(void* event) override;
156+
void* openIpcEventHandle(const std::vector<uint8_t>& handle) override;
157+
149158
void setupPrinting(int rank) override;
150159

151160
private:

0 commit comments

Comments
 (0)