Skip to content

Commit 377f6fa

Browse files
ldorauldorau
authored
Merge pull request #558 from vinser52/svinogra_ipc_api
Update L0 provider to use getpidfd function
2 parents ea64f07 + a0850d8 commit 377f6fa

17 files changed

+1061
-422
lines changed

‎src/provider/provider_level_zero.c‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -285,11 +285,16 @@ static umf_result_t ze_memory_provider_allocation_split(void *provider,
285285
return UMF_RESULT_ERROR_NOT_SUPPORTED;
286286
}
287287

288+
typedef struct ze_ipc_data_t {
289+
int pid;
290+
ze_ipc_mem_handle_t ze_handle;
291+
} ze_ipc_data_t;
292+
288293
static umf_result_t ze_memory_provider_get_ipc_handle_size(void *provider,
289294
size_t *size) {
290295
(void)provider;
291296
ASSERT(size != NULL);
292-
*size = sizeof(ze_ipc_mem_handle_t);
297+
*size = sizeof(ze_ipc_data_t);
293298
return UMF_RESULT_SUCCESS;
294299
}
295300

@@ -301,17 +306,19 @@ static umf_result_t ze_memory_provider_get_ipc_handle(void *provider,
301306
ASSERT(providerIpcData != NULL);
302307
(void)size;
303308
ze_result_t ze_result;
304-
ze_ipc_mem_handle_t *ze_ipc_handle = (ze_ipc_mem_handle_t *)providerIpcData;
309+
ze_ipc_data_t *ze_ipc_data = (ze_ipc_data_t *)providerIpcData;
305310
struct ze_memory_provider_t *ze_provider =
306311
(struct ze_memory_provider_t *)provider;
307312

308-
ze_result =
309-
g_ze_ops.zeMemGetIpcHandle(ze_provider->context, ptr, ze_ipc_handle);
313+
ze_result = g_ze_ops.zeMemGetIpcHandle(ze_provider->context, ptr,
314+
&ze_ipc_data->ze_handle);
310315
if (ze_result != ZE_RESULT_SUCCESS) {
311316
LOG_ERR("zeMemGetIpcHandle() failed.");
312317
return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
313318
}
314319

320+
ze_ipc_data->pid = utils_getpid();
321+
315322
return UMF_RESULT_SUCCESS;
316323
}
317324

@@ -322,7 +329,7 @@ static umf_result_t ze_memory_provider_put_ipc_handle(void *provider,
322329
ze_result_t ze_result;
323330
struct ze_memory_provider_t *ze_provider =
324331
(struct ze_memory_provider_t *)provider;
325-
ze_ipc_mem_handle_t *ze_ipc_handle = (ze_ipc_mem_handle_t *)providerIpcData;
332+
ze_ipc_data_t *ze_ipc_data = (ze_ipc_data_t *)providerIpcData;
326333

327334
if (g_ze_ops.zeMemPutIpcHandle == NULL) {
328335
// g_ze_ops.zeMemPutIpcHandle can be NULL because it was introduced
@@ -331,8 +338,8 @@ static umf_result_t ze_memory_provider_put_ipc_handle(void *provider,
331338
return UMF_RESULT_SUCCESS;
332339
}
333340

334-
ze_result =
335-
g_ze_ops.zeMemPutIpcHandle(ze_provider->context, *ze_ipc_handle);
341+
ze_result = g_ze_ops.zeMemPutIpcHandle(ze_provider->context,
342+
ze_ipc_data->ze_handle);
336343
if (ze_result != ZE_RESULT_SUCCESS) {
337344
LOG_ERR("zeMemPutIpcHandle() failed.");
338345
return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
@@ -347,12 +354,29 @@ static umf_result_t ze_memory_provider_open_ipc_handle(void *provider,
347354
ASSERT(providerIpcData != NULL);
348355
ASSERT(ptr != NULL);
349356
ze_result_t ze_result;
350-
ze_ipc_mem_handle_t *ze_ipc_handle = (ze_ipc_mem_handle_t *)providerIpcData;
357+
ze_ipc_data_t *ze_ipc_data = (ze_ipc_data_t *)providerIpcData;
351358
struct ze_memory_provider_t *ze_provider =
352359
(struct ze_memory_provider_t *)provider;
360+
int fd_local = -1;
361+
ze_ipc_mem_handle_t ze_ipc_handle = ze_ipc_data->ze_handle;
362+
363+
if (ze_ipc_data->pid != utils_getpid()) {
364+
int fd_remote = -1;
365+
memcpy(&fd_remote, &ze_ipc_handle, sizeof(fd_remote));
366+
umf_result_t umf_result =
367+
utils_duplicate_fd(ze_ipc_data->pid, fd_remote, &fd_local);
368+
if (umf_result != UMF_RESULT_SUCCESS) {
369+
LOG_PERR("duplicating file descriptor failed");
370+
return umf_result;
371+
}
372+
memcpy(&ze_ipc_handle, &fd_local, sizeof(fd_local));
373+
}
353374

354375
ze_result = g_ze_ops.zeMemOpenIpcHandle(
355-
ze_provider->context, ze_provider->device, *ze_ipc_handle, 0, ptr);
376+
ze_provider->context, ze_provider->device, ze_ipc_handle, 0, ptr);
377+
if (fd_local != -1) {
378+
(void)utils_close_fd(fd_local);
379+
}
356380
if (ze_result != ZE_RESULT_SUCCESS) {
357381
LOG_ERR("zeMemOpenIpcHandle() failed.");
358382
return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;

‎src/provider/provider_os_memory.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ create_fd_for_mmap(umf_os_memory_provider_params_t *in_params,
325325

326326
err_close_file:
327327
if (provider->fd > 0) {
328-
(void)os_close_fd(provider->fd);
328+
(void)utils_close_fd(provider->fd);
329329
}
330330

331331
return result;
@@ -1247,7 +1247,7 @@ static umf_result_t os_open_ipc_handle(void *provider, void *providerIpcData,
12471247
(void)os_shm_unlink(os_provider->shm_name);
12481248
} else {
12491249
umf_result_t umf_result =
1250-
os_duplicate_fd(os_ipc_data->pid, os_ipc_data->fd, &fd);
1250+
utils_duplicate_fd(os_ipc_data->pid, os_ipc_data->fd, &fd);
12511251
if (umf_result != UMF_RESULT_SUCCESS) {
12521252
LOG_PERR("duplicating file descriptor failed");
12531253
return umf_result;
@@ -1262,7 +1262,7 @@ static umf_result_t os_open_ipc_handle(void *provider, void *providerIpcData,
12621262
ret = UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
12631263
}
12641264

1265-
(void)os_close_fd(fd);
1265+
(void)utils_close_fd(fd);
12661266

12671267
return ret;
12681268
}

‎src/provider/provider_os_memory_internal.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,6 @@ size_t os_get_page_size(void);
9797

9898
void os_strerror(int errnum, char *buf, size_t buflen);
9999

100-
umf_result_t os_duplicate_fd(int pid, int fd_in, int *fd_out);
101-
102-
umf_result_t os_close_fd(int fd);
103-
104100
#ifdef __cplusplus
105101
}
106102
#endif

‎src/provider/provider_os_memory_posix.c‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -89,47 +89,3 @@ int os_purge(void *addr, size_t length, int advice) {
8989
void os_strerror(int errnum, char *buf, size_t buflen) {
9090
strerror_r(errnum, buf, buflen);
9191
}
92-
93-
umf_result_t os_duplicate_fd(int pid, int fd_in, int *fd_out) {
94-
// pidfd_getfd(2) is used to obtain a duplicate of another process's file descriptor.
95-
// Permission to duplicate another process's file descriptor
96-
// is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2))
97-
// that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface.
98-
// pidfd_getfd(2) is supported since Linux 5.6
99-
// pidfd_open(2) is supported since Linux 5.3
100-
#if defined(__NR_pidfd_open) && defined(__NR_pidfd_getfd)
101-
errno = 0;
102-
int pid_fd = syscall(SYS_pidfd_open, pid, 0);
103-
if (pid_fd == -1) {
104-
LOG_PDEBUG("SYS_pidfd_open");
105-
return UMF_RESULT_ERROR_UNKNOWN;
106-
}
107-
108-
int fd_dup = syscall(SYS_pidfd_getfd, pid_fd, fd_in, 0);
109-
close(pid_fd);
110-
if (fd_dup == -1) {
111-
LOG_PDEBUG("SYS_pidfd_getfd");
112-
return UMF_RESULT_ERROR_UNKNOWN;
113-
}
114-
115-
*fd_out = fd_dup;
116-
117-
return UMF_RESULT_SUCCESS;
118-
#else
119-
// TODO: find another way to obtain a duplicate of another process's file descriptor
120-
(void)pid; // unused
121-
(void)fd_in; // unused
122-
(void)fd_out; // unused
123-
errno = ENOTSUP;
124-
return UMF_RESULT_ERROR_NOT_SUPPORTED; // unsupported
125-
#endif /* defined(__NR_pidfd_open) && defined(__NR_pidfd_getfd) */
126-
}
127-
128-
umf_result_t os_close_fd(int fd) {
129-
if (close(fd)) {
130-
LOG_PERR("close() failed");
131-
return UMF_RESULT_ERROR_UNKNOWN;
132-
}
133-
134-
return UMF_RESULT_SUCCESS;
135-
}

‎src/provider/provider_os_memory_windows.c‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,3 @@ size_t os_get_page_size(void) {
151151
void os_strerror(int errnum, char *buf, size_t buflen) {
152152
strerror_s(buf, buflen, errnum);
153153
}
154-
155-
umf_result_t os_duplicate_fd(int pid, int fd_in, int *fd_out) {
156-
(void)pid; // unused
157-
(void)fd_in; // unused
158-
(void)fd_out; // unused
159-
return UMF_RESULT_ERROR_NOT_SUPPORTED; // unsupported
160-
}
161-
162-
umf_result_t os_close_fd(int fd) {
163-
(void)fd; // unused
164-
return UMF_RESULT_ERROR_NOT_SUPPORTED; // unsupported
165-
}

‎src/utils/utils_common.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
#include <stddef.h>
1515
#include <stdint.h>
1616

17+
#include <umf/base.h>
18+
1719
#ifdef __cplusplus
1820
extern "C" {
1921
#endif
@@ -77,6 +79,12 @@ int utils_getpid(void);
7779
// get the current thread ID
7880
int utils_gettid(void);
7981

82+
// close file descriptor
83+
int utils_close_fd(int fd);
84+
85+
// obtain a duplicate of another process's file descriptor
86+
umf_result_t utils_duplicate_fd(int pid, int fd_in, int *fd_out);
87+
8088
#ifdef __cplusplus
8189
}
8290
#endif

‎src/utils/utils_posix_common.c‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,22 @@
77
*
88
*/
99

10+
#include <errno.h>
1011
#include <stdlib.h>
1112
#include <string.h>
1213
#include <sys/syscall.h>
1314
#include <unistd.h>
1415

16+
#include "utils_common.h"
1517
#include "utils_concurrency.h"
18+
#include "utils_log.h"
19+
20+
#ifndef __NR_pidfd_open
21+
#define __NR_pidfd_open 434 /* Syscall id */
22+
#endif
23+
#ifndef __NR_pidfd_getfd
24+
#define __NR_pidfd_getfd 438 /* Syscall id */
25+
#endif
1626

1727
static UTIL_ONCE_FLAG Page_size_is_initialized = UTIL_ONCE_FLAG_INIT;
1828
static size_t Page_size;
@@ -38,3 +48,59 @@ int utils_gettid(void) {
3848
return syscall(SYS_gettid);
3949
#endif
4050
}
51+
52+
int utils_close_fd(int fd) { return close(fd); }
53+
54+
#ifndef __APPLE__
55+
static umf_result_t errno_to_umf_result(int err) {
56+
switch (err) {
57+
case EBADF:
58+
case EINVAL:
59+
case ESRCH:
60+
case EPERM:
61+
return UMF_RESULT_ERROR_INVALID_ARGUMENT;
62+
case EMFILE:
63+
case ENOMEM:
64+
return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
65+
case ENODEV:
66+
case ENOSYS:
67+
case ENOTSUP:
68+
return UMF_RESULT_ERROR_NOT_SUPPORTED;
69+
default:
70+
return UMF_RESULT_ERROR_UNKNOWN;
71+
}
72+
}
73+
#endif
74+
75+
umf_result_t utils_duplicate_fd(int pid, int fd_in, int *fd_out) {
76+
#ifdef __APPLE__
77+
(void)pid; // unused
78+
(void)fd_in; // unused
79+
(void)fd_out; // unused
80+
return UMF_RESULT_ERROR_NOT_SUPPORTED;
81+
#else
82+
// pidfd_getfd(2) is used to obtain a duplicate of another process's file descriptor.
83+
// Permission to duplicate another process's file descriptor
84+
// is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2))
85+
// that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface.
86+
// pidfd_getfd(2) is supported since Linux 5.6
87+
// pidfd_open(2) is supported since Linux 5.3
88+
errno = 0;
89+
int pid_fd = syscall(__NR_pidfd_open, pid, 0);
90+
if (pid_fd == -1) {
91+
LOG_PERR("__NR_pidfd_open");
92+
return errno_to_umf_result(errno);
93+
}
94+
95+
int fd_dup = syscall(__NR_pidfd_getfd, pid_fd, fd_in, 0);
96+
close(pid_fd);
97+
if (fd_dup == -1) {
98+
LOG_PERR("__NR_pidfd_open");
99+
return errno_to_umf_result(errno);
100+
}
101+
102+
*fd_out = fd_dup;
103+
104+
return UMF_RESULT_SUCCESS;
105+
#endif
106+
}

‎src/utils/utils_windows_common.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <processenv.h>
1313
#include <processthreadsapi.h>
1414

15+
#include "utils_common.h"
1516
#include "utils_concurrency.h"
1617

1718
#define BUFFER_SIZE 1024
@@ -33,3 +34,15 @@ size_t util_get_page_size(void) {
3334
int utils_getpid(void) { return GetCurrentProcessId(); }
3435

3536
int utils_gettid(void) { return GetCurrentThreadId(); }
37+
38+
int utils_close_fd(int fd) {
39+
(void)fd; // unused
40+
return -1;
41+
}
42+
43+
umf_result_t utils_duplicate_fd(int pid, int fd_in, int *fd_out) {
44+
(void)pid; // unused
45+
(void)fd_in; // unused
46+
(void)fd_out; // unused
47+
return UMF_RESULT_ERROR_NOT_SUPPORTED;
48+
}

‎test/CMakeLists.txt‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,14 +219,14 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
219219
# dlopen)
220220
add_umf_test(
221221
NAME provider_level_zero
222-
SRCS providers/provider_level_zero.cpp
222+
SRCS providers/provider_level_zero.cpp providers/level_zero_helpers.cpp
223223
LIBS ${UMF_UTILS_FOR_TEST} ze_loader)
224224
target_include_directories(umf_test-provider_level_zero
225225
PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS})
226226

227227
add_umf_test(
228228
NAME provider_level_zero_dlopen
229-
SRCS providers/provider_level_zero.cpp
229+
SRCS providers/provider_level_zero.cpp providers/level_zero_helpers.cpp
230230
LIBS ${UMF_UTILS_FOR_TEST})
231231
target_compile_definitions(umf_test-provider_level_zero_dlopen
232232
PUBLIC USE_DLOPEN=1)
@@ -322,6 +322,35 @@ if(LINUX)
322322
common/ipc_os_prov_common.c)
323323
add_umf_ipc_test(TEST ipc_os_prov_anon_fd)
324324
add_umf_ipc_test(TEST ipc_os_prov_shm)
325+
if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
326+
build_umf_test(
327+
NAME
328+
ipc_level_zero_prov_consumer
329+
SRCS
330+
providers/ipc_level_zero_prov_consumer.c
331+
common/ipc_common.c
332+
providers/ipc_level_zero_prov_common.c
333+
providers/level_zero_helpers.cpp
334+
LIBS
335+
ze_loader
336+
${UMF_UTILS_FOR_TEST})
337+
build_umf_test(
338+
NAME
339+
ipc_level_zero_prov_producer
340+
SRCS
341+
providers/ipc_level_zero_prov_producer.c
342+
common/ipc_common.c
343+
providers/ipc_level_zero_prov_common.c
344+
providers/level_zero_helpers.cpp
345+
LIBS
346+
ze_loader
347+
${UMF_UTILS_FOR_TEST})
348+
target_include_directories(umf_test-ipc_level_zero_prov_producer
349+
PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS})
350+
target_include_directories(umf_test-ipc_level_zero_prov_consumer
351+
PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS})
352+
add_umf_ipc_test(TEST ipc_level_zero_prov SRC_DIR providers)
353+
endif()
325354
else()
326355
message(STATUS "IPC tests are supported on Linux only - skipping")
327356
endif()

0 commit comments

Comments
 (0)