Skip to content

Commit 5b1cb52

Browse files
ryanhankinsryanhankins
committed
prov/cxi: add cxi unit tests and update man/fi_cxi.7
Add fi_writedata test and update fi_cxi.7.md Signed-off-by: Ryan Hankins <ryan.hankins@hpe.com>
1 parent cc386e6 commit 5b1cb52

File tree

4 files changed

+207
-1
lines changed

4 files changed

+207
-1
lines changed

‎man/fi_cxi.7.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1315,6 +1315,25 @@ The CXI provider checks for the following environment variables:
13151315
: Enable enforcement of triggered operation limit. Doing this can prevent
13161316
fi_control(FI_QUEUE_WORK) deadlocking at the cost of performance.
13171317

1318+
*FI_CXI_ENABLE_WRITEDATA*
1319+
: Controls provider support for the fi_writedata() and fi_inject_writedata() RMA
1320+
operations. When enabled and the domain attribute cq_data_size is non-zero,
1321+
the CXI provider implements handling to generate solicited RMA completions that
1322+
include immediate data; completions will include FI_REMOTE_CQ_DATA and will
1323+
report source information when FI_SOURCE is enabled (FI_SOURCE_ERR behavior is
1324+
followed on resolution failures).
1325+
1326+
Note that the CXI_RX_CQ_DATA capability is not required and writedata RMA
1327+
operations do not consume posted receive buffers on the target. The feature
1328+
is gated by domain/endpoint capabilities (for example, a non-zero
1329+
domain_attr->cq_data_size in the libfabric API) and endpoint support. Internally,
1330+
the combination of domain and endpoint cq_data_size sets rma_cq_data_size. Only
1331+
provider MR keys are supported.
1332+
1333+
This option is disabled by default; enable it only when applications require
1334+
immediate-data delivery on RMA completions or for controlled testing and
1335+
debugging.
1336+
13181337
*FI_CXI_MR_CACHE_EVENTS_DISABLE_POLL_NSECS*
13191338
: Max amount of time to poll when disabling an MR configured with MR match events.
13201339

‎prov/cxi/Makefile.include‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ nodist_prov_cxi_test_cxitest_SOURCES = \
130130
prov/cxi/test/fork.c \
131131
prov/cxi/test/mem_reg.c \
132132
prov/cxi/test/nic.c \
133-
prov/cxi/test/mr_cache.c
133+
prov/cxi/test/mr_cache.c \
134+
prov/cxi/test/writedata.c
134135

135136
prov_cxi_test_cxitest_CPPFLAGS = $(AM_CPPFLAGS) $(cxi_CPPFLAGS) \
136137
$(cxitest_CPPFLAGS) $(PTHREAD_CFLAGS)

‎prov/cxi/test/test.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,10 @@ unlimited_triggered_ops_test=(
153153

154154
mr_cache_test=("./cxitest --verbose --tap=cxitest-mr_cache_test.tap --filter=\"mr_cache/*\" -j 1")
155155

156+
writedata_rma_test=(
157+
"FI_CXI_ENABLE_WRITEDATA=1 ./cxitest --verbose --filter=\"@(rma*|mr*)/*\" -j 1 --tap=cxitest-writedata-rma.tap"
158+
)
159+
156160
long_test_suite=(
157161
"basic_test"
158162
"swget_test"
@@ -179,6 +183,7 @@ long_test_suite=(
179183
"fork_safe_kdreg2_test"
180184
"unlimited_triggered_ops_test"
181185
"mr_cache_test"
186+
"writedata_rma_test"
182187
)
183188

184189
# ################################################################

‎prov/cxi/test/writedata.c‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*
2+
* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only
3+
*
4+
* Copyright (c) 2025 Hewlett Packard Enterprise Development LP
5+
*/
6+
7+
#include <stdio.h>
8+
#include <stdlib.h>
9+
10+
#include <criterion/criterion.h>
11+
12+
#include "cxip.h"
13+
#include "cxip_test_common.h"
14+
15+
void cxit_setup_rma_writedata(void)
16+
{
17+
cxip_env.enable_writedata = 1;
18+
cxit_setup_getinfo();
19+
cxit_fi_hints->caps |= FI_RMA | FI_RMA_EVENT | FI_MSG | FI_SOURCE;
20+
cxit_fi_hints->domain_attr->mr_mode |= FI_MR_PROV_KEY | FI_MR_ENDPOINT;
21+
cxit_setup_rma();
22+
}
23+
24+
TestSuite(rma_writedata, .init = cxit_setup_rma_writedata, .fini = cxit_teardown_rma,
25+
.timeout = CXIT_DEFAULT_TIMEOUT);
26+
27+
Test(rma_writedata, simple)
28+
{
29+
int ret;
30+
struct mem_region mem_window;
31+
uint64_t key_val = 0x1234;
32+
struct fi_cq_tagged_entry cqe;
33+
uint64_t immediate_data = 0xDEADBEEF;
34+
size_t len = 1024;
35+
uint8_t *send_buf;
36+
struct cxip_ep *ep = container_of(cxit_ep, struct cxip_ep, ep);
37+
send_buf = calloc(1, len);
38+
cr_assert_not_null(send_buf, "send_buf alloc failed");
39+
memset(send_buf, 0xAA, len);
40+
41+
/* Manual MR creation to include FI_RMA_EVENT flag */
42+
mem_window.mem = calloc(1, len);
43+
cr_assert_not_null(mem_window.mem, "mem_window alloc failed");
44+
45+
ret = fi_mr_reg(cxit_domain, mem_window.mem, len,
46+
FI_REMOTE_WRITE | FI_REMOTE_READ | FI_SEND | FI_RECV,
47+
0, key_val, FI_RMA_EVENT, &mem_window.mr, NULL);
48+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_reg failed %d", ret);
49+
50+
ret = fi_mr_bind(mem_window.mr, &cxit_ep->fid, 0);
51+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_bind failed %d", ret);
52+
53+
if (cxit_rem_cntr) {
54+
ret = fi_mr_bind(mem_window.mr, &cxit_rem_cntr->fid, FI_REMOTE_WRITE);
55+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_bind(cntr) failed %d", ret);
56+
}
57+
58+
ret = fi_mr_enable(mem_window.mr);
59+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_enable failed %d", ret);
60+
61+
key_val = fi_mr_key(mem_window.mr);
62+
63+
/* Perform writedata */
64+
ret = fi_writedata(cxit_ep, send_buf, len, NULL, immediate_data,
65+
cxit_ep_fi_addr, 0, key_val, NULL);
66+
if (ep->ep_obj->domain->rma_cq_data_size) {
67+
cr_assert_eq(ret, FI_SUCCESS, "fi_writedata failed %d", ret);
68+
} else {
69+
cr_assert_eq(ret, -FI_ENOSYS, "fi_writedata bad return %d", ret);
70+
goto done;
71+
}
72+
73+
/* Wait for local completion (send) */
74+
ret = cxit_await_completion(cxit_tx_cq, &cqe);
75+
cr_assert_eq(ret, 1, "fi_cq_read (tx) failed %d", ret);
76+
validate_tx_event(&cqe, FI_RMA | FI_WRITE, NULL);
77+
78+
/* Wait for remote completion (recv) */
79+
ret = cxit_await_completion(cxit_rx_cq, &cqe);
80+
cr_assert_eq(ret, 1, "fi_cq_read (rx) failed %d", ret);
81+
82+
/* Validate remote completion */
83+
cr_assert(cqe.flags & FI_REMOTE_WRITE, "Missing FI_REMOTE_WRITE flag");
84+
cr_assert_eq(cqe.data, immediate_data, "Data mismatch: 0x%lx != 0x%lx",
85+
cqe.data, immediate_data);
86+
87+
/* Verify data */
88+
for (size_t i = 0; i < len; i++) {
89+
cr_assert_eq(mem_window.mem[i], send_buf[i],
90+
"Memory mismatch at index %zu: 0x%02x != 0x%02x",
91+
i, mem_window.mem[i], send_buf[i]);
92+
}
93+
done:
94+
free(send_buf);
95+
mr_destroy(&mem_window);
96+
}
97+
98+
Test(rma_writedata, with_source)
99+
{
100+
int ret;
101+
struct mem_region mem_window;
102+
uint64_t key_val = 0x1234;
103+
struct fi_cq_tagged_entry cqe;
104+
uint64_t immediate_data = 0xDEADBEEF;
105+
size_t len = 1024;
106+
uint8_t *send_buf;
107+
struct cxip_ep *ep = container_of(cxit_ep, struct cxip_ep, ep);
108+
fi_addr_t src_addr;
109+
int poll_count = 0;
110+
111+
send_buf = calloc(1, len);
112+
cr_assert_not_null(send_buf, "send_buf alloc failed");
113+
memset(send_buf, 0xAA, len);
114+
115+
/* Manual MR creation to include FI_RMA_EVENT flag */
116+
mem_window.mem = calloc(1, len);
117+
cr_assert_not_null(mem_window.mem, "mem_window alloc failed");
118+
119+
ret = fi_mr_reg(cxit_domain, mem_window.mem, len,
120+
FI_REMOTE_WRITE | FI_REMOTE_READ | FI_SEND | FI_RECV,
121+
0, key_val, FI_RMA_EVENT, &mem_window.mr, NULL);
122+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_reg failed %d", ret);
123+
124+
ret = fi_mr_bind(mem_window.mr, &cxit_ep->fid, 0);
125+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_bind failed %d", ret);
126+
127+
if (cxit_rem_cntr) {
128+
ret = fi_mr_bind(mem_window.mr, &cxit_rem_cntr->fid, FI_REMOTE_WRITE);
129+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_bind(cntr) failed %d", ret);
130+
}
131+
132+
ret = fi_mr_enable(mem_window.mr);
133+
cr_assert_eq(ret, FI_SUCCESS, "fi_mr_enable failed %d", ret);
134+
135+
key_val = fi_mr_key(mem_window.mr);
136+
137+
/* Perform writedata */
138+
ret = fi_writedata(cxit_ep, send_buf, len, NULL, immediate_data,
139+
cxit_ep_fi_addr, 0, key_val, NULL);
140+
if (ep->ep_obj->domain->rma_cq_data_size) {
141+
cr_assert_eq(ret, FI_SUCCESS, "fi_writedata failed %d", ret);
142+
} else {
143+
cr_assert_eq(ret, -FI_ENOSYS, "fi_writedata bad return %d", ret);
144+
goto done;
145+
}
146+
147+
/* Wait for local completion (send) */
148+
ret = cxit_await_completion(cxit_tx_cq, &cqe);
149+
cr_assert_eq(ret, 1, "fi_cq_read (tx) failed %d", ret);
150+
validate_tx_event(&cqe, FI_RMA | FI_WRITE, NULL);
151+
152+
/* Wait for remote completion (recv) with source address */
153+
while (poll_count < CXIT_DEFAULT_TIMEOUT * 1000) {
154+
ret = fi_cq_readfrom(cxit_rx_cq, &cqe, 1, &src_addr);
155+
if (ret == 1)
156+
break;
157+
if (ret != -FI_EAGAIN)
158+
break;
159+
poll_count++;
160+
usleep(1000);
161+
}
162+
cr_assert_eq(ret, 1, "fi_cq_readfrom (rx) failed %d", ret);
163+
164+
/* Validate remote completion */
165+
cr_assert(cqe.flags & FI_REMOTE_WRITE, "Missing FI_REMOTE_WRITE flag");
166+
cr_assert_eq(cqe.data, immediate_data, "Data mismatch: 0x%lx != 0x%lx",
167+
cqe.data, immediate_data);
168+
169+
/* Validate source address */
170+
cr_assert_eq(src_addr, cxit_ep_fi_addr, "Source address mismatch");
171+
172+
/* Verify data */
173+
for (size_t i = 0; i < len; i++) {
174+
cr_assert_eq(mem_window.mem[i], send_buf[i],
175+
"Memory mismatch at index %zu: 0x%02x != 0x%02x",
176+
i, mem_window.mem[i], send_buf[i]);
177+
}
178+
done:
179+
free(send_buf);
180+
mr_destroy(&mem_window);
181+
}

0 commit comments

Comments
 (0)