Skip to content

Commit 6abc472

Browse files
vickiegptvickiegpt
committed
update
1 parent 78cd17d commit 6abc472

File tree

5 files changed

+2011
-11
lines changed

5 files changed

+2011
-11
lines changed

‎workloads/gromacs/Makefile.tests‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# Makefile for MPI CXL Shim Tests
2+
#
3+
# Usage:
4+
# make -f Makefile.tests all # Build all tests
5+
# make -f Makefile.tests clean # Clean test binaries
6+
# make -f Makefile.tests run # Run tests without CXL shim
7+
# make -f Makefile.tests run-cxl # Run tests with CXL shim
8+
9+
MPICC = mpicc
10+
CFLAGS = -Wall -O2
11+
LDFLAGS = -lm
12+
13+
# Test programs
14+
TESTS = test_onesided test_collectives test_p2p
15+
16+
# CXL Shim library
17+
SHIM_LIB = mpi_cxl_shim.so
18+
SHIM_SRC = mpi_cxl_shim.c
19+
20+
# MPI run settings
21+
MPIRUN = mpirun
22+
NP = 2
23+
HOSTS = node0,node1
24+
25+
# CXL settings
26+
CXL_DAX_PATH ?= /dev/dax0.0
27+
CXL_SHM_NAME ?= /cxl_mpi_shim
28+
29+
.PHONY: all clean run run-cxl shim
30+
31+
all: $(TESTS)
32+
33+
test_onesided: test_onesided.c
34+
$(MPICC) $(CFLAGS) -o $@ $< $(LDFLAGS)
35+
36+
test_collectives: test_collectives.c
37+
$(MPICC) $(CFLAGS) -o $@ $< $(LDFLAGS)
38+
39+
test_p2p: test_p2p.c
40+
$(MPICC) $(CFLAGS) -o $@ $< $(LDFLAGS)
41+
42+
# Build the CXL shim library
43+
shim: $(SHIM_LIB)
44+
45+
$(SHIM_LIB): $(SHIM_SRC)
46+
gcc -Wall -Wextra -fPIC -shared -o $@ $< -ldl -lpthread -I/usr/lib/x86_64-linux-gnu/openmpi/include
47+
48+
# Run tests without CXL shim (baseline)
49+
run: $(TESTS)
50+
@echo "========================================="
51+
@echo " Running tests WITHOUT CXL shim"
52+
@echo "========================================="
53+
@echo ""
54+
@echo "--- Point-to-Point Tests ---"
55+
$(MPIRUN) -np $(NP) ./test_p2p
56+
@echo ""
57+
@echo "--- Collective Tests ---"
58+
$(MPIRUN) -np $(NP) ./test_collectives
59+
@echo ""
60+
@echo "--- One-Sided Tests ---"
61+
$(MPIRUN) -np $(NP) ./test_onesided
62+
63+
# Run tests with CXL shim
64+
run-cxl: $(TESTS) $(SHIM_LIB)
65+
@echo "========================================="
66+
@echo " Running tests WITH CXL shim"
67+
@echo " DAX: $(CXL_DAX_PATH)"
68+
@echo "========================================="
69+
@echo ""
70+
@echo "--- Point-to-Point Tests ---"
71+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
72+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
73+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
74+
./test_p2p
75+
@echo ""
76+
@echo "--- Collective Tests ---"
77+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
78+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
79+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
80+
./test_collectives
81+
@echo ""
82+
@echo "--- One-Sided Tests ---"
83+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
84+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
85+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
86+
./test_onesided
87+
88+
# Run with shared memory (for testing without DAX)
89+
run-shm: $(TESTS) $(SHIM_LIB)
90+
@echo "========================================="
91+
@echo " Running tests WITH CXL shim (SHM mode)"
92+
@echo "========================================="
93+
@echo ""
94+
$(MPIRUN) -np $(NP) \
95+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
96+
-x CXL_SHM_NAME=$(CXL_SHM_NAME) \
97+
./test_p2p
98+
@echo ""
99+
$(MPIRUN) -np $(NP) \
100+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
101+
-x CXL_SHM_NAME=$(CXL_SHM_NAME) \
102+
./test_collectives
103+
104+
# Individual test targets
105+
run-p2p: test_p2p $(SHIM_LIB)
106+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
107+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
108+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
109+
./test_p2p
110+
111+
run-coll: test_collectives $(SHIM_LIB)
112+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
113+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
114+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
115+
./test_collectives
116+
117+
run-rma: test_onesided $(SHIM_LIB)
118+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
119+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
120+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
121+
./test_onesided
122+
123+
# Debug mode with verbose output
124+
run-debug: $(TESTS) $(SHIM_LIB)
125+
$(MPIRUN) -np $(NP) --host $(HOSTS) \
126+
-x LD_PRELOAD=$(PWD)/$(SHIM_LIB) \
127+
-x CXL_DAX_PATH=$(CXL_DAX_PATH) \
128+
-x CXL_LOG_LEVEL=3 \
129+
./test_p2p
130+
131+
clean:
132+
rm -f $(TESTS) $(SHIM_LIB)
133+
134+
help:
135+
@echo "MPI CXL Shim Test Suite"
136+
@echo ""
137+
@echo "Targets:"
138+
@echo " all - Build all test programs"
139+
@echo " shim - Build the CXL shim library"
140+
@echo " run - Run tests without CXL shim"
141+
@echo " run-cxl - Run tests with CXL shim (DAX mode)"
142+
@echo " run-shm - Run tests with CXL shim (shared memory mode)"
143+
@echo " run-p2p - Run only point-to-point tests"
144+
@echo " run-coll - Run only collective tests"
145+
@echo " run-rma - Run only one-sided (RMA) tests"
146+
@echo " run-debug - Run with verbose debug output"
147+
@echo " clean - Remove built files"
148+
@echo ""
149+
@echo "Variables:"
150+
@echo " NP=N - Number of processes (default: 2)"
151+
@echo " HOSTS=h1,h2 - Comma-separated host list"
152+
@echo " CXL_DAX_PATH - Path to DAX device (default: /dev/dax0.0)"
153+
@echo " CXL_SHM_NAME - Shared memory name (default: /cxl_mpi_shim)"

‎workloads/gromacs/mpi_cxl_shim.c‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -860,8 +860,8 @@ static int cxl_send(const void *buf, size_t data_size, int dest, int tag, int so
860860

861861
// Decide between inline data and remotable pointer
862862
if (data_size <= CXL_MSG_MAX_INLINE_SIZE) {
863-
// Inline small messages
864-
memcpy(msg->inline_data, buf, data_size);
863+
// Inline small messages (use safe copy for CXL memory)
864+
cxl_safe_memcpy(msg->inline_data, buf, data_size);
865865
msg->is_inline = 1;
866866
msg->data_rptr = CXL_RPTR_NULL;
867867
LOG_TRACE("CXL send: inlined %zu bytes to rank %d slot %lu\n", data_size, dest, slot);
@@ -872,7 +872,7 @@ static int cxl_send(const void *buf, size_t data_size, int dest, int tag, int so
872872
LOG_WARN("CXL send: failed to allocate %zu bytes for large message\n", data_size);
873873
return -1;
874874
}
875-
memcpy(cxl_buf, buf, data_size);
875+
cxl_safe_memcpy(cxl_buf, buf, data_size);
876876
msg->data_rptr = ptr_to_rptr(cxl_buf);
877877
msg->is_inline = 0;
878878
LOG_TRACE("CXL send: %zu bytes via rptr=0x%lx to rank %d slot %lu\n",
@@ -939,14 +939,14 @@ static int cxl_recv(void *buf, size_t max_size, int source, int tag, size_t *act
939939
// Memory barrier to ensure we see all data
940940
__atomic_thread_fence(__ATOMIC_ACQUIRE);
941941

942-
// Copy data
942+
// Copy data (use safe copy for CXL memory)
943943
size_t copy_size = MIN(msg->data_size, max_size);
944944
if (msg->is_inline) {
945-
memcpy(buf, msg->inline_data, copy_size);
945+
cxl_safe_memcpy(buf, msg->inline_data, copy_size);
946946
} else {
947947
void *src_ptr = rptr_to_ptr(msg->data_rptr);
948948
if (src_ptr) {
949-
memcpy(buf, src_ptr, copy_size);
949+
cxl_safe_memcpy(buf, src_ptr, copy_size);
950950
} else {
951951
LOG_ERROR("CXL recv: invalid rptr 0x%lx\n", msg->data_rptr);
952952
atomic_store(&msg->state, CXL_MSG_READY); // Put it back
@@ -1417,7 +1417,7 @@ static cxl_window_t *register_cxl_window(MPI_Win win, void *base, size_t size,
14171417
// Allocate CXL memory for this window
14181418
void *cxl_base = allocate_cxl_memory(size);
14191419
if (cxl_base) {
1420-
memcpy(cxl_base, base, size);
1420+
cxl_safe_memcpy(cxl_base, base, size);
14211421
rank_info->base_rptr = ptr_to_rptr(cxl_base);
14221422
} else {
14231423
rank_info->base_rptr = CXL_RPTR_NULL;
@@ -1602,8 +1602,8 @@ int MPI_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datat
16021602
size_t origin_bytes = (size_t)origin_count * origin_size;
16031603
void *target_addr = (char *)target_base + target_disp * cxl_win->shm->disp_unit;
16041604

1605-
// Direct memory copy via CXL
1606-
memcpy(target_addr, origin_addr, origin_bytes);
1605+
// Direct memory copy via CXL (use safe copy to avoid AVX-512 issues)
1606+
cxl_safe_memcpy(target_addr, origin_addr, origin_bytes);
16071607

16081608
// Memory fence to ensure visibility
16091609
__atomic_thread_fence(__ATOMIC_RELEASE);
@@ -1654,8 +1654,8 @@ int MPI_Get(void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
16541654
// Memory fence before reading
16551655
__atomic_thread_fence(__ATOMIC_ACQUIRE);
16561656

1657-
// Direct memory copy via CXL
1658-
memcpy(origin_addr, target_addr, origin_bytes);
1657+
// Direct memory copy via CXL (use safe copy to avoid AVX-512 issues)
1658+
cxl_safe_memcpy(origin_addr, target_addr, origin_bytes);
16591659

16601660
int cxl_num = atomic_fetch_add(&cxl_get_count, 1);
16611661
LOG_DEBUG("MPI_Get[%d]: CXL direct get #%d (%zu bytes from rank %d @ offset %ld)\n",

0 commit comments

Comments
 (0)