Skip to content

Commit acbaf4a

Browse files
vickiegptvickiegpt
committed
update
1 parent c388cf5 commit acbaf4a

File tree

4 files changed

+575
-0
lines changed

4 files changed

+575
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,62 @@ else()
5959
target_include_directories(cxlmemsim_server PRIVATE include ${cxxopts_INCLUDE_DIR} ${spdlog_INCLUDE_DIR})
6060
target_link_libraries(cxlmemsim_server cxlmemsim cxxopts::cxxopts spdlog::spdlog rt)
6161
endif()
62+
# Find RDMA libraries
63+
find_package(PkgConfig REQUIRED)
64+
pkg_check_modules(RDMACM REQUIRED librdmacm)
65+
pkg_check_modules(IBV REQUIRED libibverbs)
66+
67+
# Check if RDMA is available
68+
option(ENABLE_RDMA "Enable RDMA support" ON)
69+
70+
if(ENABLE_RDMA)
71+
if(RDMACM_FOUND AND IBV_FOUND)
72+
message(STATUS "RDMA support enabled")
73+
add_definitions(-DHAVE_RDMA)
74+
75+
# Add RDMA include directories
76+
include_directories(${RDMACM_INCLUDE_DIRS})
77+
include_directories(${IBV_INCLUDE_DIRS})
78+
79+
# Add RDMA libraries
80+
set(RDMA_LIBRARIES ${RDMACM_LIBRARIES} ${IBV_LIBRARIES})
81+
82+
# Add RDMA source files
83+
set(RDMA_SOURCES
84+
src/rdma_communication.cpp
85+
qemu_integration/src/cxlmemsim_server_rdma.cpp
86+
)
87+
88+
# Create RDMA-enabled server executable
89+
add_executable(cxlmemsim_server_rdma ${RDMA_SOURCES})
90+
target_link_libraries(cxlmemsim_server_rdma
91+
${RDMA_LIBRARIES}
92+
pthread
93+
)
94+
95+
# Set RDMA-specific compile flags
96+
target_compile_options(cxlmemsim_server_rdma PRIVATE
97+
${RDMACM_CFLAGS_OTHER}
98+
${IBV_CFLAGS_OTHER}
99+
)
100+
101+
else()
102+
message(WARNING "RDMA libraries not found. RDMA support disabled.")
103+
set(ENABLE_RDMA OFF)
104+
endif()
105+
endif()
106+
107+
# Installation targets
108+
if(ENABLE_RDMA)
109+
install(TARGETS cxlmemsim_server_rdma
110+
RUNTIME DESTINATION bin)
111+
112+
install(FILES include/rdma_communication.h
113+
DESTINATION include/cxlmemsim)
114+
endif()
115+
116+
# Export RDMA configuration
117+
configure_file(
118+
"${CMAKE_CURRENT_SOURCE_DIR}/rdma_config.h.in"
119+
"${CMAKE_CURRENT_BINARY_DIR}/rdma_config.h"
120+
)

‎rdma_config.h.in‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* RDMA Configuration Header Template
3+
* This file is processed by CMake to generate rdma_config.h
4+
*/
5+
6+
#ifndef RDMA_CONFIG_H
7+
#define RDMA_CONFIG_H
8+
9+
/* Define if RDMA support is enabled */
10+
#cmakedefine ENABLE_RDMA
11+
12+
/* Define if RDMA libraries are available */
13+
#cmakedefine HAVE_RDMA
14+
15+
/* Define if libibverbs is available */
16+
#cmakedefine HAVE_LIBIBVERBS
17+
18+
/* Define if librdmacm is available */
19+
#cmakedefine HAVE_LIBRDMACM
20+
21+
/* Define if soft-RoCE is available */
22+
#cmakedefine HAVE_SOFT_ROCE
23+
24+
/* RDMA library versions */
25+
#define RDMACM_VERSION "@RDMACM_VERSION@"
26+
#define IBV_VERSION "@IBV_VERSION@"
27+
28+
/* Default RDMA configuration values */
29+
#define DEFAULT_RDMA_PORT 5555
30+
#define DEFAULT_RDMA_BUFFER_SIZE 4096
31+
#define DEFAULT_RDMA_CQ_SIZE 1024
32+
#define DEFAULT_RDMA_MAX_WR 512
33+
#define DEFAULT_RDMA_CACHELINE_SIZE 64
34+
35+
/* Transport mode options */
36+
#define TRANSPORT_MODE_TCP 0
37+
#define TRANSPORT_MODE_SHM 1
38+
#define TRANSPORT_MODE_RDMA 2
39+
40+
/* Default transport mode */
41+
#ifdef HAVE_RDMA
42+
#define DEFAULT_TRANSPORT_MODE TRANSPORT_MODE_RDMA
43+
#else
44+
#define DEFAULT_TRANSPORT_MODE TRANSPORT_MODE_TCP
45+
#endif
46+
47+
/* RDMA server defaults */
48+
#define DEFAULT_RDMA_SERVER_ADDR "0.0.0.0"
49+
#define DEFAULT_RDMA_CLIENT_ADDR "127.0.0.1"
50+
51+
/* Performance tuning parameters */
52+
#define RDMA_INLINE_DATA_SIZE 256
53+
#define RDMA_MAX_SGE 1
54+
#define RDMA_POLL_TIMEOUT_MS 1000
55+
#define RDMA_CONNECT_TIMEOUT_MS 2000
56+
#define RDMA_RESOLVE_TIMEOUT_MS 2000
57+
58+
/* Statistics collection */
59+
#cmakedefine ENABLE_RDMA_STATS
60+
61+
/* Debug options */
62+
#cmakedefine RDMA_DEBUG
63+
#cmakedefine RDMA_VERBOSE_DEBUG
64+
65+
/* Compatibility options */
66+
#cmakedefine RDMA_COMPAT_MODE
67+
68+
/* Feature flags */
69+
#cmakedefine RDMA_SUPPORT_MULTICAST
70+
#cmakedefine RDMA_SUPPORT_XRC
71+
#cmakedefine RDMA_SUPPORT_ATOMICS
72+
73+
/* Build information */
74+
#define RDMA_BUILD_DATE "@BUILD_DATE@"
75+
#define RDMA_BUILD_HOST "@BUILD_HOST@"
76+
#define RDMA_COMPILER "@CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@"
77+
78+
/* Helper macros */
79+
#ifdef HAVE_RDMA
80+
#define RDMA_ENABLED 1
81+
#define RDMA_AVAILABLE() (1)
82+
#else
83+
#define RDMA_ENABLED 0
84+
#define RDMA_AVAILABLE() (0)
85+
#endif
86+
87+
#ifdef RDMA_DEBUG
88+
#include <stdio.h>
89+
#define RDMA_LOG(fmt, ...) \
90+
fprintf(stderr, "[RDMA] " fmt "\n", ##__VA_ARGS__)
91+
#else
92+
#define RDMA_LOG(fmt, ...) do {} while(0)
93+
#endif
94+
95+
#ifdef RDMA_VERBOSE_DEBUG
96+
#define RDMA_TRACE(fmt, ...) \
97+
fprintf(stderr, "[RDMA TRACE] %s:%d: " fmt "\n", \
98+
__FILE__, __LINE__, ##__VA_ARGS__)
99+
#else
100+
#define RDMA_TRACE(fmt, ...) do {} while(0)
101+
#endif
102+
103+
/* Error handling */
104+
#define RDMA_ERROR(fmt, ...) \
105+
fprintf(stderr, "[RDMA ERROR] " fmt "\n", ##__VA_ARGS__)
106+
107+
#define RDMA_WARN(fmt, ...) \
108+
fprintf(stderr, "[RDMA WARNING] " fmt "\n", ##__VA_ARGS__)
109+
110+
/* Version checking */
111+
#define RDMA_VERSION_CHECK(major, minor) \
112+
(RDMA_VERSION_MAJOR > (major) || \
113+
(RDMA_VERSION_MAJOR == (major) && RDMA_VERSION_MINOR >= (minor)))
114+
115+
#endif /* RDMA_CONFIG_H */

‎scripts/setup_rdma.sh‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
#!/bin/bash
2+
3+
# Setup script for RDMA support in CXLMemSim
4+
# This script installs necessary dependencies and configures RDMA
5+
6+
set -e
7+
8+
echo "============================================"
9+
echo "CXLMemSim RDMA Setup Script"
10+
echo "============================================"
11+
12+
# Check if running as root
13+
if [ "$EUID" -ne 0 ]; then
14+
echo "Please run as root (use sudo)"
15+
exit 1
16+
fi
17+
18+
# Detect OS
19+
if [ -f /etc/os-release ]; then
20+
. /etc/os-release
21+
OS=$ID
22+
VER=$VERSION_ID
23+
else
24+
echo "Cannot detect OS"
25+
exit 1
26+
fi
27+
28+
echo "Detected OS: $OS $VER"
29+
30+
# Install RDMA packages based on OS
31+
case $OS in
32+
ubuntu|debian)
33+
echo "Installing RDMA packages for Ubuntu/Debian..."
34+
apt-get update
35+
apt-get install -y \
36+
rdma-core \
37+
libibverbs-dev \
38+
librdmacm-dev \
39+
ibverbs-utils \
40+
perftest \
41+
infiniband-diags \
42+
opensm
43+
;;
44+
45+
fedora|rhel|centos)
46+
echo "Installing RDMA packages for Fedora/RHEL/CentOS..."
47+
yum install -y \
48+
rdma-core \
49+
libibverbs-devel \
50+
librdmacm-devel \
51+
perftest \
52+
infiniband-diags \
53+
opensm
54+
;;
55+
56+
arch)
57+
echo "Installing RDMA packages for Arch Linux..."
58+
pacman -S --noconfirm \
59+
rdma-core \
60+
perftest
61+
;;
62+
63+
*)
64+
echo "Unsupported OS: $OS"
65+
echo "Please install RDMA packages manually:"
66+
echo " - rdma-core"
67+
echo " - libibverbs-dev/devel"
68+
echo " - librdmacm-dev/devel"
69+
exit 1
70+
;;
71+
esac
72+
73+
# Load RDMA kernel modules
74+
echo "Loading RDMA kernel modules..."
75+
modprobe ib_core
76+
modprobe ib_umad
77+
modprobe ib_uverbs
78+
modprobe rdma_cm
79+
modprobe rdma_ucm
80+
81+
# For RoCE (RDMA over Converged Ethernet)
82+
modprobe mlx4_ib || true
83+
modprobe mlx5_ib || true
84+
85+
# For iWARP
86+
modprobe iw_cxgb4 || true
87+
88+
# For soft-RoCE (RXE) - software RDMA for testing
89+
modprobe rdma_rxe || true
90+
91+
# Check RDMA devices
92+
echo "Checking RDMA devices..."
93+
if command -v ibv_devices &> /dev/null; then
94+
ibv_devices
95+
else
96+
echo "ibv_devices command not found"
97+
fi
98+
99+
# Setup soft-RoCE if no hardware RDMA devices found
100+
if ! ibv_devices 2>/dev/null | grep -q "device"; then
101+
echo "No hardware RDMA devices found. Setting up soft-RoCE..."
102+
103+
# Get primary network interface
104+
PRIMARY_IFACE=$(ip route | grep default | awk '{print $5}' | head -n1)
105+
106+
if [ -z "$PRIMARY_IFACE" ]; then
107+
echo "Could not determine primary network interface"
108+
echo "Please set up soft-RoCE manually with: rdma link add rxe0 type rxe netdev <interface>"
109+
else
110+
echo "Using network interface: $PRIMARY_IFACE"
111+
112+
# Create soft-RoCE device
113+
rdma link add rxe0 type rxe netdev $PRIMARY_IFACE 2>/dev/null || true
114+
115+
# Verify soft-RoCE device
116+
if rdma link show | grep -q rxe0; then
117+
echo "Soft-RoCE device rxe0 created successfully"
118+
else
119+
echo "Failed to create soft-RoCE device"
120+
fi
121+
fi
122+
fi
123+
124+
# Set up hugepages for better RDMA performance
125+
echo "Setting up hugepages..."
126+
echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
127+
128+
# Configure RDMA limits
129+
echo "Configuring system limits for RDMA..."
130+
cat >> /etc/security/limits.conf <<EOF
131+
* soft memlock unlimited
132+
* hard memlock unlimited
133+
EOF
134+
135+
# Create systemd service for RDMA setup (optional)
136+
cat > /etc/systemd/system/cxlmemsim-rdma.service <<EOF
137+
[Unit]
138+
Description=CXLMemSim RDMA Setup
139+
After=network.target
140+
141+
[Service]
142+
Type=oneshot
143+
ExecStart=/bin/bash -c 'modprobe rdma_rxe; rdma link add rxe0 type rxe netdev $(ip route | grep default | awk "{print $5}" | head -n1) || true'
144+
RemainAfterExit=yes
145+
146+
[Install]
147+
WantedBy=multi-user.target
148+
EOF
149+
150+
systemctl daemon-reload
151+
systemctl enable cxlmemsim-rdma.service
152+
153+
# Create configuration file
154+
echo "Creating RDMA configuration file..."
155+
mkdir -p /etc/cxlmemsim
156+
cat > /etc/cxlmemsim/rdma.conf <<EOF
157+
# CXLMemSim RDMA Configuration
158+
159+
# Transport mode: tcp, shm, or rdma
160+
CXL_TRANSPORT_MODE=rdma
161+
162+
# RDMA server address (for client)
163+
CXL_MEMSIM_RDMA_SERVER=127.0.0.1
164+
165+
# RDMA server port
166+
CXL_MEMSIM_RDMA_PORT=5555
167+
168+
# Enable RDMA statistics
169+
CXL_RDMA_STATS=1
170+
171+
# RDMA buffer size (in KB)
172+
CXL_RDMA_BUFFER_SIZE=4096
173+
174+
# RDMA queue depth
175+
CXL_RDMA_QUEUE_DEPTH=512
176+
EOF
177+
178+
echo "============================================"
179+
echo "RDMA setup complete!"
180+
echo "============================================"
181+
echo ""
182+
echo "Configuration file: /etc/cxlmemsim/rdma.conf"
183+
echo ""
184+
echo "To test RDMA setup:"
185+
echo " 1. Check devices: ibv_devices"
186+
echo " 2. Check device info: ibv_devinfo"
187+
echo " 3. Test bandwidth: ib_write_bw (server) and ib_write_bw <server_ip> (client)"
188+
echo ""
189+
echo "To use RDMA with CXLMemSim:"
190+
echo " 1. Set environment variable: export CXL_TRANSPORT_MODE=rdma"
191+
echo " 2. Start server: ./cxlmemsim_server_rdma <tcp_port> [rdma_port]"
192+
echo " 3. Configure QEMU to use RDMA"
193+
echo ""
194+
echo "Note: You may need to reboot or re-login for limits to take effect"

0 commit comments

Comments
 (0)