1+ #ifndef RDMA_COMMUNICATION_H
2+ #define RDMA_COMMUNICATION_H
3+
4+ #include < rdma/rdma_cma.h>
5+ #include < infiniband/verbs.h>
6+ #include < cstdint>
7+ #include < memory>
8+ #include < functional>
9+ #include < string>
10+ #include < atomic>
11+
12+ #define RDMA_BUFFER_SIZE 4096
13+ #define RDMA_CQ_SIZE 1024
14+ #define RDMA_MAX_WR 512
15+ #define RDMA_CACHELINE_SIZE 64
16+
17+ enum RDMAOpType {
18+ RDMA_OP_READ = 0 ,
19+ RDMA_OP_WRITE = 1 ,
20+ RDMA_OP_READ_RESP = 2 ,
21+ RDMA_OP_WRITE_RESP = 3
22+ };
23+
24+ struct RDMARequest {
25+ uint8_t op_type;
26+ uint64_t addr;
27+ uint64_t size;
28+ uint64_t timestamp;
29+ uint8_t host_id;
30+ uint64_t virtual_addr;
31+ uint8_t data[RDMA_CACHELINE_SIZE];
32+ } __attribute__((packed));
33+
34+ struct RDMAResponse {
35+ uint8_t status;
36+ uint64_t latency_ns;
37+ uint8_t cache_state;
38+ uint8_t data[RDMA_CACHELINE_SIZE];
39+ } __attribute__((packed));
40+
41+ struct RDMAMessage {
42+ RDMARequest request;
43+ RDMAResponse response;
44+ } __attribute__((packed));
45+
46+ class RDMAConnection {
47+ public:
48+ struct ConnectionInfo {
49+ struct ibv_context * context;
50+ struct ibv_pd * pd;
51+ struct ibv_mr * mr;
52+ struct ibv_cq * send_cq;
53+ struct ibv_cq * recv_cq;
54+ struct ibv_qp * qp;
55+ struct ibv_comp_channel * comp_channel;
56+ void * buffer;
57+ size_t buffer_size;
58+ std::atomic<bool > connected;
59+ };
60+
61+ using MessageHandler = std::function<void (const RDMAMessage&, RDMAMessage&)>;
62+
63+ protected:
64+ ConnectionInfo conn_info;
65+ struct rdma_cm_id * cm_id;
66+ struct rdma_event_channel * event_channel;
67+ MessageHandler message_handler;
68+ std::atomic<bool > running;
69+
70+ int setup_connection_resources ();
71+ int setup_qp_parameters (struct ibv_qp_init_attr & qp_attr);
72+ int register_memory_region ();
73+ int post_receive ();
74+ int post_send (const RDMAMessage* msg);
75+ void cleanup_resources ();
76+
77+ public:
78+ RDMAConnection ();
79+ virtual ~RDMAConnection ();
80+
81+ void set_message_handler (MessageHandler handler) { message_handler = handler; }
82+ int send_message (const RDMAMessage& msg);
83+ int receive_message (RDMAMessage& msg);
84+ bool is_connected () const { return conn_info.connected .load (); }
85+ void disconnect ();
86+ };
87+
88+ class RDMAServer : public RDMAConnection {
89+ private:
90+ std::string bind_addr;
91+ uint16_t port;
92+ struct rdma_cm_id * listen_id;
93+
94+ int handle_connection_request (struct rdma_cm_id * client_id);
95+ int handle_established ();
96+ int handle_disconnect ();
97+
98+ public:
99+ RDMAServer (const std::string& addr, uint16_t port);
100+ ~RDMAServer ();
101+
102+ int start ();
103+ int accept_connection ();
104+ void handle_client ();
105+ void stop ();
106+ };
107+
108+ class RDMAClient : public RDMAConnection {
109+ private:
110+ std::string server_addr;
111+ uint16_t server_port;
112+
113+ int resolve_addr ();
114+ int resolve_route ();
115+ int connect_to_server ();
116+
117+ public:
118+ RDMAClient (const std::string& addr, uint16_t port);
119+ ~RDMAClient ();
120+
121+ int connect ();
122+ int send_request (const RDMARequest& req, RDMAResponse& resp);
123+ };
124+
125+ class RDMATransport {
126+ public:
127+ enum Mode {
128+ MODE_TCP,
129+ MODE_SHM,
130+ MODE_RDMA
131+ };
132+
133+ static Mode get_transport_mode () {
134+ const char * mode = std::getenv (" CXL_TRANSPORT_MODE"
135+ if (mode) {
136+ if (std::string (mode) == " rdma" return MODE_RDMA;
137+ if (std::string (mode) == " shm" return MODE_SHM;
138+ }
139+ return MODE_TCP;
140+ }
141+
142+ static bool is_rdma_available () {
143+ struct ibv_device ** dev_list = ibv_get_device_list (nullptr );
144+ if (dev_list) {
145+ ibv_free_device_list (dev_list);
146+ return true ;
147+ }
148+ return false ;
149+ }
150+ };
151+
152+ #endif // RDMA_COMMUNICATION_H
0 commit comments