Skip to content

Commit 772002e

Browse files
vickiegptvickiegpt
committed
update
1 parent 8a56c60 commit 772002e

File tree

4 files changed

+90
-47
lines changed

4 files changed

+90
-47
lines changed

‎qemu_integration/include/qemu_cxl_memsim.h‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,66 @@ extern "C" {
1313
#define CXL_READ_OP 0
1414
#define CXL_WRITE_OP 1
1515

16+
/* ============================================================================
17+
* Shared Memory Coherency Protocol
18+
* ============================================================================
19+
* Lock-free MESI coherency via shared memory for low-latency communication.
20+
* Layout: /dev/shm/cxlmemsim_coherency
21+
*/
22+
23+
#define CXL_SHM_COHERENCY_PATH "/dev/shm/cxlmemsim_coherency"
24+
#define CXL_SHM_COHERENCY_MAGIC 0x43584C4D /* "CXLM" */
25+
#define CXL_SHM_COHERENCY_VERSION 1
26+
#define CXL_SHM_MAX_HOSTS 16
27+
#define CXL_SHM_MAX_CACHELINES (16 * 1024 * 1024) /* 16M cachelines = 1GB memory */
28+
29+
/* MESI states */
30+
#define CXL_MESI_INVALID 0
31+
#define CXL_MESI_SHARED 1
32+
#define CXL_MESI_EXCLUSIVE 2
33+
#define CXL_MESI_MODIFIED 3
34+
35+
/* Per-cacheline coherency state (8 bytes, atomically accessible) */
36+
typedef struct __attribute__((packed, aligned(8))) {
37+
uint8_t state; /* MESI state */
38+
uint8_t owner_id; /* Current owner host ID (0-15) */
39+
uint16_t sharers_bitmap; /* Bitmap of hosts sharing this line */
40+
uint32_t version; /* Version counter for ABA prevention */
41+
} CXLCachelineState;
42+
43+
/* Shared memory coherency header */
44+
typedef struct __attribute__((aligned(64))) {
45+
uint32_t magic; /* CXL_SHM_COHERENCY_MAGIC */
46+
uint32_t version; /* Protocol version */
47+
uint64_t num_cachelines; /* Number of cacheline entries */
48+
uint64_t memory_size; /* Total memory size being tracked */
49+
uint8_t num_hosts; /* Number of registered hosts */
50+
uint8_t reserved[7];
51+
52+
/* Per-host statistics (cache-line aligned) */
53+
struct __attribute__((aligned(64))) {
54+
uint64_t reads;
55+
uint64_t writes;
56+
uint64_t invalidations_sent;
57+
uint64_t invalidations_received;
58+
uint64_t state_transitions;
59+
uint64_t reserved[3];
60+
} host_stats[CXL_SHM_MAX_HOSTS];
61+
62+
/* Cacheline state array follows header (variable size) */
63+
/* CXLCachelineState cachelines[num_cachelines]; */
64+
} CXLCoherencyHeader;
65+
66+
/* Get pointer to cacheline state array */
67+
static inline CXLCachelineState* cxl_shm_get_cachelines(CXLCoherencyHeader *hdr) {
68+
return (CXLCachelineState*)((char*)hdr + sizeof(CXLCoherencyHeader));
69+
}
70+
71+
/* Get cacheline index from address */
72+
static inline uint64_t cxl_shm_addr_to_index(uint64_t addr) {
73+
return (addr / CACHELINE_SIZE) % CXL_SHM_MAX_CACHELINES;
74+
}
75+
1676
typedef struct {
1777
char host[256];
1878
int port;

‎qemu_integration/launch_qemu_cxl.sh‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,9 @@ VM_MEMORY=${VM_MEMORY:-2G}
77
CXL_MEMORY=${CXL_MEMORY:-4G}
88
DISK_IMAGE=${DISK_IMAGE:-plucky-server-cloudimg-amd64.img}
99

10-
# Enable RDMA mode
10+
# Enable SHM mode with lock-free coherency
1111
export CXL_TRANSPORT_MODE=shm
12-
# Also set TCP fallback
13-
export CXL_MEMSIM_HOST=127.0.0.1
14-
export CXL_MEMSIM_PORT=9999
12+
export CXL_HOST_ID=0
1513
exec $QEMU_BINARY \
1614
--enable-kvm -cpu qemu64,+xsave,+rdtscp,+avx,+avx2,+sse4.1,+sse4.2,+avx512f,+avx512dq,+avx512ifma,+avx512cd,+avx512bw,+avx512vl,+avx512vbmi,+clflushopt \
1715
-m 16G,maxmem=32G,slots=8 \
@@ -28,9 +26,9 @@ exec $QEMU_BINARY \
2826
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=0 \
2927
-device cxl-rp,port=1,bus=cxl.1,id=root_port14,chassis=0,slot=1 \
3028
-device cxl-type3,bus=root_port13,persistent-memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0,sn=0x1 \
31-
-device cxl-type1,bus=root_port14,size=2G,cache-size=64M \
29+
-device cxl-type1,bus=root_port14,size=1G,cache-size=64M \
3230
-device virtio-cxl-accel-pci,bus=pcie.0 \
33-
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/dev/shm/cxlmemsim_shared,size=2G \
34-
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/dev/shm/lsa0.raw,size=2G \
31+
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/dev/shm/cxlmemsim_shared,size=1G \
32+
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/dev/shm/lsa0.raw,size=1G \
3533
-M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G \
3634
-nographic

‎qemu_integration/launch_qemu_cxl1.sh‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,10 @@ VM_MEMORY=${VM_MEMORY:-2G}
77
CXL_MEMORY=${CXL_MEMORY:-4G}
88
DISK_IMAGE=${DISK_IMAGE:-plucky-server-cloudimg-amd64.img}
99

10-
# Enable RDMA mode
10+
# Enable SHM mode with lock-free coherency
1111
export CXL_TRANSPORT_MODE=shm
12-
# Also set TCP fallback
13-
export CXL_MEMSIM_HOST=127.0.0.1
14-
export CXL_MEMSIM_PORT=9999
15-
exec $QEMU_BINARY \
12+
export CXL_HOST_ID=1
13+
$QEMU_BINARY \
1614
--enable-kvm -cpu qemu64,+xsave,+rdtscp,+avx,+avx2,+sse4.1,+sse4.2,+avx512f,+avx512dq,+avx512ifma,+avx512cd,+avx512bw,+avx512vl,+avx512vbmi,+clflushopt \
1715
-m 16G,maxmem=32G,slots=8 \
1816
-smp 4 \
@@ -28,9 +26,9 @@ exec $QEMU_BINARY \
2826
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=0 \
2927
-device cxl-rp,port=1,bus=cxl.1,id=root_port14,chassis=0,slot=1 \
3028
-device cxl-type3,bus=root_port13,persistent-memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0,sn=0x1 \
31-
-device cxl-type1,bus=root_port14,size=2G,cache-size=64M \
29+
-device cxl-type1,bus=root_port14,size=1G,cache-size=64M \
3230
-device virtio-cxl-accel-pci,bus=pcie.0 \
33-
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/dev/shm/cxlmemsim_shared,size=2G \
34-
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/dev/shm/lsa1.raw,size=2G \
31+
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/dev/shm/cxlmemsim_shared,size=1G \
32+
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/dev/shm/lsa1.raw,size=1G \
3533
-M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G \
3634
-nographic

‎qemu_integration/launch_qemu_type2_gpu.sh‎

Lines changed: 19 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ CXL_TYPE2_MEM_SIZE=${CXL_TYPE2_MEM_SIZE:-4G}
2323
GPU_DEVICE=${GPU_DEVICE:-""}
2424

2525
# Communication mode: tcp or shm (shared memory)
26-
export CXL_TRANSPORT_MODE=${CXL_TRANSPORT_MODE:-shm}
26+
export CXL_TRANSPORT_MODE=shm
2727
export CXL_MEMSIM_HOST=$CXL_MEMSIM_HOST
2828
export CXL_MEMSIM_PORT=$CXL_MEMSIM_PORT
2929

@@ -37,12 +37,18 @@ echo "Type 2 Device Memory: $CXL_TYPE2_MEM_SIZE"
3737
echo "GPU Device: ${GPU_DEVICE:-'Not configured (simulation mode)'}"
3838
echo "==================================================================="
3939

40-
# Build QEMU command
41-
QEMU_CMD="$QEMU_BINARY \
40+
# Add GPU device parameter if specified
41+
if [ -n "$GPU_DEVICE" ]; then
42+
QEMU_CMD="$QEMU_CMD,gpu-device=$GPU_DEVICE"
43+
echo "GPU passthrough enabled for: $GPU_DEVICE"
44+
fi
45+
46+
# Add traditional Type 3 memory device for comparison
47+
exec $QEMU_BINARY \
4248
--enable-kvm \
4349
-cpu qemu64,+xsave,+rdtscp,+avx,+avx2,+sse4.1,+sse4.2,+avx512f,+avx512dq,+avx512ifma,+avx512cd,+avx512bw,+avx512vl,+avx512vbmi,+clflushopt \
4450
-kernel ./bzImage \
45-
-append \"root=/dev/sda rw console=ttyS0,115200 ignore_loglevel nokaslr nosmp nopti nospectre_v2 mem=2G\" \
51+
-append "root=/dev/sda rw console=ttyS0,115200 ignore_loglevel nokaslr nosmp nopti nospectre_v2" \
4652
-netdev tap,id=network0,ifname=tap1,script=no,downscript=no \
4753
-device e1000,netdev=network0,mac=52:54:00:00:00:02 \
4854
-netdev user,id=netssh0,hostfwd=tcp::10022-:22 \
@@ -54,36 +60,17 @@ QEMU_CMD="$QEMU_BINARY \
5460
-smp 4 \
5561
-device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
5662
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=0 \
57-
-device cxl-rp,port=1,bus=cxl.1,id=root_port14,chassis=0,slot=1"
58-
59-
# Add CXL Type 2 device with coherent memory and GPU passthrough support
60-
QEMU_CMD="$QEMU_CMD \
63+
-device cxl-rp,port=1,bus=cxl.1,id=root_port14,chassis=0,slot=1 \
6164
-device cxl-type2,bus=root_port13,\
62-
cache-size=$CXL_TYPE2_CACHE_SIZE,\
63-
mem-size=$CXL_TYPE2_MEM_SIZE,\
64-
sn=0x2,\
65-
cxlmemsim-addr=$CXL_MEMSIM_HOST,\
66-
cxlmemsim-port=$CXL_MEMSIM_PORT,\
67-
coherency-enabled=true,\
68-
id=cxl-type2-gpu0"
69-
70-
# Add GPU device parameter if specified
71-
if [ -n "$GPU_DEVICE" ]; then
72-
QEMU_CMD="$QEMU_CMD,gpu-device=$GPU_DEVICE"
73-
echo "GPU passthrough enabled for: $GPU_DEVICE"
74-
fi
75-
76-
# Add traditional Type 3 memory device for comparison
77-
QEMU_CMD="$QEMU_CMD \
65+
cache-size=$CXL_TYPE2_CACHE_SIZE,\
66+
mem-size=$CXL_TYPE2_MEM_SIZE,\
67+
sn=0x2,\
68+
cxlmemsim-addr=$CXL_MEMSIM_HOST,\
69+
cxlmemsim-port=$CXL_MEMSIM_PORT,\
70+
coherency-enabled=true,\
71+
id=cxl-type2-gpu0 \
7872
-device cxl-type3,bus=root_port14,persistent-memdev=cxl-mem1,lsa=cxl-lsa1,id=cxl-pmem0,sn=0x3 \
7973
-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=128G \
8074
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=256M \
8175
-M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G \
82-
-nographic"
83-
84-
echo ""
85-
echo "Starting QEMU with CXL Type 2 device..."
86-
echo ""
87-
88-
# Execute QEMU
89-
exec $QEMU_CMD
76+
-nographic

0 commit comments

Comments
 (0)