Skip to content

Commit 905962f

Browse files
terryzbaiterryzbai
committed
WIP: refactor ixgbe driver
Signed-off-by: Terry Bai <tianyi.bai@unsw.edu.au>
1 parent df8d4a6 commit 905962f

File tree

13 files changed

+298
-250
lines changed

13 files changed

+298
-250
lines changed

‎drivers/blk/virtio/pci/config.json‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@
1818
"size": 2097152
1919
}
2020
],
21+
"ioports": [
22+
{
23+
"size": 128
24+
}
25+
],
2126
"irqs": [
2227
{
2328
"irq_type": "msix",
@@ -26,8 +31,13 @@
2631
],
2732
"pci_bars": [
2833
{
29-
"region_idx": 0,
34+
"res_idx": 0,
3035
"bar_id": 4
36+
},
37+
{
38+
"res_idx": 0,
39+
"ioport": true,
40+
"bar_id": 0
3141
}
3242
]
3343
}

‎drivers/network/ixgbe/config.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"compatible": [
3+
"ixgbe"
4+
],
5+
"resources": {
6+
"regions": [
7+
{
8+
"name": "device_regs",
9+
"perms": "rw",
10+
"setvar_vaddr": "dev_regs_base",
11+
"size": 1048576
12+
},
13+
{
14+
"name": "hw_rx_ring_buffer",
15+
"size": 16384
16+
},
17+
{
18+
"name": "hw_tx_ring_buffer",
19+
"size": 16384
20+
}
21+
],
22+
"irqs": [
23+
{
24+
"irq_type": "ioapic",
25+
"ioapic": 3,
26+
"pin": 4,
27+
"vector": 2
28+
}
29+
],
30+
"pci_bars": [
31+
{
32+
"res_idx": 0,
33+
"bar_id": 0
34+
}
35+
]
36+
}
37+
}

‎drivers/network/ixgbe/eth_driver.mk‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212

1313
ETHERNET_DRIVER_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
1414
CHECK_NETDRV_FLAGS_MD5:=.netdrv_cflags-$(shell echo -- ${CFLAGS} ${CFLAGS_network} | shasum | sed 's/ *-//')
15+
# This ethernet driver needs a configured timer driver
16+
NET_NEED_TIMER := 1
1517

1618
${CHECK_NETDRV_FLAGS_MD5}:
1719
-rm -f .netdrv_cflags-*

‎drivers/network/ixgbe/ethernet.c‎

Lines changed: 32 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ __attribute__((__section__(".timer_client_config"))) timer_client_config_t timer
2222

2323
__attribute__((__section__(".net_driver_config"))) net_driver_config_t config;
2424

25+
uintptr_t dev_regs_base;
26+
2527
#define MASK(n) (~BIT(n))
2628

2729
#define IRQ_CH 0
@@ -36,10 +38,10 @@ __attribute__((__section__(".net_driver_config"))) net_driver_config_t config;
3638
// #define IRQ_INTERVAL 0
3739
#define IRQ_INTERVAL 40
3840

39-
const uint64_t hw_rx_ring_paddr = 0x10000000;
40-
const uint64_t hw_rx_ring_vaddr = 0x2400000;
41-
const uint64_t hw_tx_ring_paddr = 0x10004000;
42-
const uint64_t hw_tx_ring_vaddr = 0x2404000;
41+
uintptr_t hw_rx_ring_paddr;
42+
uintptr_t hw_rx_ring_vaddr;
43+
uintptr_t hw_tx_ring_paddr;
44+
uintptr_t hw_tx_ring_vaddr;
4345

4446
static bool achieved_something;
4547

@@ -80,12 +82,12 @@ net_queue_handle_t tx_queue;
8082
volatile struct enet_regs *eth;
8183

8284
static inline void set_reg(uintptr_t reg, uint32_t val) {
83-
asm volatile("movl %0,%1" : : "r" (val), "m" (*(volatile uint32_t *)reg));
85+
asm volatile("movl %0,%1" : : "r" (val), "m" (*(volatile uint32_t *)(reg + dev_regs_base)));
8486
}
8587

8688
static inline uint32_t get_reg(uintptr_t reg) {
8789
uint32_t ret;
88-
asm volatile("movl %1,%0" : "=r" (ret) : "m" (*(volatile uint32_t *)reg) : "memory");
90+
asm volatile("movl %1,%0" : "=r" (ret) : "m" (*(volatile uint32_t *)(reg + dev_regs_base)) : "memory");
8991
return ret;
9092
}
9193

@@ -98,12 +100,12 @@ static inline void clear_flags(uintptr_t reg, uint32_t flags) {
98100
}
99101

100102
static inline void set_reg16(uintptr_t reg, uint16_t val) {
101-
asm volatile("movw %0,%1" : : "r" (val), "m" (*(volatile uint16_t *)reg));
103+
asm volatile("movw %0,%1" : : "r" (val), "m" (*(volatile uint16_t *)(reg + dev_regs_base)));
102104
}
103105

104106
static inline uint16_t get_reg16(uintptr_t reg) {
105107
uint16_t ret;
106-
asm volatile("movw %1,%0" : "=r" (ret) : "m" (*(volatile uint16_t *)reg) : "memory");
108+
asm volatile("movw %1,%0" : "=r" (ret) : "m" (*(volatile uint16_t *)(reg + dev_regs_base)) : "memory");
107109
return ret;
108110
}
109111

@@ -368,20 +370,25 @@ void init(void)
368370

369371
// Enable MSI-X, see PCI Express Technology 3.0 Chapter 17 for more details.
370372
// Disable legacy interrupts. TODO: this should be done by PCI driver.
371-
set_flags16(PCI_COMMAND_16, BIT(10));
373+
/* set_flags16(PCI_COMMAND_16, BIT(10)); */
372374
// Set vector message address to Local APIC of CPU0
373-
set_reg(DEVICE_MSIX_TABLE + 0x0, 0xFEEu << 20);
374-
set_reg(DEVICE_MSIX_TABLE + 0x4, 0);
375-
// Set vector data to Interrupt Vector
376-
set_reg(DEVICE_MSIX_TABLE + 0x8, 0x32);
377-
// Unmask vector 0 to enable interrupts through it
378-
set_reg(DEVICE_MSIX_TABLE + 0xC, 0xFFFFFFFE);
379-
// Enable MSI-X. TODO: this should be set by PCI driver.
380-
set_flags(PCI_MSIX_CTRL, BIT(31));
375+
/* set_reg(DEVICE_MSIX_TABLE + 0x0, 0xFEEu << 20); */
376+
/* set_reg(DEVICE_MSIX_TABLE + 0x4, 0); */
377+
/* // Set vector data to Interrupt Vector */
378+
/* set_reg(DEVICE_MSIX_TABLE + 0x8, 0x32); */
379+
/* // Unmask vector 0 to enable interrupts through it */
380+
/* set_reg(DEVICE_MSIX_TABLE + 0xC, 0xFFFFFFFE); */
381+
/* // Enable MSI-X. TODO: this should be set by PCI driver. */
382+
/* set_flags(PCI_MSIX_CTRL, BIT(31)); */
381383

382384
// Initialise the statistic registers. Must keep. TODO: why?
383385
set_reg(RQSMR(0), 0);
384386

387+
hw_rx_ring_vaddr = (uintptr_t)device_resources.regions[1].region.vaddr;
388+
hw_rx_ring_paddr = device_resources.regions[1].io_addr;
389+
hw_tx_ring_vaddr = (uintptr_t)device_resources.regions[2].region.vaddr;
390+
hw_tx_ring_paddr = device_resources.regions[2].io_addr;
391+
385392
device.rx_ring = (void *)hw_rx_ring_vaddr;
386393
device.tx_ring = (void *)hw_tx_ring_vaddr;
387394

@@ -395,6 +402,13 @@ void init(void)
395402
// Disable Interrupts, see Section 4.6.3.1
396403
disable_interrupts();
397404

405+
/* sddf_dprintf("\n\n\n\n\n"); */
406+
/* for (int j = 0; j < 256; j++) { */
407+
/* if (j && j % 16 == 0) sddf_dprintf("\n"); */
408+
/* sddf_dprintf("%02x ", *(uint8_t *)(dev_regs_base + j)); */
409+
/* } */
410+
/* sddf_dprintf("\n"); */
411+
398412
// Master disable prior to link reset, see Section 4.2.1.7
399413
set_reg(CTRL, IXGBE_CTRL_PCIE_MASTER_DISABLE);
400414
while (get_reg(STATUS) & IXGBE_STATUS_PCIE_MASTER_STATUS);
@@ -403,6 +417,7 @@ void init(void)
403417
set_reg(CTRL, get_reg(CTRL) | IXGBE_CTRL_RST);
404418
while ((get_reg(CTRL) & IXGBE_CTRL_RST_MASK) != 0);
405419

420+
sddf_dprintf("wait for 100ns\n");
406421
// Wait at least 10ms
407422
sddf_timer_set_timeout(timer_config.driver_id, 100 * NS_IN_MS);
408423
}
@@ -601,20 +616,6 @@ void notified(microkit_channel ch)
601616
init_2();
602617
} else if (device.init_stage == 2) {
603618
init_3();
604-
sddf_dprintf("\n\n\n");
605-
sddf_dprintf("BAR0: 0x%x\n", get_reg(PCIE_CONFIG_BASE + 0x10));
606-
sddf_dprintf("BAR1: 0x%x\n", get_reg(PCIE_CONFIG_BASE + 0x14));
607-
sddf_dprintf("BAR2: 0x%x\n", get_reg(PCIE_CONFIG_BASE + 0x18));
608-
sddf_dprintf("BAR3: 0x%x\n", get_reg(PCIE_CONFIG_BASE + 0x1c));
609-
sddf_dprintf("BAR4: 0x%x\n", get_reg(PCIE_CONFIG_BASE + 0x20));
610-
sddf_dprintf("MSI CTRL: 0x%x\n", get_reg(PCIE_CONFIG_BASE + 0x50));
611-
sddf_dprintf("MSI-X CTRL: 0x%x\n", get_reg(PCI_MSIX_CTRL));
612-
sddf_dprintf("MSI-X OFFSET: 0x%x\n", get_reg(PCI_MSIX_OFFSET));
613-
sddf_dprintf("MSI-X PENDING: 0x%x\n", get_reg(PCI_MSIX_PENDING));
614-
sddf_dprintf("MSI-X Table - vector0 address low: 0x%x\n", get_reg(DEVICE_MSIX_TABLE));
615-
sddf_dprintf("MSI-X Table - vector0 address high: 0x%x\n", get_reg(DEVICE_MSIX_TABLE + 0x4));
616-
sddf_dprintf("MSI-X Table - vector0 data: 0x%x\n", get_reg(DEVICE_MSIX_TABLE + 0x8));
617-
sddf_dprintf("MSI-X Table - vector0 control: 0x%x\n", get_reg(DEVICE_MSIX_TABLE + 0xC));
618619
}
619620
} else if (ch == device_resources.irqs[0].id){
620621
/* bench->eth_irq_count++; */

‎drivers/network/ixgbe/ethernet.h‎

Lines changed: 60 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -3,98 +3,79 @@
33
#include <stdint.h>
44
#include <sddf/util/printf.h>
55

6-
#define PCIE_CONFIG_BASE 0x3000000lu
7-
#define DEVICE_BASE 0x2000000lu
8-
#define DEVICE_MSIX_TABLE 0x4000000lu
6+
#define declare_register(name, offset) \
7+
uintptr_t name = (uintptr_t)offset;
98

10-
#define declare_register(base, name, offset) \
11-
uintptr_t name = (uintptr_t)(base + offset);
12-
13-
#define declare_array_register(base, name, offset, count, multiplier) \
9+
#define declare_array_register(name, offset, count, multiplier) \
1410
static inline \
1511
uintptr_t \
1612
name(int index) { \
1713
if (index >= count) { \
18-
sddf_dprintf("array register index out of bounds\n"); \
14+
sddf_dprintf("array register index out of bounds\n"); \
1915
return 0; \
2016
} \
21-
return base + offset + multiplier * index; \
17+
return offset + multiplier * index; \
2218
} \
2319

24-
declare_register(PCIE_CONFIG_BASE, PCI_VENDOR_ID_16, 0x00);
25-
declare_register(PCIE_CONFIG_BASE, PCI_DEVICE_ID_16, 0x02);
26-
declare_register(PCIE_CONFIG_BASE, PCI_COMMAND_16, 0x04);
27-
declare_register(PCIE_CONFIG_BASE, PCI_STATUS_16, 0x06);
28-
declare_register(PCIE_CONFIG_BASE, PCI_INTERRUPT_PIN_LINE_16, 0x3C);
29-
declare_register(PCIE_CONFIG_BASE, PCI_MSI_MESSAGE_CONTROL_16, 0x52);
30-
declare_register(PCIE_CONFIG_BASE, PCI_MSI_MESSAGE_ADDRESS_LOW, 0x54);
31-
declare_register(PCIE_CONFIG_BASE, PCI_MSI_MESSAGE_ADDRESS_HIGH, 0x58);
32-
declare_register(PCIE_CONFIG_BASE, PCI_MSI_MESSAGE_DATA_16, 0x5C);
33-
declare_register(PCIE_CONFIG_BASE, PCI_MSI_MASK, 0x60);
34-
declare_register(PCIE_CONFIG_BASE, PCI_MSI_PENDING, 0x64);
35-
declare_register(PCIE_CONFIG_BASE, PCI_MSIX_CTRL, 0x70);
36-
declare_register(PCIE_CONFIG_BASE, PCI_MSIX_OFFSET, 0x74);
37-
declare_register(PCIE_CONFIG_BASE, PCI_MSIX_PENDING, 0x78);
38-
39-
declare_register(DEVICE_BASE, CTRL, 0x00000);
40-
declare_register(DEVICE_BASE, STATUS, 0x00008);
41-
declare_register(DEVICE_BASE, CTRL_EXT, 0x00018);
42-
declare_register(DEVICE_BASE, EEC, 0x10010);
43-
declare_register(DEVICE_BASE, GPRC, 0x04074);
44-
declare_register(DEVICE_BASE, GPTC, 0x04080);
45-
declare_register(DEVICE_BASE, GORCL, 0x04088);
46-
declare_register(DEVICE_BASE, GORCH, 0x0408C);
47-
declare_register(DEVICE_BASE, GOTCL, 0x04090);
48-
declare_register(DEVICE_BASE, GOTCH, 0x04094);
49-
declare_register(DEVICE_BASE, HLREG0, 0x04240);
50-
declare_register(DEVICE_BASE, LINKS, 0x042A4);
51-
declare_register(DEVICE_BASE, FCTRL, 0x05080);
52-
declare_register(DEVICE_BASE, RXCTRL, 0x03000);
53-
declare_register(DEVICE_BASE, RDRXCTL, 0x02F00);
54-
declare_register(DEVICE_BASE, DTXMXSZRQ, 0x08100);
55-
declare_register(DEVICE_BASE, DMATXCTL, 0x04A80);
56-
declare_register(DEVICE_BASE, RTTDCS, 0x04900);
57-
declare_register(DEVICE_BASE, EICR, 0x00800);
58-
declare_register(DEVICE_BASE, EICS, 0x00808);
59-
declare_register(DEVICE_BASE, EIMS, 0x00880);
60-
declare_register(DEVICE_BASE, EIMC, 0x00888);
61-
declare_register(DEVICE_BASE, EIAC, 0x00810);
62-
declare_register(DEVICE_BASE, GPIE, 0x00898);
63-
declare_register(DEVICE_BASE, TXDGPC, 0x087A0);
64-
declare_register(DEVICE_BASE, TXDGBCL, 0x087A4);
65-
declare_register(DEVICE_BASE, TXDGBCH, 0x087A8);
66-
declare_register(DEVICE_BASE, FACTPS, 0x10150);
67-
declare_array_register(DEVICE_BASE, RDBAL, 0x01000, 64, 0x40);
68-
declare_array_register(DEVICE_BASE, RDBAH, 0x01004, 64, 0x40);
69-
declare_array_register(DEVICE_BASE, RDLEN, 0x01008, 64, 0x60);
70-
declare_array_register(DEVICE_BASE, RDH, 0x01010, 64, 0x40);
71-
declare_array_register(DEVICE_BASE, RDT, 0x01018, 64, 0x40);
72-
declare_array_register(DEVICE_BASE, SRRCTL, 0x01014, 64, 0x40);
73-
declare_array_register(DEVICE_BASE, RXPBSIZE, 0x03C00, 8, 0x4);
74-
declare_array_register(DEVICE_BASE, DCA_RXCTRL, 0x0100C, 64, 0x40);
75-
declare_array_register(DEVICE_BASE, RXDCTL, 0x01028, 64, 0x40);
76-
declare_array_register(DEVICE_BASE, RSCCTL, 0x0102C, 64, 0x40);
77-
declare_array_register(DEVICE_BASE, TDBAL, 0x06000, 64, 0x40);
78-
declare_array_register(DEVICE_BASE, TDBAH, 0x06004, 64, 0x40);
79-
declare_array_register(DEVICE_BASE, TDLEN, 0x06008, 64, 0x40);
80-
declare_array_register(DEVICE_BASE, TDH, 0x06010, 64, 0x40);
81-
declare_array_register(DEVICE_BASE, TDT, 0x06018, 64, 0x40);
82-
declare_array_register(DEVICE_BASE, TXPBSIZE, 0x0CC00, 8, 0x4);
83-
declare_array_register(DEVICE_BASE, TXPBTHRESH, 0x04950, 8, 0x4);
84-
declare_array_register(DEVICE_BASE, TXDCTL, 0x06028, 64, 0x40);
85-
declare_array_register(DEVICE_BASE, IVAR, 0x00900, 64, 0x4);
86-
declare_array_register(DEVICE_BASE, EITR, 0x00820, 24, 0x4);
87-
declare_array_register(DEVICE_BASE, QPTC, 0x08680, 16, 0x4);
88-
declare_array_register(DEVICE_BASE, RAL, 0x0A200, 128, 0x8);
89-
declare_array_register(DEVICE_BASE, RAH, 0x0A204, 128, 0x8);
90-
declare_array_register(DEVICE_BASE, RSCINT, 0x12000, 128, 0x4);
20+
declare_register(CTRL, 0x00000);
21+
declare_register(STATUS, 0x00008);
22+
declare_register(CTRL_EXT, 0x00018);
23+
declare_register(EEC, 0x10010);
24+
declare_register(GPRC, 0x04074);
25+
declare_register(GPTC, 0x04080);
26+
declare_register(GORCL, 0x04088);
27+
declare_register(GORCH, 0x0408C);
28+
declare_register(GOTCL, 0x04090);
29+
declare_register(GOTCH, 0x04094);
30+
declare_register(HLREG0, 0x04240);
31+
declare_register(LINKS, 0x042A4);
32+
declare_register(FCTRL, 0x05080);
33+
declare_register(RXCTRL, 0x03000);
34+
declare_register(RDRXCTL, 0x02F00);
35+
declare_register(DTXMXSZRQ, 0x08100);
36+
declare_register(DMATXCTL, 0x04A80);
37+
declare_register(RTTDCS, 0x04900);
38+
declare_register(EICR, 0x00800);
39+
declare_register(EICS, 0x00808);
40+
declare_register(EIMS, 0x00880);
41+
declare_register(EIMC, 0x00888);
42+
declare_register(EIAC, 0x00810);
43+
declare_register(GPIE, 0x00898);
44+
declare_register(TXDGPC, 0x087A0);
45+
declare_register(TXDGBCL, 0x087A4);
46+
declare_register(TXDGBCH, 0x087A8);
47+
declare_register(FACTPS, 0x10150);
48+
declare_array_register(RDBAL, 0x01000, 64, 0x40);
49+
declare_array_register(RDBAH, 0x01004, 64, 0x40);
50+
declare_array_register(RDLEN, 0x01008, 64, 0x60);
51+
declare_array_register(RDH, 0x01010, 64, 0x40);
52+
declare_array_register(RDT, 0x01018, 64, 0x40);
53+
declare_array_register(SRRCTL, 0x01014, 64, 0x40);
54+
declare_array_register(RXPBSIZE, 0x03C00, 8, 0x4);
55+
declare_array_register(DCA_RXCTRL, 0x0100C, 64, 0x40);
56+
declare_array_register(RXDCTL, 0x01028, 64, 0x40);
57+
declare_array_register(RSCCTL, 0x0102C, 64, 0x40);
58+
declare_array_register(TDBAL, 0x06000, 64, 0x40);
59+
declare_array_register(TDBAH, 0x06004, 64, 0x40);
60+
declare_array_register(TDLEN, 0x06008, 64, 0x40);
61+
declare_array_register(TDH, 0x06010, 64, 0x40);
62+
declare_array_register(TDT, 0x06018, 64, 0x40);
63+
declare_array_register(TXPBSIZE, 0x0CC00, 8, 0x4);
64+
declare_array_register(TXPBTHRESH, 0x04950, 8, 0x4);
65+
declare_array_register(TXDCTL, 0x06028, 64, 0x40);
66+
declare_array_register(IVAR, 0x00900, 64, 0x4);
67+
declare_array_register(EITR, 0x00820, 24, 0x4);
68+
declare_array_register(QPTC, 0x08680, 16, 0x4);
69+
declare_array_register(RAL, 0x0A200, 128, 0x8);
70+
declare_array_register(RAH, 0x0A204, 128, 0x8);
71+
declare_array_register(RSCINT, 0x12000, 128, 0x4);
9172

9273
// Queue Packets Received Count
93-
declare_array_register(DEVICE_BASE, QPRC, 0x01030, 16, 0x40);
74+
declare_array_register(QPRC, 0x01030, 16, 0x40);
9475
// Queue Packets Received Drop Count
95-
declare_array_register(DEVICE_BASE, QPRDC, 0x01430, 16, 0x40);
76+
declare_array_register(QPRDC, 0x01430, 16, 0x40);
9677
// Receive Queue Statistic Mapping Registers
97-
declare_array_register(DEVICE_BASE, RQSMR, 0x02300, 32, 0x4);
78+
declare_array_register(RQSMR, 0x02300, 32, 0x4);
9879

9980
const uint64_t IXGBE_CTRL_LNK_RST = 0x00000008; /* Link Reset. Resets everything. */
10081
const uint64_t IXGBE_CTRL_RST = 0x04000000; /* Reset (SW) */

‎drivers/network/virtio/pci/config.json‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
"size": 65536
1515
}
1616
],
17+
"ioports": [
18+
{
19+
"size": 128
20+
}
21+
],
1722
"irqs": [
1823
{
1924
"irq_type": "ioapic",
@@ -23,8 +28,13 @@
2328
],
2429
"pci_bars": [
2530
{
26-
"region_idx": 0,
31+
"res_idx": 0,
2732
"bar_id": 4
33+
},
34+
{
35+
"res_idx": 0,
36+
"ioport": true,
37+
"bar_id": 0
2838
}
2939
]
3040
}

0 commit comments

Comments
 (0)