Skip to content

Commit e06231b

Browse files
jwhitedraggi
jwhited
authored and
raggi
committed
conn, device: use UDP GSO and GRO on Linux
StdNetBind probes for UDP GSO and GRO support at runtime. UDP GSO is dependent on checksum offload support on the egress netdev. UDP GSO will be disabled in the event sendmmsg() returns EIO, which is a strong signal that the egress netdev does not support checksum offload. The iperf3 results below demonstrate the effect of this commit between two Linux computers with i5-12400 CPUs. There is roughly ~13us of round trip latency between them. The first result is from commit 052af4a without UDP GSO or GRO. Starting Test: protocol: TCP, 1 streams, 131072 byte blocks [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-10.00 sec 9.85 GBytes 8.46 Gbits/sec 1139 3.01 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 9.85 GBytes 8.46 Gbits/sec 1139 sender [ 5] 0.00-10.04 sec 9.85 GBytes 8.42 Gbits/sec receiver The second result is with UDP GSO and GRO. Starting Test: protocol: TCP, 1 streams, 131072 byte blocks [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-10.00 sec 12.3 GBytes 10.6 Gbits/sec 232 3.15 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 12.3 GBytes 10.6 Gbits/sec 232 sender [ 5] 0.00-10.04 sec 12.3 GBytes 10.6 Gbits/sec receiver Reviewed-by: Adrian Dewhurst <[email protected]> Signed-off-by: Jordan Whited <[email protected]>
1 parent e26adb8 commit e06231b

11 files changed

+674
-142
lines changed

‎conn/bind_std.go

Lines changed: 266 additions & 132 deletions
Large diffs are not rendered by default.

‎conn/bind_std_test.go

Lines changed: 229 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
package conn
22

3-
import "testing"
3+
import (
4+
"encoding/binary"
5+
"net"
6+
"testing"
7+
8+
"golang.org/x/net/ipv6"
9+
)
410

511
func TestStdNetBindReceiveFuncAfterClose(t *testing.T) {
612
bind := NewStdNetBind().(*StdNetBind)
@@ -20,3 +26,225 @@ func TestStdNetBindReceiveFuncAfterClose(t *testing.T) {
2026
fn(bufs, sizes, eps)
2127
}
2228
}
29+
30+
func mockSetGSOSize(control *[]byte, gsoSize uint16) {
31+
*control = (*control)[:cap(*control)]
32+
binary.LittleEndian.PutUint16(*control, gsoSize)
33+
}
34+
35+
func Test_coalesceMessages(t *testing.T) {
36+
cases := []struct {
37+
name string
38+
buffs [][]byte
39+
wantLens []int
40+
wantGSO []int
41+
}{
42+
{
43+
name: "one message no coalesce",
44+
buffs: [][]byte{
45+
make([]byte, 1, 1),
46+
},
47+
wantLens: []int{1},
48+
wantGSO: []int{0},
49+
},
50+
{
51+
name: "two messages equal len coalesce",
52+
buffs: [][]byte{
53+
make([]byte, 1, 2),
54+
make([]byte, 1, 1),
55+
},
56+
wantLens: []int{2},
57+
wantGSO: []int{1},
58+
},
59+
{
60+
name: "two messages unequal len coalesce",
61+
buffs: [][]byte{
62+
make([]byte, 2, 3),
63+
make([]byte, 1, 1),
64+
},
65+
wantLens: []int{3},
66+
wantGSO: []int{2},
67+
},
68+
{
69+
name: "three messages second unequal len coalesce",
70+
buffs: [][]byte{
71+
make([]byte, 2, 3),
72+
make([]byte, 1, 1),
73+
make([]byte, 2, 2),
74+
},
75+
wantLens: []int{3, 2},
76+
wantGSO: []int{2, 0},
77+
},
78+
{
79+
name: "three messages limited cap coalesce",
80+
buffs: [][]byte{
81+
make([]byte, 2, 4),
82+
make([]byte, 2, 2),
83+
make([]byte, 2, 2),
84+
},
85+
wantLens: []int{4, 2},
86+
wantGSO: []int{2, 0},
87+
},
88+
}
89+
90+
for _, tt := range cases {
91+
t.Run(tt.name, func(t *testing.T) {
92+
addr := &net.UDPAddr{
93+
IP: net.ParseIP("127.0.0.1").To4(),
94+
Port: 1,
95+
}
96+
msgs := make([]ipv6.Message, len(tt.buffs))
97+
for i := range msgs {
98+
msgs[i].Buffers = make([][]byte, 1)
99+
msgs[i].OOB = make([]byte, 0, 2)
100+
}
101+
got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, msgs, mockSetGSOSize)
102+
if got != len(tt.wantLens) {
103+
t.Fatalf("got len %d want: %d", got, len(tt.wantLens))
104+
}
105+
for i := 0; i < got; i++ {
106+
if msgs[i].Addr != addr {
107+
t.Errorf("msgs[%d].Addr != passed addr", i)
108+
}
109+
gotLen := len(msgs[i].Buffers[0])
110+
if gotLen != tt.wantLens[i] {
111+
t.Errorf("len(msgs[%d].Buffers[0]) %d != %d", i, gotLen, tt.wantLens[i])
112+
}
113+
gotGSO, err := mockGetGSOSize(msgs[i].OOB)
114+
if err != nil {
115+
t.Fatalf("msgs[%d] getGSOSize err: %v", i, err)
116+
}
117+
if gotGSO != tt.wantGSO[i] {
118+
t.Errorf("msgs[%d] gsoSize %d != %d", i, gotGSO, tt.wantGSO[i])
119+
}
120+
}
121+
})
122+
}
123+
}
124+
125+
func mockGetGSOSize(control []byte) (int, error) {
126+
if len(control) < 2 {
127+
return 0, nil
128+
}
129+
return int(binary.LittleEndian.Uint16(control)), nil
130+
}
131+
132+
func Test_splitCoalescedMessages(t *testing.T) {
133+
newMsg := func(n, gso int) ipv6.Message {
134+
msg := ipv6.Message{
135+
Buffers: [][]byte{make([]byte, 1<<16-1)},
136+
N: n,
137+
OOB: make([]byte, 2),
138+
}
139+
binary.LittleEndian.PutUint16(msg.OOB, uint16(gso))
140+
if gso > 0 {
141+
msg.NN = 2
142+
}
143+
return msg
144+
}
145+
146+
cases := []struct {
147+
name string
148+
msgs []ipv6.Message
149+
firstMsgAt int
150+
wantNumEval int
151+
wantMsgLens []int
152+
wantErr bool
153+
}{
154+
{
155+
name: "second last split last empty",
156+
msgs: []ipv6.Message{
157+
newMsg(0, 0),
158+
newMsg(0, 0),
159+
newMsg(3, 1),
160+
newMsg(0, 0),
161+
},
162+
firstMsgAt: 2,
163+
wantNumEval: 3,
164+
wantMsgLens: []int{1, 1, 1, 0},
165+
wantErr: false,
166+
},
167+
{
168+
name: "second last no split last empty",
169+
msgs: []ipv6.Message{
170+
newMsg(0, 0),
171+
newMsg(0, 0),
172+
newMsg(1, 0),
173+
newMsg(0, 0),
174+
},
175+
firstMsgAt: 2,
176+
wantNumEval: 1,
177+
wantMsgLens: []int{1, 0, 0, 0},
178+
wantErr: false,
179+
},
180+
{
181+
name: "second last no split last no split",
182+
msgs: []ipv6.Message{
183+
newMsg(0, 0),
184+
newMsg(0, 0),
185+
newMsg(1, 0),
186+
newMsg(1, 0),
187+
},
188+
firstMsgAt: 2,
189+
wantNumEval: 2,
190+
wantMsgLens: []int{1, 1, 0, 0},
191+
wantErr: false,
192+
},
193+
{
194+
name: "second last no split last split",
195+
msgs: []ipv6.Message{
196+
newMsg(0, 0),
197+
newMsg(0, 0),
198+
newMsg(1, 0),
199+
newMsg(3, 1),
200+
},
201+
firstMsgAt: 2,
202+
wantNumEval: 4,
203+
wantMsgLens: []int{1, 1, 1, 1},
204+
wantErr: false,
205+
},
206+
{
207+
name: "second last split last split",
208+
msgs: []ipv6.Message{
209+
newMsg(0, 0),
210+
newMsg(0, 0),
211+
newMsg(2, 1),
212+
newMsg(2, 1),
213+
},
214+
firstMsgAt: 2,
215+
wantNumEval: 4,
216+
wantMsgLens: []int{1, 1, 1, 1},
217+
wantErr: false,
218+
},
219+
{
220+
name: "second last no split last split overflow",
221+
msgs: []ipv6.Message{
222+
newMsg(0, 0),
223+
newMsg(0, 0),
224+
newMsg(1, 0),
225+
newMsg(4, 1),
226+
},
227+
firstMsgAt: 2,
228+
wantNumEval: 4,
229+
wantMsgLens: []int{1, 1, 1, 1},
230+
wantErr: true,
231+
},
232+
}
233+
234+
for _, tt := range cases {
235+
t.Run(tt.name, func(t *testing.T) {
236+
got, err := splitCoalescedMessages(tt.msgs, 2, mockGetGSOSize)
237+
if err != nil && !tt.wantErr {
238+
t.Fatalf("err: %v", err)
239+
}
240+
if got != tt.wantNumEval {
241+
t.Fatalf("got to eval: %d want: %d", got, tt.wantNumEval)
242+
}
243+
for i, msg := range tt.msgs {
244+
if msg.N != tt.wantMsgLens[i] {
245+
t.Fatalf("msg[%d].N: %d want: %d", i, msg.N, tt.wantMsgLens[i])
246+
}
247+
}
248+
})
249+
}
250+
}

‎conn/sticky_default.go renamed to ‎conn/control_default.go

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//go:build !linux || android
1+
//go:build !(linux && !android)
22

33
/* SPDX-License-Identifier: MIT
44
*
@@ -21,8 +21,9 @@ func (e *StdNetEndpoint) SrcToString() string {
2121
return ""
2222
}
2323

24-
// TODO: macOS, FreeBSD and other BSDs likely do support this feature set, but
25-
// use alternatively named flags and need ports and require testing.
24+
// TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets
25+
// ({get,set}srcControl feature set, but use alternatively named flags and need
26+
// ports and require testing.
2627

2728
// getSrcFromControl parses the control for PKTINFO and if found updates ep with
2829
// the source information found.
@@ -34,8 +35,17 @@ func getSrcFromControl(control []byte, ep *StdNetEndpoint) {
3435
func setSrcControl(control *[]byte, ep *StdNetEndpoint) {
3536
}
3637

37-
// srcControlSize returns the recommended buffer size for pooling sticky control
38-
// data.
39-
const srcControlSize = 0
38+
// getGSOSize parses control for UDP_GRO and if found returns its GSO size data.
39+
func getGSOSize(control []byte) (int, error) {
40+
return 0, nil
41+
}
42+
43+
// setGSOSize sets a UDP_SEGMENT in control based on gsoSize.
44+
func setGSOSize(control *[]byte, gsoSize uint16) {
45+
}
46+
47+
// controlSize returns the recommended buffer size for pooling sticky and UDP
48+
// offloading control data.
49+
const controlSize = 0
4050

4151
const StdNetSupportsStickySockets = false

‎conn/sticky_linux.go renamed to ‎conn/control_linux.go

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
package conn
99

1010
import (
11+
"fmt"
1112
"net/netip"
1213
"unsafe"
1314

@@ -105,6 +106,54 @@ func setSrcControl(control *[]byte, ep *StdNetEndpoint) {
105106
*control = append(*control, ep.src...)
106107
}
107108

108-
var srcControlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo)
109+
const (
110+
sizeOfGSOData = 2
111+
)
112+
113+
// getGSOSize parses control for UDP_GRO and if found returns its GSO size data.
114+
func getGSOSize(control []byte) (int, error) {
115+
var (
116+
hdr unix.Cmsghdr
117+
data []byte
118+
rem = control
119+
err error
120+
)
121+
122+
for len(rem) > unix.SizeofCmsghdr {
123+
hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem)
124+
if err != nil {
125+
return 0, fmt.Errorf("error parsing socket control message: %w", err)
126+
}
127+
if hdr.Level == socketOptionLevelUDP && hdr.Type == socketOptionUDPGRO && len(data) >= sizeOfGSOData {
128+
var gso uint16
129+
copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData])
130+
return int(gso), nil
131+
}
132+
}
133+
return 0, nil
134+
}
135+
136+
// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing
137+
// data in control untouched.
138+
func setGSOSize(control *[]byte, gsoSize uint16) {
139+
existingLen := len(*control)
140+
avail := cap(*control) - existingLen
141+
space := unix.CmsgSpace(sizeOfGSOData)
142+
if avail < space {
143+
return
144+
}
145+
*control = (*control)[:cap(*control)]
146+
gsoControl := (*control)[existingLen:]
147+
hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0]))
148+
hdr.Level = socketOptionLevelUDP
149+
hdr.Type = socketOptionUDPSegment
150+
hdr.SetLen(unix.CmsgLen(sizeOfGSOData))
151+
copy((gsoControl)[unix.SizeofCmsghdr:], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData))
152+
*control = (*control)[:existingLen+space]
153+
}
154+
155+
// controlSize returns the recommended buffer size for pooling sticky and UDP
156+
// offloading control data.
157+
var controlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) + unix.CmsgSpace(sizeOfGSOData)
109158

110159
const StdNetSupportsStickySockets = true

‎conn/sticky_linux_test.go renamed to ‎conn/control_linux_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func Test_setSrcControl(t *testing.T) {
6060
}
6161
setSrc(ep, netip.MustParseAddr("127.0.0.1"), 5)
6262

63-
control := make([]byte, srcControlSize)
63+
control := make([]byte, controlSize)
6464

6565
setSrcControl(&control, ep)
6666

@@ -89,7 +89,7 @@ func Test_setSrcControl(t *testing.T) {
8989
}
9090
setSrc(ep, netip.MustParseAddr("::1"), 5)
9191

92-
control := make([]byte, srcControlSize)
92+
control := make([]byte, controlSize)
9393

9494
setSrcControl(&control, ep)
9595

‎conn/controlfns_linux.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,5 +57,13 @@ func init() {
5757
}
5858
return err
5959
},
60+
61+
// Attempt to enable UDP_GRO
62+
func(network, address string, c syscall.RawConn) error {
63+
c.Control(func(fd uintptr) {
64+
_ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO, 1)
65+
})
66+
return nil
67+
},
6068
)
6169
}

‎conn/errors_default.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
//go:build !linux
2+
3+
/* SPDX-License-Identifier: MIT
4+
*
5+
* Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
6+
*/
7+
8+
package conn
9+
10+
func errShouldDisableUDPGSO(err error) bool {
11+
return false
12+
}

0 commit comments

Comments
 (0)