Skip to content

Commit 65cd6ee

Browse files
jwhitedjwhited
committed
conn,device: provide 8 free bytes at packet head to conn.Bind.Send()
This enables a conn.Bind to bring its own encapsulating transport, e.g. VXLAN/Geneve. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited <[email protected]>
1 parent 0225465 commit 65cd6ee

File tree

10 files changed

+58
-39
lines changed

10 files changed

+58
-39
lines changed

‎conn/bind_std.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ func (e ErrUDPGSODisabled) Unwrap() error {
341341
return e.RetryErr
342342
}
343343

344-
func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error {
344+
func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
345345
s.mu.Lock()
346346
blackhole := s.blackhole4
347347
conn := s.ipv4
@@ -384,7 +384,7 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error {
384384
)
385385
retry:
386386
if offload {
387-
n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, *msgs, setGSOSize)
387+
n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, offset, *msgs, setGSOSize)
388388
err = s.send(conn, br, (*msgs)[:n])
389389
if err != nil && offload && errShouldDisableUDPGSO(err) {
390390
offload = false
@@ -401,7 +401,7 @@ retry:
401401
} else {
402402
for i := range bufs {
403403
(*msgs)[i].Addr = ua
404-
(*msgs)[i].Buffers[0] = bufs[i]
404+
(*msgs)[i].Buffers[0] = bufs[i][offset:]
405405
setSrcControl(&(*msgs)[i].OOB, endpoint.(*StdNetEndpoint))
406406
}
407407
err = s.send(conn, br, (*msgs)[:len(bufs)])
@@ -450,7 +450,7 @@ const (
450450

451451
type setGSOFunc func(control *[]byte, gsoSize uint16)
452452

453-
func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, msgs []ipv6.Message, setGSO setGSOFunc) int {
453+
func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offset int, msgs []ipv6.Message, setGSO setGSOFunc) int {
454454
var (
455455
base = -1 // index of msg we are currently coalescing into
456456
gsoSize int // segmentation size of msgs[base]
@@ -462,6 +462,7 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, msgs
462462
maxPayloadLen = maxIPv6PayloadLen
463463
}
464464
for i, buf := range bufs {
465+
buf = buf[offset:]
465466
if i > 0 {
466467
msgLen := len(buf)
467468
baseLenBefore := len(msgs[base].Buffers[0])

‎conn/bind_std_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ func Test_coalesceMessages(t *testing.T) {
9898
msgs[i].Buffers = make([][]byte, 1)
9999
msgs[i].OOB = make([]byte, 0, 2)
100100
}
101-
got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, msgs, mockSetGSOSize)
101+
got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, 0, msgs, mockSetGSOSize)
102102
if got != len(tt.wantLens) {
103103
t.Fatalf("got len %d want: %d", got, len(tt.wantLens))
104104
}

‎conn/bind_windows.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,14 +486,15 @@ func (bind *afWinRingBind) Send(buf []byte, nend *WinRingEndpoint, isOpen *atomi
486486
return winrio.SendEx(bind.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0)
487487
}
488488

489-
func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint) error {
489+
func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
490490
nend, ok := endpoint.(*WinRingEndpoint)
491491
if !ok {
492492
return ErrWrongEndpointType
493493
}
494494
bind.mu.RLock()
495495
defer bind.mu.RUnlock()
496496
for _, buf := range bufs {
497+
buf = buf[offset:]
497498
switch nend.family {
498499
case windows.AF_INET:
499500
if bind.v4.blackhole {

‎conn/bindtest/bindtest.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,9 @@ func (c *ChannelBind) makeReceiveFunc(ch chan []byte) conn.ReceiveFunc {
107107
}
108108
}
109109

110-
func (c *ChannelBind) Send(bufs [][]byte, ep conn.Endpoint) error {
110+
func (c *ChannelBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error {
111111
for _, b := range bufs {
112+
b = b[offset:]
112113
select {
113114
case <-c.closeSignal:
114115
return net.ErrClosed

‎conn/conn.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@ type Bind interface {
4545
// This mark is passed to the kernel as the socket option SO_MARK.
4646
SetMark(mark uint32) error
4747

48-
// Send writes one or more packets in bufs to address ep. The length of
49-
// bufs must not exceed BatchSize().
50-
Send(bufs [][]byte, ep Endpoint) error
48+
// Send writes one or more packets in bufs to address ep. A nonzero offset
49+
// can be used to instruct the Bind on where packet data begins in each
50+
// element of the bufs slice. Space preceding offset is free to use for
51+
// additional encapsulation. The length of bufs must not exceed BatchSize().
52+
Send(bufs [][]byte, ep Endpoint, offset int) error
5153

5254
// ParseEndpoint creates a new endpoint from a string.
5355
ParseEndpoint(s string) (Endpoint, error)

‎device/constants.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@ const (
2727
)
2828

2929
const (
30-
MinMessageSize = MessageKeepaliveSize // minimum size of transport message (keepalive)
31-
MaxMessageSize = MaxSegmentSize // maximum size of transport message
32-
MaxContentSize = MaxSegmentSize - MessageTransportSize // maximum size of transport message content
30+
MinMessageSize = MessageKeepaliveSize // minimum size of transport message (keepalive)
31+
MaxMessageSize = MaxSegmentSize // maximum size of transport message
32+
MaxContentSize = MaxSegmentSize - MessageTransportSize - MessageEncapsulatingTransportSize // maximum size of transport message content
3333
)
3434

3535
/* Implementation constants */

‎device/device_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -426,11 +426,11 @@ type fakeBindSized struct {
426426
func (b *fakeBindSized) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) {
427427
return nil, 0, nil
428428
}
429-
func (b *fakeBindSized) Close() error { return nil }
430-
func (b *fakeBindSized) SetMark(mark uint32) error { return nil }
431-
func (b *fakeBindSized) Send(bufs [][]byte, ep conn.Endpoint) error { return nil }
432-
func (b *fakeBindSized) ParseEndpoint(s string) (conn.Endpoint, error) { return nil, nil }
433-
func (b *fakeBindSized) BatchSize() int { return b.size }
429+
func (b *fakeBindSized) Close() error { return nil }
430+
func (b *fakeBindSized) SetMark(mark uint32) error { return nil }
431+
func (b *fakeBindSized) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { return nil }
432+
func (b *fakeBindSized) ParseEndpoint(s string) (conn.Endpoint, error) { return nil, nil }
433+
func (b *fakeBindSized) BatchSize() int { return b.size }
434434

435435
type fakeTUNDeviceSized struct {
436436
size int

‎device/noise-protocol.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,14 @@ const (
6161
)
6262

6363
const (
64-
MessageInitiationSize = 148 // size of handshake initiation message
65-
MessageResponseSize = 92 // size of response message
66-
MessageCookieReplySize = 64 // size of cookie reply message
67-
MessageTransportHeaderSize = 16 // size of data preceding content in transport message
68-
MessageTransportSize = MessageTransportHeaderSize + poly1305.TagSize // size of empty transport
69-
MessageKeepaliveSize = MessageTransportSize // size of keepalive
70-
MessageHandshakeSize = MessageInitiationSize // size of largest handshake related message
64+
MessageInitiationSize = 148 // size of handshake initiation message
65+
MessageResponseSize = 92 // size of response message
66+
MessageCookieReplySize = 64 // size of cookie reply message
67+
MessageTransportHeaderSize = 16 // size of data preceding content in transport message
68+
MessageEncapsulatingTransportSize = 8 // size of optional, free (for use by conn.Bind.Send()) space preceding the transport header
69+
MessageTransportSize = MessageTransportHeaderSize + poly1305.TagSize // size of empty transport
70+
MessageKeepaliveSize = MessageTransportSize // size of keepalive
71+
MessageHandshakeSize = MessageInitiationSize // size of largest handshake related message
7172
)
7273

7374
const (

‎device/peer.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
113113
return peer, nil
114114
}
115115

116+
// SendBuffers sends buffers to peer. WireGuard packet data in each element of
117+
// buffers must be preceded by MessageEncapsulatingTransportSize number of
118+
// bytes.
116119
func (peer *Peer) SendBuffers(buffers [][]byte) error {
117120
peer.device.net.RLock()
118121
defer peer.device.net.RUnlock()
@@ -133,7 +136,7 @@ func (peer *Peer) SendBuffers(buffers [][]byte) error {
133136
}
134137
peer.endpoint.Unlock()
135138

136-
err := peer.device.net.bind.Send(buffers, endpoint)
139+
err := peer.device.net.bind.Send(buffers, endpoint, MessageEncapsulatingTransportSize)
137140
if err == nil {
138141
var totalLen uint64
139142
for _, b := range buffers {

‎device/send.go

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,15 @@ import (
4545
*/
4646

4747
type QueueOutboundElement struct {
48-
buffer *[MaxMessageSize]byte // slice holding the packet data
49-
packet []byte // slice of "buffer" (always!)
50-
nonce uint64 // nonce for encryption
51-
keypair *Keypair // keypair for encryption
52-
peer *Peer // related peer
48+
buffer *[MaxMessageSize]byte // slice holding the packet data
49+
// packet is always a slice of "buffer". The starting offset in buffer
50+
// is either:
51+
// a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext)
52+
// b) 0 (post-encryption)
53+
packet []byte
54+
nonce uint64 // nonce for encryption
55+
keypair *Keypair // keypair for encryption
56+
peer *Peer // related peer
5357
}
5458

5559
type QueueOutboundElementsContainer struct {
@@ -123,14 +127,15 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
123127
return err
124128
}
125129

126-
packet := make([]byte, MessageInitiationSize)
130+
buf := make([]byte, MessageEncapsulatingTransportSize+MessageInitiationSize)
131+
packet := buf[MessageEncapsulatingTransportSize:]
127132
_ = msg.marshal(packet)
128133
peer.cookieGenerator.AddMacs(packet)
129134

130135
peer.timersAnyAuthenticatedPacketTraversal()
131136
peer.timersAnyAuthenticatedPacketSent()
132137

133-
err = peer.SendBuffers([][]byte{packet})
138+
err = peer.SendBuffers([][]byte{buf})
134139
if err != nil {
135140
peer.device.log.Errorf("%v - Failed to send handshake initiation: %v", peer, err)
136141
}
@@ -152,7 +157,8 @@ func (peer *Peer) SendHandshakeResponse() error {
152157
return err
153158
}
154159

155-
packet := make([]byte, MessageResponseSize)
160+
buf := make([]byte, MessageEncapsulatingTransportSize+MessageResponseSize)
161+
packet := buf[MessageEncapsulatingTransportSize:]
156162
_ = response.marshal(packet)
157163
peer.cookieGenerator.AddMacs(packet)
158164

@@ -167,7 +173,7 @@ func (peer *Peer) SendHandshakeResponse() error {
167173
peer.timersAnyAuthenticatedPacketSent()
168174

169175
// TODO: allocation could be avoided
170-
err = peer.SendBuffers([][]byte{packet})
176+
err = peer.SendBuffers([][]byte{buf})
171177
if err != nil {
172178
peer.device.log.Errorf("%v - Failed to send handshake response: %v", peer, err)
173179
}
@@ -184,10 +190,11 @@ func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement)
184190
return err
185191
}
186192

187-
packet := make([]byte, MessageCookieReplySize)
193+
buf := make([]byte, MessageEncapsulatingTransportSize+MessageCookieReplySize)
194+
packet := buf[MessageEncapsulatingTransportSize:]
188195
_ = reply.marshal(packet)
189196
// TODO: allocation could be avoided
190-
device.net.bind.Send([][]byte{packet}, initiatingElem.endpoint)
197+
device.net.bind.Send([][]byte{buf}, initiatingElem.endpoint, MessageEncapsulatingTransportSize)
191198

192199
return nil
193200
}
@@ -220,7 +227,7 @@ func (device *Device) RoutineReadFromTUN() {
220227
elemsByPeer = make(map[*Peer]*QueueOutboundElementsContainer, batchSize)
221228
count = 0
222229
sizes = make([]int, batchSize)
223-
offset = MessageTransportHeaderSize
230+
offset = MessageEncapsulatingTransportSize + MessageTransportHeaderSize
224231
)
225232

226233
for i := range elems {
@@ -446,7 +453,7 @@ func (device *Device) RoutineEncryption(id int) {
446453
for elemsContainer := range device.queue.encryption.c {
447454
for _, elem := range elemsContainer.elems {
448455
// populate header fields
449-
header := elem.buffer[:MessageTransportHeaderSize]
456+
header := elem.buffer[MessageEncapsulatingTransportSize : MessageEncapsulatingTransportSize+MessageTransportHeaderSize]
450457

451458
fieldType := header[0:4]
452459
fieldReceiver := header[4:8]
@@ -469,6 +476,9 @@ func (device *Device) RoutineEncryption(id int) {
469476
elem.packet,
470477
nil,
471478
)
479+
480+
// re-slice packet to include encapsulating transport space
481+
elem.packet = elem.buffer[:MessageEncapsulatingTransportSize+len(elem.packet)]
472482
}
473483
elemsContainer.Unlock()
474484
}

0 commit comments

Comments
 (0)