Skip to content

Commit e38400d

Browse files
authored
Merge pull request #764 from openziti/use-pool-for-payload-ingest
use pool for payload ingest
2 parents a77a36e + f718ff3 commit e38400d

File tree

16 files changed

+141
-166
lines changed

16 files changed

+141
-166
lines changed

CHANGELOG.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@
33
## Issues Fixed and Dependency Updates
44

55
* github.com/openziti/sdk-golang: [v1.1.2 -> v1.1.3](https://github.com/openziti/sdk-golang/compare/v1.1.2...v1.1.3)
6+
* [Issue #763](https://github.com/openziti/sdk-golang/issues/763) - Use a go-routine pool for payload ingest
7+
* [Issue #761](https://github.com/openziti/sdk-golang/issues/761) - Use cmap.ConcurrentMap for message multiplexer
68
* [Issue #754](https://github.com/openziti/sdk-golang/issues/754) - panic: unaligned 64-bit atomic operation when running on 32-bit raspberry pi
79
* [Issue #757](https://github.com/openziti/sdk-golang/issues/757) - Not authenticated check fails on session create when using OIDC
810

11+
* github.com/golang-jwt/jwt/v5: v5.2.2 -> v5.2.3
912
* github.com/openziti/channel/v4: [v4.2.0 -> v4.2.15](https://github.com/openziti/channel/compare/v4.2.0...v4.2.15)
1013
* [Issue #194](https://github.com/openziti/channel/issues/194) - Add GetUnderlays and GetUnderlayCountsByType to Channel
1114

@@ -27,7 +30,6 @@
2730
* golang.org/x/term: v0.32.0 -> v0.33.0
2831
* golang.org/x/text: v0.25.0 -> v0.27.0
2932

30-
3133
# Release notes 1.1.2
3234

3335
## Issues Fixed and Dependency Updates

edge-apis/pool.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"math/rand/v2"
2525
"net"
2626
"net/url"
27+
"slices"
2728
"sync/atomic"
2829
"time"
2930
)
@@ -35,7 +36,7 @@ type ApiClientTransport struct {
3536

3637
// ClientTransportPool abstracts the concept of multiple `runtime.ClientTransport` (openapi interface) representing one
3738
// target OpenZiti network. In situations where controllers are running in HA mode (multiple controllers) this
38-
// interface can attempt to try different controller during outages or partitioning.
39+
// interface can attempt to try a different controller during outages or partitioning.
3940
type ClientTransportPool interface {
4041
runtime.ClientTransport
4142

@@ -265,6 +266,5 @@ func selectAndRemoveRandom[T any](slice []T, zero T) (selected T, modifiedSlice
265266
rng := rand.New(rand.NewPCG(seed, seed))
266267
index := rng.IntN(len(slice))
267268
selected = slice[index]
268-
modifiedSlice = append(slice[:index], slice[index+1:]...)
269-
return selected, modifiedSlice
269+
return selected, slices.Delete(slice, index, index+1)
270270
}

example/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ require (
5656
github.com/go-openapi/swag v0.23.0 // indirect
5757
github.com/go-openapi/validate v0.24.0 // indirect
5858
github.com/go-resty/resty/v2 v2.16.5 // indirect
59-
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
59+
github.com/golang-jwt/jwt/v5 v5.2.3 // indirect
6060
github.com/golang/protobuf v1.5.4 // indirect
61-
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 // indirect
61+
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b // indirect
6262
github.com/gorilla/schema v1.4.1 // indirect
6363
github.com/gorilla/securecookie v1.1.2 // indirect
6464
github.com/gorilla/websocket v1.5.3 // indirect

example/go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,8 @@ github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptd
155155
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
156156
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
157157
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
158-
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
159-
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
158+
github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0=
159+
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
160160
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
161161
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
162162
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -191,8 +191,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
191191
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
192192
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
193193
github.com/gomarkdown/markdown v0.0.0-20191123064959-2c17d62f5098/go.mod h1:aii0r/K0ZnHv7G0KF7xy1v0A7s2Ljrb5byB7MO5p6TU=
194-
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 h1:EcQR3gusLHN46TAD+G+EbaaqJArt5vHhNpXAa12PQf4=
195-
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
194+
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk=
195+
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
196196
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
197197
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
198198
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=

example/influxdb-client-go/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ require (
5454
github.com/go-playground/validator/v10 v10.15.4 // indirect
5555
github.com/go-resty/resty/v2 v2.16.5 // indirect
5656
github.com/goccy/go-json v0.10.2 // indirect
57-
github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
57+
github.com/golang-jwt/jwt/v5 v5.2.3 // indirect
5858
github.com/golang/snappy v0.0.4 // indirect
59-
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 // indirect
59+
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b // indirect
6060
github.com/google/uuid v1.6.0 // indirect
6161
github.com/gorilla/css v1.0.0 // indirect
6262
github.com/gorilla/mux v1.8.1 // indirect

example/influxdb-client-go/go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
180180
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
181181
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
182182
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
183-
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
184-
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
183+
github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0=
184+
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
185185
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
186186
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
187187
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -213,8 +213,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
213213
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
214214
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
215215
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
216-
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 h1:EcQR3gusLHN46TAD+G+EbaaqJArt5vHhNpXAa12PQf4=
217-
github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
216+
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk=
217+
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
218218
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
219219
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
220220
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ require (
1212
github.com/go-openapi/runtime v0.28.0
1313
github.com/go-openapi/strfmt v0.23.0
1414
github.com/go-resty/resty/v2 v2.16.5
15-
github.com/golang-jwt/jwt/v5 v5.2.2
15+
github.com/golang-jwt/jwt/v5 v5.2.3
1616
github.com/google/uuid v1.6.0
1717
github.com/kataras/go-events v0.0.3
1818
github.com/michaelquigley/pfxlog v0.6.10

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,8 @@ github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptd
122122
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
123123
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
124124
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
125-
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
126-
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
125+
github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0=
126+
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
127127
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
128128
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
129129
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

xgress/circuit_inspections.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,12 @@ type InspectDetail struct {
6262
Goroutines []string `json:"goroutines"`
6363
Sequence uint64 `json:"sequence"`
6464
Flags string `json:"flags"`
65+
LastSizeSent uint32 `json:"lastSizeSent"`
6566
}
6667

6768
type SendBufferDetail struct {
6869
WindowSize uint32 `json:"windowSize"`
70+
QueuedPayloadCount int `json:"queuedPayloadCount"`
6971
LinkSendBufferSize uint32 `json:"linkSendBufferSize"`
7072
LinkRecvBufferSize uint32 `json:"linkRecvBufferSize"`
7173
Accumulator uint32 `json:"accumulator"`
@@ -85,7 +87,6 @@ type SendBufferDetail struct {
8587
type RecvBufferDetail struct {
8688
Size uint32 `json:"size"`
8789
PayloadCount uint32 `json:"payloadCount"`
88-
LastSizeSent uint32 `json:"lastSizeSent"`
8990
Sequence int32 `json:"sequence"`
9091
MaxSequence int32 `json:"maxSequence"`
9192
NextPayload string `json:"nextPayload"`

xgress/link_receive_buffer.go

Lines changed: 45 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -21,22 +21,24 @@ import (
2121
"github.com/emirpasic/gods/trees/btree"
2222
"github.com/emirpasic/gods/utils"
2323
"github.com/michaelquigley/pfxlog"
24+
"sync"
2425
"sync/atomic"
25-
"time"
2626
)
2727

2828
type LinkReceiveBuffer struct {
29-
tree *btree.Tree
30-
sequence int32
31-
maxSequence int32
32-
size uint32
33-
lastBufferSizeSent uint32
29+
sync.Mutex
30+
tree *btree.Tree
31+
sequence int32
32+
maxSequence int32
33+
size uint32
34+
txQueue chan *Payload
3435
}
3536

36-
func NewLinkReceiveBuffer() *LinkReceiveBuffer {
37+
func NewLinkReceiveBuffer(txQueueSize int32) *LinkReceiveBuffer {
3738
return &LinkReceiveBuffer{
3839
tree: btree.NewWith(10240, utils.Int32Comparator),
3940
sequence: -1,
41+
txQueue: make(chan *Payload, txQueueSize),
4042
}
4143
}
4244

@@ -45,6 +47,9 @@ func (buffer *LinkReceiveBuffer) Size() uint32 {
4547
}
4648

4749
func (buffer *LinkReceiveBuffer) ReceiveUnordered(x *Xgress, payload *Payload, maxSize uint32) bool {
50+
buffer.Lock()
51+
defer buffer.Unlock()
52+
4853
if payload.GetSequence() <= buffer.sequence {
4954
x.dataPlane.GetMetrics().MarkDuplicatePayload()
5055
return true
@@ -67,47 +72,56 @@ func (buffer *LinkReceiveBuffer) ReceiveUnordered(x *Xgress, payload *Payload, m
6772
} else {
6873
x.dataPlane.GetMetrics().MarkDuplicatePayload()
6974
}
75+
76+
buffer.queueNext()
77+
7078
return true
7179
}
7280

73-
func (buffer *LinkReceiveBuffer) PeekHead() *Payload {
81+
func (buffer *LinkReceiveBuffer) queueNext() {
7482
if val := buffer.tree.LeftValue(); val != nil {
7583
payload := val.(*Payload)
7684
if payload.Sequence == buffer.sequence+1 {
77-
return payload
85+
select {
86+
case buffer.txQueue <- payload:
87+
buffer.tree.Remove(payload.Sequence)
88+
buffer.sequence = payload.Sequence
89+
default:
90+
}
7891
}
7992
}
80-
return nil
8193
}
8294

83-
func (buffer *LinkReceiveBuffer) Remove(payload *Payload) {
84-
buffer.tree.Remove(payload.Sequence)
85-
buffer.sequence = payload.Sequence
86-
}
95+
func (buffer *LinkReceiveBuffer) NextPayload(closeNotify <-chan struct{}) *Payload {
96+
select {
97+
case payload := <-buffer.txQueue:
98+
return payload
99+
default:
100+
}
87101

88-
func (buffer *LinkReceiveBuffer) getLastBufferSizeSent() uint32 {
89-
return atomic.LoadUint32(&buffer.lastBufferSizeSent)
90-
}
102+
buffer.Lock()
103+
buffer.queueNext()
104+
buffer.Unlock()
91105

92-
func (buffer *LinkReceiveBuffer) Inspect(x *Xgress) *RecvBufferDetail {
93-
timeout := time.After(100 * time.Millisecond)
94-
inspectEvent := &receiveBufferInspectEvent{
95-
buffer: buffer,
96-
notifyComplete: make(chan *RecvBufferDetail, 1),
106+
select {
107+
case payload := <-buffer.txQueue:
108+
return payload
109+
case <-closeNotify:
97110
}
98111

99-
if x.dataPlane.GetPayloadIngester().inspect(inspectEvent, timeout) {
100-
select {
101-
case result := <-inspectEvent.notifyComplete:
102-
return result
103-
case <-timeout:
104-
}
112+
// closed, check if there's anything pending in the queue
113+
select {
114+
case payload := <-buffer.txQueue:
115+
return payload
116+
default:
117+
return nil
105118
}
106-
107-
return buffer.inspectIncomplete()
108119
}
109120

110-
func (buffer *LinkReceiveBuffer) inspectComplete() *RecvBufferDetail {
121+
func (buffer *LinkReceiveBuffer) Inspect() *RecvBufferDetail {
122+
buffer.Lock()
123+
defer buffer.Unlock()
124+
111125
nextPayload := "none"
112126
if head := buffer.tree.LeftValue(); head != nil {
113127
payload := head.(*Payload)
@@ -117,31 +131,9 @@ func (buffer *LinkReceiveBuffer) inspectComplete() *RecvBufferDetail {
117131
return &RecvBufferDetail{
118132
Size: buffer.Size(),
119133
PayloadCount: uint32(buffer.tree.Size()),
120-
LastSizeSent: buffer.getLastBufferSizeSent(),
121134
Sequence: buffer.sequence,
122135
MaxSequence: buffer.maxSequence,
123136
NextPayload: nextPayload,
124137
AcquiredSafely: true,
125138
}
126139
}
127-
128-
func (buffer *LinkReceiveBuffer) inspectIncomplete() *RecvBufferDetail {
129-
return &RecvBufferDetail{
130-
Size: buffer.Size(),
131-
LastSizeSent: buffer.getLastBufferSizeSent(),
132-
Sequence: buffer.sequence,
133-
MaxSequence: buffer.maxSequence,
134-
NextPayload: "unsafe to check",
135-
AcquiredSafely: false,
136-
}
137-
}
138-
139-
type receiveBufferInspectEvent struct {
140-
buffer *LinkReceiveBuffer
141-
notifyComplete chan *RecvBufferDetail
142-
}
143-
144-
func (self *receiveBufferInspectEvent) handle() {
145-
result := self.buffer.inspectComplete()
146-
self.notifyComplete <- result
147-
}

0 commit comments

Comments
 (0)