Skip to content

Commit 5a8e1fd

Browse files
cnvergencecnvergence
committed
Allow configuration of transport channel size in agent and server
Signed-off-by: Karol Szwaj <[email protected]>
1 parent 94bd4ac commit 5a8e1fd

File tree

7 files changed

+48
-28
lines changed

7 files changed

+48
-28
lines changed

‎cmd/agent/app/options/options.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ type GrpcProxyAgentOptions struct {
7878
// The check is an "unlocked" read but is still use at your own peril.
7979
WarnOnChannelLimit bool
8080

81-
SyncForever bool
81+
SyncForever bool
82+
XrfChannelSize int
8283
}
8384

8485
func (o *GrpcProxyAgentOptions) ClientSetConfig(dialOptions ...grpc.DialOption) *agent.ClientSetConfig {
@@ -93,6 +94,7 @@ func (o *GrpcProxyAgentOptions) ClientSetConfig(dialOptions ...grpc.DialOption)
9394
ServiceAccountTokenPath: o.ServiceAccountTokenPath,
9495
WarnOnChannelLimit: o.WarnOnChannelLimit,
9596
SyncForever: o.SyncForever,
97+
XrfChannelSize: o.XrfChannelSize,
9698
}
9799
}
98100

@@ -119,6 +121,7 @@ func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet {
119121
flags.StringVar(&o.AgentIdentifiers, "agent-identifiers", o.AgentIdentifiers, "Identifiers of the agent that will be used by the server when choosing agent. N.B. the list of identifiers must be in URL encoded format. e.g.,host=localhost&host=node1.mydomain.com&cidr=127.0.0.1/16&ipv4=1.2.3.4&ipv4=5.6.7.8&ipv6=:::::&default-route=true")
120122
flags.BoolVar(&o.WarnOnChannelLimit, "warn-on-channel-limit", o.WarnOnChannelLimit, "Turns on a warning if the system is going to push to a full channel. The check involves an unsafe read.")
121123
flags.BoolVar(&o.SyncForever, "sync-forever", o.SyncForever, "If true, the agent continues syncing, in order to support server count changes.")
124+
flags.IntVar(&o.XrfChannelSize, "channel-size", 150, "Set the size of the channel")
122125
return flags
123126
}
124127

@@ -144,6 +147,7 @@ func (o *GrpcProxyAgentOptions) Print() {
144147
klog.V(1).Infof("AgentIdentifiers set to %s.\n", util.PrettyPrintURL(o.AgentIdentifiers))
145148
klog.V(1).Infof("WarnOnChannelLimit set to %t.\n", o.WarnOnChannelLimit)
146149
klog.V(1).Infof("SyncForever set to %v.\n", o.SyncForever)
150+
klog.V(1).Infof("ChannelSize set to %d.\n", o.XrfChannelSize)
147151
}
148152

149153
func (o *GrpcProxyAgentOptions) Validate() error {
@@ -177,6 +181,9 @@ func (o *GrpcProxyAgentOptions) Validate() error {
177181
if o.AdminServerPort <= 0 {
178182
return fmt.Errorf("admin server port %d must be greater than 0", o.AdminServerPort)
179183
}
184+
if o.XrfChannelSize <= 0 {
185+
return fmt.Errorf("channel size %d must be greater than 0", o.XrfChannelSize)
186+
}
180187
if o.EnableContentionProfiling && !o.EnableProfiling {
181188
return fmt.Errorf("if --enable-contention-profiling is set, --enable-profiling must also be set")
182189
}
@@ -235,6 +242,7 @@ func NewGrpcProxyAgentOptions() *GrpcProxyAgentOptions {
235242
ServiceAccountTokenPath: "",
236243
WarnOnChannelLimit: false,
237244
SyncForever: false,
245+
XrfChannelSize: 150,
238246
}
239247
return &o
240248
}

‎cmd/server/app/options/options.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,8 @@ type ProxyRunOptions struct {
100100
// also checks if given comma separated list contains cipher from tls.InsecureCipherSuites().
101101
// NOTE that cipher suites are not configurable for TLS1.3,
102102
// see: https://pkg.go.dev/crypto/tls#Config, so in that case, this option won't have any effect.
103-
CipherSuites []string
103+
CipherSuites []string
104+
XrfChannelSize int
104105
}
105106

106107
func (o *ProxyRunOptions) Flags() *pflag.FlagSet {
@@ -136,6 +137,7 @@ func (o *ProxyRunOptions) Flags() *pflag.FlagSet {
136137
flags.StringVar(&o.AuthenticationAudience, "authentication-audience", o.AuthenticationAudience, "Expected agent's token authentication audience (used with agent-namespace, agent-service-account, kubeconfig).")
137138
flags.StringVar(&o.ProxyStrategies, "proxy-strategies", o.ProxyStrategies, "The list of proxy strategies used by the server to pick an agent/tunnel, available strategies are: default, destHost, defaultRoute.")
138139
flags.StringSliceVar(&o.CipherSuites, "cipher-suites", o.CipherSuites, "The comma separated list of allowed cipher suites. Has no effect on TLS1.3. Empty means allow default list.")
140+
flags.IntVar(&o.XrfChannelSize, "xfr-channel-size", o.XrfChannelSize, "The size of the channel for transferring data between the proxy server and the agent.")
139141

140142
flags.Bool("warn-on-channel-limit", true, "This behavior is now thread safe and always on. This flag will be removed in a future release.")
141143
flags.MarkDeprecated("warn-on-channel-limit", "This behavior is now thread safe and always on. This flag will be removed in a future release.")
@@ -175,6 +177,7 @@ func (o *ProxyRunOptions) Print() {
175177
klog.V(1).Infof("KubeconfigBurst set to %d.\n", o.KubeconfigBurst)
176178
klog.V(1).Infof("ProxyStrategies set to %q.\n", o.ProxyStrategies)
177179
klog.V(1).Infof("CipherSuites set to %q.\n", o.CipherSuites)
180+
klog.V(1).Infof("XrfChannelSize set to %d.\n", o.XrfChannelSize)
178181
}
179182

180183
func (o *ProxyRunOptions) Validate() error {
@@ -297,7 +300,9 @@ func (o *ProxyRunOptions) Validate() error {
297300
if _, err := server.ParseProxyStrategies(o.ProxyStrategies); err != nil {
298301
return fmt.Errorf("invalid proxy strategies: %v", err)
299302
}
300-
303+
if o.XrfChannelSize <= 0 {
304+
return fmt.Errorf("channel size %d must be greater than 0", o.XrfChannelSize)
305+
}
301306
// validate the cipher suites
302307
if len(o.CipherSuites) != 0 {
303308
acceptedCiphers := util.GetAcceptedCiphers()
@@ -345,6 +350,7 @@ func NewProxyRunOptions() *ProxyRunOptions {
345350
AuthenticationAudience: "",
346351
ProxyStrategies: "default",
347352
CipherSuites: make([]string, 0),
353+
XrfChannelSize: 10,
348354
}
349355
return &o
350356
}

‎cmd/server/app/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func (p *Proxy) Run(o *options.ProxyRunOptions, stopCh <-chan struct{}) error {
132132
if err != nil {
133133
return err
134134
}
135-
p.server = server.NewProxyServer(o.ServerID, ps, int(o.ServerCount), authOpt)
135+
p.server = server.NewProxyServer(o.ServerID, ps, int(o.ServerCount), authOpt, o.XrfChannelSize)
136136

137137
frontendStop, err := p.runFrontendServer(ctx, o, p.server)
138138
if err != nil {

‎pkg/agent/client.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ import (
4343
)
4444

4545
const dialTimeout = 5 * time.Second
46-
const xfrChannelSize = 150
46+
47+
//const xfrChannelSize = 150
4748

4849
// endpointConn tracks a connection from agent to node network.
4950
type endpointConn struct {
@@ -68,7 +69,7 @@ func (e *endpointConn) send(msg []byte) {
6869
klog.InfoS("Recovered from attempt to write to closed channel")
6970
}
7071
}()
71-
if e.warnChLim && len(e.dataCh) >= xfrChannelSize {
72+
if e.warnChLim && len(e.dataCh) >= cap(e.dataCh) {
7273
klog.V(2).InfoS("Data channel on agent is full", "connectionID", e.connID)
7374
}
7475

@@ -377,7 +378,7 @@ func (a *Client) Serve() {
377378
dialResp.GetDialResponse().Random = dialReq.Random
378379

379380
connID := atomic.AddInt64(&a.nextConnID, 1)
380-
dataCh := make(chan []byte, xfrChannelSize)
381+
dataCh := make(chan []byte, a.cs.xfrChannelSize)
381382
dialDone := make(chan struct{})
382383
eConn := &endpointConn{
383384
dataCh: dataCh,

‎pkg/agent/clientset.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ type ClientSet struct {
6161
// by the server when choosing agent
6262

6363
warnOnChannelLimit bool
64+
xfrChannelSize int
6465

6566
syncForever bool // Continue syncing (support dynamic server count).
6667
}
@@ -141,6 +142,7 @@ type ClientSetConfig struct {
141142
ServiceAccountTokenPath string
142143
WarnOnChannelLimit bool
143144
SyncForever bool
145+
XrfChannelSize int
144146
}
145147

146148
func (cc *ClientSetConfig) NewAgentClientSet(drainCh, stopCh <-chan struct{}) *ClientSet {
@@ -157,6 +159,7 @@ func (cc *ClientSetConfig) NewAgentClientSet(drainCh, stopCh <-chan struct{}) *C
157159
warnOnChannelLimit: cc.WarnOnChannelLimit,
158160
syncForever: cc.SyncForever,
159161
drainCh: drainCh,
162+
xfrChannelSize: cc.XrfChannelSize,
160163
stopCh: stopCh,
161164
}
162165
}

‎pkg/server/server.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@ import (
4747
"sigs.k8s.io/apiserver-network-proxy/proto/header"
4848
)
4949

50-
const xfrChannelSize = 10
51-
5250
type key int
5351

5452
type GrpcFrontend struct {
@@ -218,6 +216,7 @@ type ProxyServer struct {
218216

219217
// TODO: move strategies into BackendStorage
220218
proxyStrategies []ProxyStrategy
219+
xfrChannelSize int
221220
}
222221

223222
// AgentTokenAuthenticationOptions contains list of parameters required for agent token based authentication
@@ -376,7 +375,7 @@ func (s *ProxyServer) removeEstablishedForStream(streamUID string) []*ProxyClien
376375
}
377376

378377
// NewProxyServer creates a new ProxyServer instance
379-
func NewProxyServer(serverID string, proxyStrategies []ProxyStrategy, serverCount int, agentAuthenticationOptions *AgentTokenAuthenticationOptions) *ProxyServer {
378+
func NewProxyServer(serverID string, proxyStrategies []ProxyStrategy, serverCount int, agentAuthenticationOptions *AgentTokenAuthenticationOptions, channelSize int) *ProxyServer {
380379
var bms []BackendManager
381380
for _, ps := range proxyStrategies {
382381
switch ps {
@@ -401,6 +400,7 @@ func NewProxyServer(serverID string, proxyStrategies []ProxyStrategy, serverCoun
401400
// use the first backend-manager as the Readiness Manager
402401
Readiness: bms[0],
403402
proxyStrategies: proxyStrategies,
403+
xfrChannelSize: channelSize,
404404
}
405405
}
406406

@@ -417,7 +417,7 @@ func (s *ProxyServer) Proxy(stream client.ProxyService_ProxyServer) error {
417417
streamUID := uuid.New().String()
418418
klog.V(5).InfoS("Proxy request from client", "userAgent", userAgent, "serverID", s.serverID, "streamUID", streamUID)
419419

420-
recvCh := make(chan *client.Packet, xfrChannelSize)
420+
recvCh := make(chan *client.Packet, s.xfrChannelSize)
421421
stopCh := make(chan error, 1)
422422

423423
frontend := GrpcFrontend{
@@ -745,7 +745,7 @@ func (s *ProxyServer) Connect(stream agent.AgentService_ConnectServer) error {
745745
s.addBackend(backend)
746746
defer s.removeBackend(backend)
747747

748-
recvCh := make(chan *client.Packet, xfrChannelSize)
748+
recvCh := make(chan *client.Packet, s.xfrChannelSize)
749749

750750
go runpprof.Do(context.Background(), labels, func(context.Context) { s.serveRecvBackend(backend, agentID, recvCh) })
751751

‎pkg/server/server_test.go

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ import (
4444
"sigs.k8s.io/apiserver-network-proxy/proto/header"
4545
)
4646

47+
const xfrChannelSize = 10
48+
4749
func TestAgentTokenAuthenticationErrorsToken(t *testing.T) {
4850
stub := gomock.NewController(t)
4951
defer stub.Finish()
@@ -172,7 +174,7 @@ func TestAgentTokenAuthenticationErrorsToken(t *testing.T) {
172174
KubernetesClient: kcs,
173175
AgentNamespace: tc.wantNamespace,
174176
AgentServiceAccount: tc.wantServiceAccount,
175-
})
177+
}, xfrChannelSize)
176178

177179
err := p.Connect(conn)
178180
if tc.wantError {
@@ -195,7 +197,7 @@ func TestRemovePendingDialForStream(t *testing.T) {
195197
pending3 := &ProxyClientConnection{frontend: &GrpcFrontend{streamUID: streamUID}}
196198
pending4 := &ProxyClientConnection{frontend: &GrpcFrontend{streamUID: "different-uid"}}
197199
pending5 := &ProxyClientConnection{frontend: &GrpcFrontend{streamUID: ""}}
198-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
200+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
199201
p.PendingDial.Add(1, pending1)
200202
p.PendingDial.Add(2, pending2)
201203
p.PendingDial.Add(3, pending3)
@@ -223,15 +225,15 @@ func TestAddRemoveFrontends(t *testing.T) {
223225
agent2ConnID2 := new(ProxyClientConnection)
224226
agent3ConnID1 := new(ProxyClientConnection)
225227

226-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
228+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
227229
p.addEstablished("agent1", int64(1), agent1ConnID1)
228230
p.removeEstablished("agent1", int64(1))
229231
expectedFrontends := make(map[string]map[int64]*ProxyClientConnection)
230232
if e, a := expectedFrontends, p.established; !reflect.DeepEqual(e, a) {
231233
t.Errorf("expected %v, got %v", e, a)
232234
}
233235

234-
p = NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
236+
p = NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
235237
p.addEstablished("agent1", int64(1), agent1ConnID1)
236238
p.addEstablished("agent1", int64(2), agent1ConnID2)
237239
p.addEstablished("agent2", int64(1), agent2ConnID1)
@@ -261,7 +263,7 @@ func TestAddRemoveBackends_DefaultStrategy(t *testing.T) {
261263
backend2, _ := NewBackend(mockAgentConn(ctrl, "agent2", []string{}))
262264
backend3, _ := NewBackend(mockAgentConn(ctrl, "agent3", []string{}))
263265

264-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
266+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
265267

266268
p.addBackend(backend1)
267269

@@ -293,7 +295,7 @@ func TestAddRemoveBackends_DefaultRouteStrategy(t *testing.T) {
293295
backend2, _ := NewBackend(mockAgentConn(ctrl, "agent2", []string{"default-route=false"}))
294296
backend3, _ := NewBackend(mockAgentConn(ctrl, "agent3", []string{"default-route=true"}))
295297

296-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefaultRoute}, 1, nil)
298+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefaultRoute}, 1, nil, xfrChannelSize)
297299

298300
p.addBackend(backend1)
299301

@@ -335,7 +337,7 @@ func TestAddRemoveBackends_DestHostStrategy(t *testing.T) {
335337
backend2, _ := NewBackend(mockAgentConn(ctrl, "agent2", []string{"default-route=true"}))
336338
backend3, _ := NewBackend(mockAgentConn(ctrl, "agent3", []string{"host=node2.mydomain.com&ipv4=5.6.7.8&ipv6=::"}))
337339

338-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost}, 1, nil)
340+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost}, 1, nil, xfrChannelSize)
339341

340342
p.addBackend(backend1)
341343
p.addBackend(backend2)
@@ -382,7 +384,7 @@ func TestAddRemoveBackends_DestHostSanitizeRequest(t *testing.T) {
382384
backend1, _ := NewBackend(mockAgentConn(ctrl, "agent1", []string{"host=localhost&host=node1.mydomain.com&ipv4=1.2.3.4&ipv6=9878::7675:1292:9183:7562"}))
383385
backend2, _ := NewBackend(mockAgentConn(ctrl, "agent2", []string{"host=node2.mydomain.com&ipv4=5.6.7.8&ipv6=::"}))
384386

385-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost}, 1, nil)
387+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost}, 1, nil, xfrChannelSize)
386388

387389
p.addBackend(backend1)
388390
p.addBackend(backend2)
@@ -406,7 +408,7 @@ func TestAddRemoveBackends_DestHostWithDefault(t *testing.T) {
406408
backend2, _ := NewBackend(mockAgentConn(ctrl, "agent2", []string{"default-route=false"}))
407409
backend3, _ := NewBackend(mockAgentConn(ctrl, "agent3", []string{"host=node2.mydomain.com&ipv4=5.6.7.8&ipv6=::"}))
408410

409-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost, ProxyStrategyDefault}, 1, nil)
411+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost, ProxyStrategyDefault}, 1, nil, xfrChannelSize)
410412

411413
p.addBackend(backend1)
412414
p.addBackend(backend2)
@@ -459,7 +461,7 @@ func TestAddRemoveBackends_DestHostWithDuplicateIdents(t *testing.T) {
459461
backend2, _ := NewBackend(mockAgentConn(ctrl, "agent2", []string{"host=localhost&host=node1.mydomain.com&ipv4=1.2.3.4&ipv6=9878::7675:1292:9183:7562"}))
460462
backend3, _ := NewBackend(mockAgentConn(ctrl, "agent3", []string{"host=localhost&host=node2.mydomain.com&ipv4=5.6.7.8&ipv6=::"}))
461463

462-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost, ProxyStrategyDefault}, 1, nil)
464+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDestHost, ProxyStrategyDefault}, 1, nil, xfrChannelSize)
463465

464466
p.addBackend(backend1)
465467
p.addBackend(backend2)
@@ -516,7 +518,7 @@ func TestEstablishedConnsMetric(t *testing.T) {
516518
agent2ConnID2 := new(ProxyClientConnection)
517519
agent3ConnID1 := new(ProxyClientConnection)
518520

519-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
521+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
520522
p.addEstablished("agent1", int64(1), agent1ConnID1)
521523
assertEstablishedConnsMetric(t, 1)
522524
p.addEstablished("agent1", int64(2), agent1ConnID2)
@@ -548,7 +550,7 @@ func TestRemoveEstablishedForBackendConn(t *testing.T) {
548550
agent2ConnID1 := &ProxyClientConnection{backend: backend2}
549551
agent2ConnID2 := &ProxyClientConnection{backend: backend2}
550552
agent3ConnID1 := &ProxyClientConnection{backend: backend3}
551-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
553+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
552554
p.addEstablished("agent1", int64(1), agent1ConnID1)
553555
p.addEstablished("agent1", int64(2), agent1ConnID2)
554556
p.addEstablished("agent2", int64(1), agent2ConnID1)
@@ -579,7 +581,7 @@ func TestRemoveEstablishedForStream(t *testing.T) {
579581
agent2ConnID1 := &ProxyClientConnection{backend: backend2, frontend: &GrpcFrontend{streamUID: streamUID}}
580582
agent2ConnID2 := &ProxyClientConnection{backend: backend2}
581583
agent3ConnID1 := &ProxyClientConnection{backend: backend3, frontend: &GrpcFrontend{streamUID: streamUID}}
582-
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil)
584+
p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, xfrChannelSize)
583585
p.addEstablished("agent1", int64(1), agent1ConnID1)
584586
p.addEstablished("agent1", int64(2), agent1ConnID2)
585587
p.addEstablished("agent2", int64(1), agent2ConnID1)
@@ -639,7 +641,7 @@ func baseServerProxyTestWithoutBackend(t *testing.T, validate func(*agentmock.Mo
639641
defer ctrl.Finish()
640642

641643
frontendConn := prepareFrontendConn(ctrl)
642-
proxyServer := NewProxyServer(uuid.New().String(), []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{})
644+
proxyServer := NewProxyServer(uuid.New().String(), []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{}, xfrChannelSize)
643645

644646
validate(frontendConn)
645647

@@ -653,7 +655,7 @@ func baseServerProxyTestWithBackend(t *testing.T, validate func(*agentmock.MockA
653655
frontendConn := prepareFrontendConn(ctrl)
654656

655657
// prepare proxy server
656-
proxyServer := NewProxyServer(uuid.New().String(), []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{})
658+
proxyServer := NewProxyServer(uuid.New().String(), []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{}, xfrChannelSize)
657659

658660
agentConn, _ := prepareAgentConnMD(t, ctrl, proxyServer)
659661

@@ -859,7 +861,7 @@ func TestReadyBackendsMetric(t *testing.T) {
859861

860862
metrics.Metrics.Reset()
861863

862-
p := NewProxyServer(uuid.New().String(), []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{})
864+
p := NewProxyServer(uuid.New().String(), []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{}, xfrChannelSize)
863865
assertReadyBackendsMetric(t, 0)
864866

865867
_, backend := prepareAgentConnMD(t, ctrl, p)

0 commit comments

Comments
 (0)