Skip to content

Commit 71d3800

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #754 from maqiuyujoyce/202506-total-backend-metric
Support proxy_server_ready_backends metric
2 parents ea89bf0 + 25229a2 commit 71d3800

File tree

12 files changed

+321
-228
lines changed

12 files changed

+321
-228
lines changed

‎cmd/server/app/options/options.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"k8s.io/klog/v2"
2828

2929
"sigs.k8s.io/apiserver-network-proxy/pkg/server"
30+
"sigs.k8s.io/apiserver-network-proxy/pkg/server/proxystrategies"
3031
"sigs.k8s.io/apiserver-network-proxy/pkg/util"
3132
)
3233

@@ -314,7 +315,7 @@ func (o *ProxyRunOptions) Validate() error {
314315
if len(o.ProxyStrategies) == 0 {
315316
return fmt.Errorf("ProxyStrategies cannot be empty")
316317
}
317-
if _, err := server.ParseProxyStrategies(o.ProxyStrategies); err != nil {
318+
if _, err := proxystrategies.ParseProxyStrategies(o.ProxyStrategies); err != nil {
318319
return fmt.Errorf("invalid proxy strategies: %v", err)
319320
}
320321
if o.XfrChannelSize <= 0 {

‎cmd/server/app/server.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import (
4646
"sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
4747
"sigs.k8s.io/apiserver-network-proxy/pkg/server"
4848
"sigs.k8s.io/apiserver-network-proxy/pkg/server/leases"
49+
"sigs.k8s.io/apiserver-network-proxy/pkg/server/proxystrategies"
4950
"sigs.k8s.io/apiserver-network-proxy/pkg/util"
5051
"sigs.k8s.io/apiserver-network-proxy/proto/agent"
5152
)
@@ -134,7 +135,7 @@ func (p *Proxy) Run(o *options.ProxyRunOptions, stopCh <-chan struct{}) error {
134135
AuthenticationAudience: o.AuthenticationAudience,
135136
}
136137
klog.V(1).Infoln("Starting frontend server for client connections.")
137-
ps, err := server.ParseProxyStrategies(o.ProxyStrategies)
138+
ps, err := proxystrategies.ParseProxyStrategies(o.ProxyStrategies)
138139
if err != nil {
139140
return err
140141
}

‎pkg/server/backend_manager.go

Lines changed: 18 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"io"
2323
"math/rand"
2424
"slices"
25-
"strings"
2625
"sync"
2726
"time"
2827

@@ -32,72 +31,11 @@ import (
3231
commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics"
3332
client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
3433
"sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics"
34+
"sigs.k8s.io/apiserver-network-proxy/pkg/server/proxystrategies"
3535
"sigs.k8s.io/apiserver-network-proxy/proto/agent"
3636
"sigs.k8s.io/apiserver-network-proxy/proto/header"
3737
)
3838

39-
type ProxyStrategy int
40-
41-
const (
42-
// With this strategy the Proxy Server will randomly pick a backend from
43-
// the current healthy backends to establish the tunnel over which to
44-
// forward requests.
45-
ProxyStrategyDefault ProxyStrategy = iota + 1
46-
// With this strategy the Proxy Server will pick a backend that has the same
47-
// associated host as the request.Host to establish the tunnel.
48-
ProxyStrategyDestHost
49-
// ProxyStrategyDefaultRoute will only forward traffic to agents that have explicity advertised
50-
// they serve the default route through an agent identifier. Typically used in combination with destHost
51-
ProxyStrategyDefaultRoute
52-
)
53-
54-
func (ps ProxyStrategy) String() string {
55-
switch ps {
56-
case ProxyStrategyDefault:
57-
return "default"
58-
case ProxyStrategyDestHost:
59-
return "destHost"
60-
case ProxyStrategyDefaultRoute:
61-
return "defaultRoute"
62-
}
63-
panic(fmt.Sprintf("unhandled ProxyStrategy: %d", ps))
64-
}
65-
66-
func ParseProxyStrategy(s string) (ProxyStrategy, error) {
67-
switch s {
68-
case ProxyStrategyDefault.String():
69-
return ProxyStrategyDefault, nil
70-
case ProxyStrategyDestHost.String():
71-
return ProxyStrategyDestHost, nil
72-
case ProxyStrategyDefaultRoute.String():
73-
return ProxyStrategyDefaultRoute, nil
74-
default:
75-
return 0, fmt.Errorf("unknown proxy strategy: %s", s)
76-
}
77-
}
78-
79-
// GenProxyStrategiesFromStr generates the list of proxy strategies from the
80-
// comma-seperated string, i.e., destHost.
81-
func ParseProxyStrategies(proxyStrategies string) ([]ProxyStrategy, error) {
82-
var result []ProxyStrategy
83-
84-
strs := strings.Split(proxyStrategies, ",")
85-
for _, s := range strs {
86-
if len(s) == 0 {
87-
continue
88-
}
89-
ps, err := ParseProxyStrategy(s)
90-
if err != nil {
91-
return nil, err
92-
}
93-
result = append(result, ps)
94-
}
95-
if len(result) == 0 {
96-
return nil, fmt.Errorf("proxy strategies cannot be empty")
97-
}
98-
return result, nil
99-
}
100-
10139
// Backend abstracts a connected Konnectivity agent.
10240
//
10341
// In the only currently supported case (gRPC), it wraps an
@@ -271,24 +209,31 @@ type DefaultBackendStorage struct {
271209
// e.g., when associating to the DestHostBackendManager, it can only use the
272210
// identifiers of types, IPv4, IPv6 and Host.
273211
idTypes []header.IdentifierType
212+
// proxyStrategy is the proxy strategy of the backend manager this storage
213+
// belongs to.
214+
// It is used to record metrics.
215+
proxyStrategy proxystrategies.ProxyStrategy
274216
}
275217

276218
// NewDefaultBackendManager returns a DefaultBackendManager.
277219
func NewDefaultBackendManager() *DefaultBackendManager {
278220
return &DefaultBackendManager{
279221
DefaultBackendStorage: NewDefaultBackendStorage(
280-
[]header.IdentifierType{header.UID})}
222+
[]header.IdentifierType{header.UID}, proxystrategies.ProxyStrategyDefault)}
281223
}
282224

283225
// NewDefaultBackendStorage returns a DefaultBackendStorage
284-
func NewDefaultBackendStorage(idTypes []header.IdentifierType) *DefaultBackendStorage {
226+
func NewDefaultBackendStorage(idTypes []header.IdentifierType, proxyStrategy proxystrategies.ProxyStrategy) *DefaultBackendStorage {
285227
// Set an explicit value, so that the metric is emitted even when
286228
// no agent ever successfully connects.
287-
metrics.Metrics.SetBackendCount(0)
229+
metrics.Metrics.SetBackendCountDeprecated(0)
230+
metrics.Metrics.SetTotalBackendCount(proxyStrategy, 0)
231+
288232
return &DefaultBackendStorage{
289-
backends: make(map[string][]*Backend),
290-
random: rand.New(rand.NewSource(time.Now().UnixNano())), /* #nosec G404 */
291-
idTypes: idTypes,
233+
backends: make(map[string][]*Backend),
234+
random: rand.New(rand.NewSource(time.Now().UnixNano())), /* #nosec G404 */
235+
idTypes: idTypes,
236+
proxyStrategy: proxyStrategy,
292237
}
293238
}
294239

@@ -317,7 +262,8 @@ func (s *DefaultBackendStorage) addBackend(identifier string, idType header.Iden
317262
return
318263
}
319264
s.backends[identifier] = []*Backend{backend}
320-
metrics.Metrics.SetBackendCount(len(s.backends))
265+
metrics.Metrics.SetBackendCountDeprecated(len(s.backends))
266+
metrics.Metrics.SetTotalBackendCount(s.proxyStrategy, len(s.backends))
321267
s.agentIDs = append(s.agentIDs, identifier)
322268
}
323269

@@ -358,7 +304,8 @@ func (s *DefaultBackendStorage) removeBackend(identifier string, idType header.I
358304
if !found {
359305
klog.V(1).InfoS("Could not find connection matching identifier to remove", "agentID", identifier, "idType", idType)
360306
}
361-
metrics.Metrics.SetBackendCount(len(s.backends))
307+
metrics.Metrics.SetBackendCountDeprecated(len(s.backends))
308+
metrics.Metrics.SetTotalBackendCount(s.proxyStrategy, len(s.backends))
362309
}
363310

364311
// NumBackends resturns the number of available backends

‎pkg/server/backend_manager_test.go

Lines changed: 0 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,9 @@ package server
1818

1919
import (
2020
"context"
21-
"fmt"
2221
"reflect"
2322
"testing"
2423

25-
"github.com/stretchr/testify/assert"
2624
"go.uber.org/mock/gomock"
2725
"google.golang.org/grpc/metadata"
2826

@@ -385,123 +383,3 @@ func TestDestHostBackendManager_WithDuplicateIdents(t *testing.T) {
385383
t.Errorf("expected %v, got %v", e, a)
386384
}
387385
}
388-
389-
func TestProxyStrategy(t *testing.T) {
390-
for desc, tc := range map[string]struct {
391-
input ProxyStrategy
392-
want string
393-
wantPanic string
394-
}{
395-
"default": {
396-
input: ProxyStrategyDefault,
397-
want: "default",
398-
},
399-
"destHost": {
400-
input: ProxyStrategyDestHost,
401-
want: "destHost",
402-
},
403-
"defaultRoute": {
404-
input: ProxyStrategyDefaultRoute,
405-
want: "defaultRoute",
406-
},
407-
"unrecognized": {
408-
input: ProxyStrategy(0),
409-
wantPanic: "unhandled ProxyStrategy: 0",
410-
},
411-
} {
412-
t.Run(desc, func(t *testing.T) {
413-
if tc.wantPanic != "" {
414-
assert.PanicsWithValue(t, tc.wantPanic, func() {
415-
_ = tc.input.String()
416-
})
417-
} else {
418-
got := tc.input.String()
419-
if got != tc.want {
420-
t.Errorf("ProxyStrategy.String(): got %v, want %v", got, tc.want)
421-
}
422-
}
423-
})
424-
}
425-
}
426-
427-
func TestParseProxyStrategy(t *testing.T) {
428-
for desc, tc := range map[string]struct {
429-
input string
430-
want ProxyStrategy
431-
wantErr error
432-
}{
433-
"empty": {
434-
input: "",
435-
wantErr: fmt.Errorf("unknown proxy strategy: "),
436-
},
437-
"unrecognized": {
438-
input: "unrecognized",
439-
wantErr: fmt.Errorf("unknown proxy strategy: unrecognized"),
440-
},
441-
"default": {
442-
input: "default",
443-
want: ProxyStrategyDefault,
444-
},
445-
"destHost": {
446-
input: "destHost",
447-
want: ProxyStrategyDestHost,
448-
},
449-
"defaultRoute": {
450-
input: "defaultRoute",
451-
want: ProxyStrategyDefaultRoute,
452-
},
453-
} {
454-
t.Run(desc, func(t *testing.T) {
455-
got, err := ParseProxyStrategy(tc.input)
456-
assert.Equal(t, tc.wantErr, err, "ParseProxyStrategy(%s): got error %q, want %v", tc.input, err, tc.wantErr)
457-
if got != tc.want {
458-
t.Errorf("ParseProxyStrategy(%s): got %v, want %v", tc.input, got, tc.want)
459-
}
460-
})
461-
}
462-
}
463-
464-
func TestParseProxyStrategies(t *testing.T) {
465-
for desc, tc := range map[string]struct {
466-
input string
467-
want []ProxyStrategy
468-
wantErr error
469-
}{
470-
"empty": {
471-
input: "",
472-
wantErr: fmt.Errorf("proxy strategies cannot be empty"),
473-
},
474-
"unrecognized": {
475-
input: "unrecognized",
476-
wantErr: fmt.Errorf("unknown proxy strategy: unrecognized"),
477-
},
478-
"default": {
479-
input: "default",
480-
want: []ProxyStrategy{ProxyStrategyDefault},
481-
},
482-
"destHost": {
483-
input: "destHost",
484-
want: []ProxyStrategy{ProxyStrategyDestHost},
485-
},
486-
"defaultRoute": {
487-
input: "defaultRoute",
488-
want: []ProxyStrategy{ProxyStrategyDefaultRoute},
489-
},
490-
"duplicate": {
491-
input: "destHost,defaultRoute,defaultRoute,default",
492-
want: []ProxyStrategy{ProxyStrategyDestHost, ProxyStrategyDefaultRoute, ProxyStrategyDefaultRoute, ProxyStrategyDefault},
493-
},
494-
"multiple": {
495-
input: "destHost,defaultRoute,default",
496-
want: []ProxyStrategy{ProxyStrategyDestHost, ProxyStrategyDefaultRoute, ProxyStrategyDefault},
497-
},
498-
} {
499-
t.Run(desc, func(t *testing.T) {
500-
got, err := ParseProxyStrategies(tc.input)
501-
assert.Equal(t, tc.wantErr, err, "ParseProxyStrategies(%s): got error %q, want %v", tc.input, err, tc.wantErr)
502-
if !reflect.DeepEqual(got, tc.want) {
503-
t.Errorf("ParseProxyStrategies(%s): got %v, want %v", tc.input, got, tc.want)
504-
}
505-
})
506-
}
507-
}

‎pkg/server/default_route_backend_manager.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121

2222
"k8s.io/klog/v2"
23+
"sigs.k8s.io/apiserver-network-proxy/pkg/server/proxystrategies"
2324
"sigs.k8s.io/apiserver-network-proxy/proto/header"
2425
)
2526

@@ -32,7 +33,7 @@ var _ BackendManager = &DefaultRouteBackendManager{}
3233
func NewDefaultRouteBackendManager() *DefaultRouteBackendManager {
3334
return &DefaultRouteBackendManager{
3435
DefaultBackendStorage: NewDefaultBackendStorage(
35-
[]header.IdentifierType{header.DefaultRoute})}
36+
[]header.IdentifierType{header.DefaultRoute}, proxystrategies.ProxyStrategyDefaultRoute)}
3637
}
3738

3839
// Backend tries to get a backend that advertises default route, with random selection.

‎pkg/server/desthost_backend_manager.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121

2222
"k8s.io/klog/v2"
23+
"sigs.k8s.io/apiserver-network-proxy/pkg/server/proxystrategies"
2324
"sigs.k8s.io/apiserver-network-proxy/proto/header"
2425
)
2526

@@ -32,7 +33,7 @@ var _ BackendManager = &DestHostBackendManager{}
3233
func NewDestHostBackendManager() *DestHostBackendManager {
3334
return &DestHostBackendManager{
3435
DefaultBackendStorage: NewDefaultBackendStorage(
35-
[]header.IdentifierType{header.IPv4, header.IPv6, header.Host})}
36+
[]header.IdentifierType{header.IPv4, header.IPv6, header.Host}, proxystrategies.ProxyStrategyDestHost)}
3637
}
3738

3839
func (dibm *DestHostBackendManager) AddBackend(backend *Backend) {

0 commit comments

Comments
 (0)