Skip to content

Commit 5d55b02

Browse files
bjee19sjberman
bjee19
authored and
sjberman
committed
CP/DP Split: Collect telemetry for cp dp split (#3352)
Collect telemetry for number of data plane pods, control plane pods, and nginx proxy resources attached to a gateway. Problem: With the control data plane split, there are a few telemetry metrics which need to be updated or added. Solution: Update/Add telemetry metrics.
1 parent 7268b22 commit 5d55b02

File tree

7 files changed

+131
-22
lines changed

7 files changed

+131
-22
lines changed

‎internal/mode/static/telemetry/collector.go‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,10 @@ type Data struct {
6060
// then lastly by directive string.
6161
SnippetsFiltersDirectivesCount []int64
6262
NGFResourceCounts // embedding is required by the generator.
63-
// NGFReplicaCount is the number of replicas of the NGF Pod.
64-
NGFReplicaCount int64
63+
// NginxPodCount is the total number of Nginx data plane Pods.
64+
NginxPodCount int64
65+
// ControlPlanePodCount is the total number of NGF control plane Pods.
66+
ControlPlanePodCount int64
6567
}
6668

6769
// NGFResourceCounts stores the counts of all relevant resources that NGF processes and generates configuration from.
@@ -99,6 +101,8 @@ type NGFResourceCounts struct {
99101
SnippetsFilterCount int64
100102
// UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies.
101103
UpstreamSettingsPolicyCount int64
104+
// GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway.
105+
GatewayAttachedNpCount int64
102106
}
103107

104108
// DataCollectorConfig holds configuration parameters for DataCollectorImpl.
@@ -164,6 +168,8 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
164168

165169
snippetsFiltersDirectives, snippetsFiltersDirectivesCount := collectSnippetsFilterDirectives(g)
166170

171+
nginxPodCount := getNginxPodCount(g)
172+
167173
data := Data{
168174
Data: tel.Data{
169175
ProjectName: "NGF",
@@ -179,9 +185,10 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
179185
ImageSource: c.cfg.ImageSource,
180186
FlagNames: c.cfg.Flags.Names,
181187
FlagValues: c.cfg.Flags.Values,
182-
NGFReplicaCount: int64(replicaCount),
183188
SnippetsFiltersDirectives: snippetsFiltersDirectives,
184189
SnippetsFiltersDirectivesCount: snippetsFiltersDirectivesCount,
190+
NginxPodCount: nginxPodCount,
191+
ControlPlanePodCount: int64(replicaCount),
185192
}
186193

187194
return data, nil
@@ -241,6 +248,18 @@ func collectGraphResourceCount(
241248
ngfResourceCounts.NginxProxyCount = int64(len(g.ReferencedNginxProxies))
242249
ngfResourceCounts.SnippetsFilterCount = int64(len(g.SnippetsFilters))
243250

251+
var gatewayAttachedNPCount int64
252+
if g.GatewayClass != nil && g.GatewayClass.NginxProxy != nil {
253+
gatewayClassNP := g.GatewayClass.NginxProxy
254+
for _, np := range g.ReferencedNginxProxies {
255+
if np != gatewayClassNP {
256+
gatewayAttachedNPCount++
257+
}
258+
}
259+
}
260+
261+
ngfResourceCounts.GatewayAttachedNpCount = gatewayAttachedNPCount
262+
244263
return ngfResourceCounts
245264
}
246265

@@ -495,3 +514,22 @@ func parseDirectiveContextMapIntoLists(directiveContextMap map[sfDirectiveContex
495514

496515
return directiveContextList, countList
497516
}
517+
518+
func getNginxPodCount(g *graph.Graph) int64 {
519+
var count int64
520+
for _, gateway := range g.Gateways {
521+
replicas := int64(1)
522+
523+
np := gateway.EffectiveNginxProxy
524+
if np != nil &&
525+
np.Kubernetes != nil &&
526+
np.Kubernetes.Deployment != nil &&
527+
np.Kubernetes.Deployment.Replicas != nil {
528+
replicas = int64(*np.Kubernetes.Deployment.Replicas)
529+
}
530+
531+
count += replicas
532+
}
533+
534+
return count
535+
}

‎internal/mode/static/telemetry/collector_test.go‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ import (
1818
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
1919

2020
ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1"
21+
"github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
22+
"github.com/nginx/nginx-gateway-fabric/internal/framework/helpers"
2123
"github.com/nginx/nginx-gateway-fabric/internal/framework/kinds"
2224
"github.com/nginx/nginx-gateway-fabric/internal/framework/kubernetes/kubernetesfakes"
2325
"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
@@ -170,7 +172,7 @@ var _ = Describe("Collector", Ordered, func() {
170172
ClusterNodeCount: 1,
171173
},
172174
NGFResourceCounts: telemetry.NGFResourceCounts{},
173-
NGFReplicaCount: 1,
175+
ControlPlanePodCount: 1,
174176
ImageSource: "local",
175177
FlagNames: flags.Names,
176178
FlagValues: flags.Values,
@@ -262,6 +264,24 @@ var _ = Describe("Collector", Ordered, func() {
262264

263265
k8sClientReader.ListCalls(createListCallsFunc(nodes))
264266

267+
k8sClientReader.GetCalls(mergeGetCallsWithBase(createGetCallsFunc(
268+
&appsv1.ReplicaSet{
269+
Spec: appsv1.ReplicaSetSpec{
270+
Replicas: helpers.GetPointer(int32(2)),
271+
},
272+
ObjectMeta: metav1.ObjectMeta{
273+
Name: "replica",
274+
OwnerReferences: []metav1.OwnerReference{
275+
{
276+
Kind: "Deployment",
277+
Name: "Deployment1",
278+
UID: "test-uid-replicaSet",
279+
},
280+
},
281+
},
282+
},
283+
)))
284+
265285
secret1 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1"}}
266286
secret2 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret2"}}
267287
nilsecret := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "nilsecret"}}
@@ -270,11 +290,33 @@ var _ = Describe("Collector", Ordered, func() {
270290
svc2 := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc2"}}
271291
nilsvc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nilsvc"}}
272292

293+
gcNP := graph.NginxProxy{
294+
Source: nil,
295+
ErrMsgs: nil,
296+
Valid: false,
297+
}
298+
273299
graph := &graph.Graph{
274-
GatewayClass: &graph.GatewayClass{},
300+
GatewayClass: &graph.GatewayClass{NginxProxy: &gcNP},
275301
Gateways: map[types.NamespacedName]*graph.Gateway{
276-
{Name: "gateway1"}: {},
277-
{Name: "gateway2"}: {},
302+
{Name: "gateway1"}: {
303+
EffectiveNginxProxy: &graph.EffectiveNginxProxy{
304+
Kubernetes: &v1alpha2.KubernetesSpec{
305+
Deployment: &v1alpha2.DeploymentSpec{
306+
Replicas: helpers.GetPointer(int32(1)),
307+
},
308+
},
309+
},
310+
},
311+
{Name: "gateway2"}: {
312+
EffectiveNginxProxy: &graph.EffectiveNginxProxy{
313+
Kubernetes: &v1alpha2.KubernetesSpec{
314+
Deployment: &v1alpha2.DeploymentSpec{
315+
Replicas: helpers.GetPointer(int32(3)),
316+
},
317+
},
318+
},
319+
},
278320
{Name: "gateway3"}: {},
279321
},
280322
IgnoredGatewayClasses: map[types.NamespacedName]*gatewayv1.GatewayClass{
@@ -335,9 +377,11 @@ var _ = Describe("Collector", Ordered, func() {
335377
}: {},
336378
},
337379
ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
338-
{Namespace: "test", Name: "NginxProxy-1"}: {},
339-
{Namespace: "test", Name: "NginxProxy-2"}: {},
340-
}, SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
380+
{Namespace: "test", Name: "NginxProxy-1"}: &gcNP,
381+
{Namespace: "test", Name: "NginxProxy-2"}: {Valid: true},
382+
{Namespace: "test", Name: "NginxProxy-3"}: {Valid: true},
383+
},
384+
SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
341385
{Namespace: "test", Name: "sf-1"}: {
342386
Snippets: map[ngfAPI.NginxContext]string{
343387
ngfAPI.NginxContextMain: "worker_priority 0;",
@@ -432,9 +476,10 @@ var _ = Describe("Collector", Ordered, func() {
432476
GatewayAttachedClientSettingsPolicyCount: 1,
433477
RouteAttachedClientSettingsPolicyCount: 2,
434478
ObservabilityPolicyCount: 1,
435-
NginxProxyCount: 2,
479+
NginxProxyCount: 3,
436480
SnippetsFilterCount: 3,
437481
UpstreamSettingsPolicyCount: 1,
482+
GatewayAttachedNpCount: 2,
438483
}
439484
expData.ClusterVersion = "1.29.2"
440485
expData.ClusterPlatform = "kind"
@@ -462,6 +507,11 @@ var _ = Describe("Collector", Ordered, func() {
462507
1,
463508
}
464509

510+
// one gateway with one replica + one gateway with three replicas + one gateway with replica field
511+
// empty
512+
expData.NginxPodCount = int64(5)
513+
expData.ControlPlanePodCount = int64(2)
514+
465515
data, err := dataCollector.Collect(ctx)
466516
Expect(err).ToNot(HaveOccurred())
467517

@@ -593,7 +643,7 @@ var _ = Describe("Collector", Ordered, func() {
593643
svc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc1"}}
594644

595645
graph1 = &graph.Graph{
596-
GatewayClass: &graph.GatewayClass{},
646+
GatewayClass: &graph.GatewayClass{NginxProxy: &graph.NginxProxy{Valid: true}},
597647
Gateways: map[types.NamespacedName]*graph.Gateway{
598648
{Name: "gateway1"}: {},
599649
},
@@ -634,12 +684,14 @@ var _ = Describe("Collector", Ordered, func() {
634684
}: {},
635685
},
636686
ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
637-
{Namespace: "test", Name: "NginxProxy-1"}: {},
638-
{Namespace: "test", Name: "NginxProxy-2"}: {},
687+
{Namespace: "test", Name: "NginxProxy-1"}: {Valid: true},
639688
},
640689
SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
641690
{Namespace: "test", Name: "sf-1"}: {},
642691
},
692+
BackendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{
693+
{Namespace: "test", Name: "BackendTLSPolicy-1"}: {},
694+
},
643695
}
644696

645697
config1 = []*dataplane.Configuration{
@@ -716,10 +768,13 @@ var _ = Describe("Collector", Ordered, func() {
716768
GatewayAttachedClientSettingsPolicyCount: 1,
717769
RouteAttachedClientSettingsPolicyCount: 1,
718770
ObservabilityPolicyCount: 1,
719-
NginxProxyCount: 2,
771+
NginxProxyCount: 1,
720772
SnippetsFilterCount: 1,
721773
UpstreamSettingsPolicyCount: 1,
774+
GatewayAttachedNpCount: 1,
775+
BackendTLSPolicyCount: 1,
722776
}
777+
expData.NginxPodCount = 1
723778

724779
data, err := dataCollector.Collect(ctx)
725780

‎internal/mode/static/telemetry/data.avdl‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,14 @@ attached at the Gateway level. */
102102
/** UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies. */
103103
long? UpstreamSettingsPolicyCount = null;
104104

105-
/** NGFReplicaCount is the number of replicas of the NGF Pod. */
106-
long? NGFReplicaCount = null;
105+
/** GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway. */
106+
long? GatewayAttachedNpCount = null;
107+
108+
/** NginxPodCount is the total number of Nginx data plane Pods. */
109+
long? NginxPodCount = null;
110+
111+
/** ControlPlanePodCount is the total number of NGF control plane Pods. */
112+
long? ControlPlanePodCount = null;
107113

108114
}
109115
}

‎internal/mode/static/telemetry/data_attributes_generated.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ func (d *Data) Attributes() []attribute.KeyValue {
2020
attrs = append(attrs, attribute.StringSlice("SnippetsFiltersDirectives", d.SnippetsFiltersDirectives))
2121
attrs = append(attrs, attribute.Int64Slice("SnippetsFiltersDirectivesCount", d.SnippetsFiltersDirectivesCount))
2222
attrs = append(attrs, d.NGFResourceCounts.Attributes()...)
23-
attrs = append(attrs, attribute.Int64("NGFReplicaCount", d.NGFReplicaCount))
23+
attrs = append(attrs, attribute.Int64("NginxPodCount", d.NginxPodCount))
24+
attrs = append(attrs, attribute.Int64("ControlPlanePodCount", d.ControlPlanePodCount))
2425

2526
return attrs
2627
}

‎internal/mode/static/telemetry/data_test.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,12 @@ func TestDataAttributes(t *testing.T) {
4040
NginxProxyCount: 12,
4141
SnippetsFilterCount: 13,
4242
UpstreamSettingsPolicyCount: 14,
43+
GatewayAttachedNpCount: 15,
4344
},
44-
NGFReplicaCount: 3,
4545
SnippetsFiltersDirectives: []string{"main-three-count", "http-two-count", "server-one-count"},
4646
SnippetsFiltersDirectivesCount: []int64{3, 2, 1},
47+
NginxPodCount: 3,
48+
ControlPlanePodCount: 3,
4749
}
4850

4951
expected := []attribute.KeyValue{
@@ -79,7 +81,9 @@ func TestDataAttributes(t *testing.T) {
7981
attribute.Int64("NginxProxyCount", 12),
8082
attribute.Int64("SnippetsFilterCount", 13),
8183
attribute.Int64("UpstreamSettingsPolicyCount", 14),
82-
attribute.Int64("NGFReplicaCount", 3),
84+
attribute.Int64("GatewayAttachedNpCount", 15),
85+
attribute.Int64("NginxPodCount", 3),
86+
attribute.Int64("ControlPlanePodCount", 3),
8387
}
8488

8589
result := data.Attributes()
@@ -122,7 +126,9 @@ func TestDataAttributesWithEmptyData(t *testing.T) {
122126
attribute.Int64("NginxProxyCount", 0),
123127
attribute.Int64("SnippetsFilterCount", 0),
124128
attribute.Int64("UpstreamSettingsPolicyCount", 0),
125-
attribute.Int64("NGFReplicaCount", 0),
129+
attribute.Int64("GatewayAttachedNpCount", 0),
130+
attribute.Int64("NginxPodCount", 0),
131+
attribute.Int64("ControlPlanePodCount", 0),
126132
}
127133

128134
result := data.Attributes()

‎internal/mode/static/telemetry/ngfresourcecounts_attributes_generated.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ func (d *NGFResourceCounts) Attributes() []attribute.KeyValue {
2727
attrs = append(attrs, attribute.Int64("NginxProxyCount", d.NginxProxyCount))
2828
attrs = append(attrs, attribute.Int64("SnippetsFilterCount", d.SnippetsFilterCount))
2929
attrs = append(attrs, attribute.Int64("UpstreamSettingsPolicyCount", d.UpstreamSettingsPolicyCount))
30+
attrs = append(attrs, attribute.Int64("GatewayAttachedNpCount", d.GatewayAttachedNpCount))
3031

3132
return attrs
3233
}

‎tests/suite/telemetry_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,9 @@ var _ = Describe("Telemetry test with OTel collector", Label("telemetry"), func(
9292
"NginxProxyCount: Int(1)",
9393
"SnippetsFilterCount: Int(0)",
9494
"UpstreamSettingsPolicyCount: Int(0)",
95-
"NGFReplicaCount: Int(1)",
95+
"GatewayAttachedNpCount: Int(0)",
96+
"NginxPodCount: Int(0)",
97+
"ControlPlanePodCount: Int(1)",
9698
},
9799
)
98100
})

0 commit comments

Comments
 (0)