Skip to content

Commit f63e68f

Browse files
committed
Fix TargetAllocator reconcilation loop and missing RBAC for events.k8s.io
Assisted by Claude Code.
1 parent 9d3ca7b commit f63e68f

File tree

10 files changed

+153
-27
lines changed

10 files changed

+153
-27
lines changed
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: bug_fix
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
5+
component: operator
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: "Fix AnyConfig.DeepCopyInto performing shallow copy, causing TargetAllocator Deployment infinite reconciliation loop"
9+
10+
# One or more tracking issues related to the change
11+
issues: [4950]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext: |
17+
AnyConfig.DeepCopyInto used maps.Copy which only copied top-level map entries, leaving nested
18+
maps as shared references. When ApplyDefaults injected TLS profile settings (min_version) into
19+
the collector's scrape config, it mutated the informer cache through the shared reference. This
20+
caused the TargetAllocator config hash to alternate between two values on every reconciliation,
21+
triggering an infinite Deployment update loop. The fix uses JSON round-tripping for a true deep copy.

.chloggen/fix-events-rbac.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: bug_fix
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
5+
component: operator
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: Add missing RBAC permission for events.k8s.io API group
9+
10+
# One or more tracking issues related to the change
11+
issues: [4950]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext: |
17+
The operator uses k8s.io/client-go/tools/events which targets the events.k8s.io API group,
18+
but the ClusterRole only granted permission for the core API group. This caused "Server rejected
19+
event" errors when recording events on managed resources in other namespaces.

apis/v1beta1/config.go

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,27 @@ type AnyConfig struct {
5151
Object map[string]any `json:"-" yaml:",inline"`
5252
}
5353

54-
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
54+
// DeepCopyInto copies all properties into another AnyConfig instance.
55+
// It performs a true deep copy by JSON round-tripping to ensure nested maps/slices
56+
// are fully independent of the source, preventing shared-reference mutations.
5557
func (c *AnyConfig) DeepCopyInto(out *AnyConfig) {
5658
*out = *c
5759
if c.Object != nil {
58-
in, out := &c.Object, &out.Object
59-
*out = make(map[string]any, len(*in))
60-
maps.Copy((*out), *in)
60+
b, err := json.Marshal(c.Object)
61+
if err != nil {
62+
// Fallback to shallow copy if JSON marshal fails.
63+
in, out := &c.Object, &out.Object
64+
*out = make(map[string]any, len(*in))
65+
maps.Copy((*out), *in)
66+
return
67+
}
68+
out.Object = make(map[string]any)
69+
if err = json.Unmarshal(b, &out.Object); err != nil {
70+
// Fallback to shallow copy if JSON unmarshal fails.
71+
in, out := &c.Object, &out.Object
72+
*out = make(map[string]any, len(*in))
73+
maps.Copy((*out), *in)
74+
}
6175
}
6276
}
6377

apis/v1beta1/config_test.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1480,3 +1480,69 @@ func TestTelemetryIncompleteConfigAppliesDefaults(t *testing.T) {
14801480
require.Equal(t, "0.0.0.0", *telemetry.Metrics.Readers[0].Pull.Exporter.Prometheus.Host)
14811481
require.Equal(t, 8888, *telemetry.Metrics.Readers[0].Pull.Exporter.Prometheus.Port)
14821482
}
1483+
1484+
func TestAnyConfigDeepCopyInto_NestedMapIndependence(t *testing.T) {
1485+
src := AnyConfig{Object: map[string]any{
1486+
"prometheus": map[string]any{
1487+
"config": map[string]any{
1488+
"scrape_configs": []any{
1489+
map[string]any{
1490+
"job_name": "kubelet",
1491+
"tls_config": map[string]any{
1492+
"ca_file": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
1493+
"insecure_skip_verify": true,
1494+
},
1495+
},
1496+
},
1497+
},
1498+
},
1499+
}}
1500+
1501+
dst := src.DeepCopy()
1502+
1503+
// Mutate a nested map in the copy (simulates TLS profile injection).
1504+
scrapeConfigs := dst.Object["prometheus"].(map[string]any)["config"].(map[string]any)["scrape_configs"].([]any)
1505+
tlsConfig := scrapeConfigs[0].(map[string]any)["tls_config"].(map[string]any)
1506+
tlsConfig["min_version"] = "TLS12"
1507+
1508+
// Source nested map must be unaffected.
1509+
srcTLS := src.Object["prometheus"].(map[string]any)["config"].(map[string]any)["scrape_configs"].([]any)[0].(map[string]any)["tls_config"].(map[string]any)
1510+
assert.NotContains(t, srcTLS, "min_version", "DeepCopy must produce independent nested maps; source was mutated through the copy")
1511+
}
1512+
1513+
func TestAnyConfigDeepCopyInto_NilObject(t *testing.T) {
1514+
src := AnyConfig{Object: nil}
1515+
dst := src.DeepCopy()
1516+
assert.Nil(t, dst.Object)
1517+
}
1518+
1519+
func TestAnyConfigDeepCopyInto_EmptyObject(t *testing.T) {
1520+
src := AnyConfig{Object: map[string]any{}}
1521+
dst := src.DeepCopy()
1522+
assert.NotNil(t, dst.Object)
1523+
assert.Empty(t, dst.Object)
1524+
// Mutating dst should not affect src.
1525+
dst.Object["key"] = "value"
1526+
assert.Empty(t, src.Object)
1527+
}
1528+
1529+
func TestAnyConfigDeepCopyInto_PreservesValues(t *testing.T) {
1530+
src := AnyConfig{Object: map[string]any{
1531+
"string_val": "hello",
1532+
"number_val": float64(42),
1533+
"bool_val": true,
1534+
"nested": map[string]any{
1535+
"inner": "value",
1536+
"list": []any{"a", "b"},
1537+
},
1538+
}}
1539+
1540+
dst := src.DeepCopy()
1541+
1542+
assert.Equal(t, "hello", dst.Object["string_val"])
1543+
assert.Equal(t, float64(42), dst.Object["number_val"])
1544+
assert.Equal(t, true, dst.Object["bool_val"])
1545+
nested := dst.Object["nested"].(map[string]any)
1546+
assert.Equal(t, "value", nested["inner"])
1547+
assert.Equal(t, []any{"a", "b"}, nested["list"])
1548+
}

bundle/community/manifests/opentelemetry-operator.clusterserviceversion.yaml

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ metadata:
9999
categories: Logging & Tracing,Monitoring,Observability
100100
certified: "false"
101101
containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator
102-
createdAt: "2026-03-27T16:38:21Z"
102+
createdAt: "2026-04-10T09:34:47Z"
103103
description: Provides the OpenTelemetry components, including the Collector
104104
operators.operatorframework.io/builder: operator-sdk-v1.29.0
105105
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -315,13 +315,6 @@ spec:
315315
- patch
316316
- update
317317
- watch
318-
- apiGroups:
319-
- ""
320-
resources:
321-
- events
322-
verbs:
323-
- create
324-
- patch
325318
- apiGroups:
326319
- ""
327320
resources:
@@ -331,6 +324,14 @@ spec:
331324
- get
332325
- list
333326
- watch
327+
- apiGroups:
328+
- ""
329+
- events.k8s.io
330+
resources:
331+
- events
332+
verbs:
333+
- create
334+
- patch
334335
- apiGroups:
335336
- apps
336337
resources:

bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ metadata:
9999
categories: Logging & Tracing,Monitoring,Observability
100100
certified: "false"
101101
containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator
102-
createdAt: "2026-03-27T16:38:22Z"
102+
createdAt: "2026-04-10T09:34:52Z"
103103
description: Provides the OpenTelemetry components, including the Collector
104104
operators.operatorframework.io/builder: operator-sdk-v1.29.0
105105
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -315,13 +315,6 @@ spec:
315315
- patch
316316
- update
317317
- watch
318-
- apiGroups:
319-
- ""
320-
resources:
321-
- events
322-
verbs:
323-
- create
324-
- patch
325318
- apiGroups:
326319
- ""
327320
resources:
@@ -331,6 +324,14 @@ spec:
331324
- get
332325
- list
333326
- watch
327+
- apiGroups:
328+
- ""
329+
- events.k8s.io
330+
resources:
331+
- events
332+
verbs:
333+
- create
334+
- patch
334335
- apiGroups:
335336
- apps
336337
resources:

config/rbac/role.yaml

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,6 @@ rules:
2323
- patch
2424
- update
2525
- watch
26-
- apiGroups:
27-
- ""
28-
resources:
29-
- events
30-
verbs:
31-
- create
32-
- patch
3326
- apiGroups:
3427
- ""
3528
resources:
@@ -39,6 +32,14 @@ rules:
3932
- get
4033
- list
4134
- watch
35+
- apiGroups:
36+
- ""
37+
- events.k8s.io
38+
resources:
39+
- events
40+
verbs:
41+
- create
42+
- patch
4243
- apiGroups:
4344
- apps
4445
resources:

internal/controllers/clusterobservability_controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ func NewClusterObservabilityReconciler(params ClusterObservabilityReconcilerPara
8282
//+kubebuilder:rbac:groups=opentelemetry.io,resources=instrumentations,verbs=get;list;watch;create;update;patch;delete
8383
//+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch
8484
//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch
85+
//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
8586
//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
8687
//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch
8788
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch

internal/controllers/opentelemetrycollector_controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ func NewReconciler(p Params) *OpenTelemetryCollectorReconciler {
209209

210210
// +kubebuilder:rbac:groups="",resources=pods;configmaps;services;serviceaccounts,verbs=get;list;watch;create;update;patch;delete
211211
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
212+
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
212213
// +kubebuilder:rbac:groups=apps,resources=daemonsets;deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete
213214
// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
214215
// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete

internal/controllers/targetallocator_controller.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ func NewTargetAllocatorReconciler(
137137

138138
// +kubebuilder:rbac:groups="",resources=pods;configmaps;services;serviceaccounts,verbs=get;list;watch;create;update;patch;delete
139139
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
140+
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
140141
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
141142
// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
142143
// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;podmonitors,verbs=get;list;watch;create;update;patch;delete

0 commit comments

Comments
 (0)