Skip to content

Commit bb8fe7d

Browse files
Fix infinite reconcile loop in SetDeprecationStatus
Address review feedback from Per: change "remove all, then add" pattern to "if adding then set, else remove" to preserve lastTransitionTime when status unchanged. Prevents infinite reconciliation loops. Add tests verifying multiple reconciles stabilize correctly.
1 parent c9df78c commit bb8fe7d

File tree

2 files changed

+219
-7
lines changed

2 files changed

+219
-7
lines changed

internal/operator-controller/controllers/clusterextension_controller.go

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -217,12 +217,11 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, installedBundleName string
217217
channelMessages := collectDeprecationMessages(info.ChannelEntries)
218218
bundleMessages := collectDeprecationMessages(info.BundleEntries)
219219

220-
// Clear all deprecation conditions first, then only add the ones we need.
220+
// Strategy: Only remove conditions when we're NOT going to re-add them.
221+
// If we're setting a condition, call SetStatusCondition directly - it preserves
222+
// lastTransitionTime when status/reason/message haven't changed, preventing
223+
// infinite reconciliation loops.
221224
// Absence of a deprecation condition means "not deprecated" - keeps output clean.
222-
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypeDeprecated)
223-
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypePackageDeprecated)
224-
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypeChannelDeprecated)
225-
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypeBundleDeprecated)
226225

227226
if !hasCatalogData {
228227
// When catalog is unavailable, set all to Unknown.
@@ -264,8 +263,8 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, installedBundleName string
264263
return
265264
}
266265

267-
// Only add conditions when there's something to report (True or Unknown states).
268-
// False (not deprecated) is represented by absence of the condition.
266+
// Handle catalog data available: set conditions to True when deprecated,
267+
// or remove them when not deprecated (absence = not deprecated).
269268
messages := slices.Concat(packageMessages, channelMessages, bundleMessages)
270269
if len(messages) > 0 {
271270
SetStatusCondition(&ext.Status.Conditions, metav1.Condition{
@@ -275,6 +274,9 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, installedBundleName string
275274
Message: strings.Join(messages, "\n"),
276275
ObservedGeneration: ext.GetGeneration(),
277276
})
277+
} else {
278+
// Only remove if we're not setting it - prevents unnecessary lastTransitionTime updates
279+
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypeDeprecated)
278280
}
279281

280282
if len(packageMessages) > 0 {
@@ -285,6 +287,8 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, installedBundleName string
285287
Message: strings.Join(packageMessages, "\n"),
286288
ObservedGeneration: ext.GetGeneration(),
287289
})
290+
} else {
291+
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypePackageDeprecated)
288292
}
289293

290294
if len(channelMessages) > 0 {
@@ -295,6 +299,8 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, installedBundleName string
295299
Message: strings.Join(channelMessages, "\n"),
296300
ObservedGeneration: ext.GetGeneration(),
297301
})
302+
} else {
303+
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypeChannelDeprecated)
298304
}
299305

300306
// BundleDeprecated: Unknown when no bundle installed, True when deprecated, absent otherwise
@@ -314,6 +320,8 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, installedBundleName string
314320
Message: strings.Join(bundleMessages, "\n"),
315321
ObservedGeneration: ext.GetGeneration(),
316322
})
323+
} else {
324+
apimeta.RemoveStatusCondition(&ext.Status.Conditions, ocv1.TypeBundleDeprecated)
317325
}
318326
}
319327

internal/operator-controller/controllers/clusterextension_controller_test.go

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"helm.sh/helm/v3/pkg/chart"
1616
"helm.sh/helm/v3/pkg/release"
1717
"helm.sh/helm/v3/pkg/storage/driver"
18+
"k8s.io/apimachinery/pkg/api/equality"
1819
apimeta "k8s.io/apimachinery/pkg/api/meta"
1920
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2021
"k8s.io/apimachinery/pkg/types"
@@ -1960,6 +1961,209 @@ func TestSetDeprecationStatus(t *testing.T) {
19601961
}
19611962
}
19621963

1964+
// TestSetDeprecationStatus_NoInfiniteReconcileLoop verifies that calling SetDeprecationStatus
1965+
// multiple times with the same inputs does not cause infinite reconciliation loops.
1966+
//
1967+
// The issue: If we always remove and re-add conditions, lastTransitionTime updates every time,
1968+
// which causes DeepEqual to fail, triggering another reconcile indefinitely.
1969+
//
1970+
// The fix: Only remove conditions when we're NOT re-adding them. When setting a condition,
1971+
// call SetStatusCondition directly - it preserves lastTransitionTime when status/reason/message
1972+
// haven't changed.
1973+
func TestSetDeprecationStatus_NoInfiniteReconcileLoop(t *testing.T) {
1974+
tests := []struct {
1975+
name string
1976+
installedBundleName string
1977+
deprecation *declcfg.Deprecation
1978+
hasCatalogData bool
1979+
setupConditions func(*ocv1.ClusterExtension)
1980+
expectConditionsCount int
1981+
description string
1982+
}{
1983+
{
1984+
name: "deprecated package - should stabilize after first reconcile",
1985+
installedBundleName: "test.v1.0.0",
1986+
deprecation: &declcfg.Deprecation{
1987+
Entries: []declcfg.DeprecationEntry{
1988+
{
1989+
Reference: declcfg.PackageScopedReference{
1990+
Schema: declcfg.SchemaPackage,
1991+
},
1992+
Message: "package is deprecated",
1993+
},
1994+
},
1995+
},
1996+
hasCatalogData: true,
1997+
setupConditions: func(ext *ocv1.ClusterExtension) {
1998+
// No conditions initially
1999+
},
2000+
expectConditionsCount: 2, // Deprecated and PackageDeprecated
2001+
description: "First call adds conditions, second call preserves lastTransitionTime",
2002+
},
2003+
{
2004+
name: "not deprecated - migration from False to absent",
2005+
installedBundleName: "", // No bundle installed
2006+
deprecation: nil,
2007+
hasCatalogData: true,
2008+
setupConditions: func(ext *ocv1.ClusterExtension) {
2009+
// Simulate old behavior: False conditions present
2010+
apimeta.SetStatusCondition(&ext.Status.Conditions, metav1.Condition{
2011+
Type: ocv1.TypeDeprecated,
2012+
Status: metav1.ConditionFalse,
2013+
Reason: ocv1.ReasonDeprecated,
2014+
Message: "",
2015+
ObservedGeneration: 1,
2016+
})
2017+
apimeta.SetStatusCondition(&ext.Status.Conditions, metav1.Condition{
2018+
Type: ocv1.TypePackageDeprecated,
2019+
Status: metav1.ConditionFalse,
2020+
Reason: ocv1.ReasonDeprecated,
2021+
Message: "",
2022+
ObservedGeneration: 1,
2023+
})
2024+
},
2025+
expectConditionsCount: 1, // Only BundleDeprecated Unknown (no bundle installed)
2026+
description: "Migrates from False to absent, then stabilizes",
2027+
},
2028+
{
2029+
name: "catalog unavailable - should stabilize with Unknown conditions",
2030+
installedBundleName: "test.v1.0.0",
2031+
deprecation: nil,
2032+
hasCatalogData: false,
2033+
setupConditions: func(ext *ocv1.ClusterExtension) {
2034+
// No conditions initially
2035+
},
2036+
expectConditionsCount: 4, // All four Unknown conditions
2037+
description: "Sets Unknown conditions, then preserves them",
2038+
},
2039+
}
2040+
2041+
for _, tt := range tests {
2042+
t.Run(tt.name, func(t *testing.T) {
2043+
ext := &ocv1.ClusterExtension{
2044+
ObjectMeta: metav1.ObjectMeta{
2045+
Generation: 1,
2046+
},
2047+
Status: ocv1.ClusterExtensionStatus{
2048+
Conditions: []metav1.Condition{},
2049+
},
2050+
}
2051+
2052+
// Setup initial conditions if specified
2053+
if tt.setupConditions != nil {
2054+
tt.setupConditions(ext)
2055+
}
2056+
2057+
// First reconcile: should add/update conditions
2058+
controllers.SetDeprecationStatus(ext, tt.installedBundleName, tt.deprecation, tt.hasCatalogData)
2059+
2060+
firstReconcileConditions := make([]metav1.Condition, len(ext.Status.Conditions))
2061+
copy(firstReconcileConditions, ext.Status.Conditions)
2062+
2063+
// Verify expected number of conditions
2064+
deprecationConditions := filterDeprecationConditions(ext.Status.Conditions)
2065+
require.Len(t, deprecationConditions, tt.expectConditionsCount,
2066+
"First reconcile should have %d deprecation conditions", tt.expectConditionsCount)
2067+
2068+
// Second reconcile: should preserve lastTransitionTime (no changes)
2069+
controllers.SetDeprecationStatus(ext, tt.installedBundleName, tt.deprecation, tt.hasCatalogData)
2070+
2071+
secondReconcileConditions := ext.Status.Conditions
2072+
2073+
// Verify conditions are identical (including lastTransitionTime)
2074+
require.Len(t, secondReconcileConditions, len(firstReconcileConditions),
2075+
"Number of conditions should remain the same")
2076+
2077+
for i, firstCond := range firstReconcileConditions {
2078+
secondCond := secondReconcileConditions[i]
2079+
require.Equal(t, firstCond.Type, secondCond.Type, "Condition type should match")
2080+
require.Equal(t, firstCond.Status, secondCond.Status, "Condition status should match")
2081+
require.Equal(t, firstCond.Reason, secondCond.Reason, "Condition reason should match")
2082+
require.Equal(t, firstCond.Message, secondCond.Message, "Condition message should match")
2083+
2084+
// This is the critical check: lastTransitionTime should NOT change
2085+
require.Equal(t, firstCond.LastTransitionTime, secondCond.LastTransitionTime,
2086+
"lastTransitionTime should be preserved (prevents infinite reconcile loop)")
2087+
}
2088+
2089+
// Third reconcile: verify it remains stable
2090+
controllers.SetDeprecationStatus(ext, tt.installedBundleName, tt.deprecation, tt.hasCatalogData)
2091+
2092+
thirdReconcileConditions := ext.Status.Conditions
2093+
require.Len(t, thirdReconcileConditions, len(secondReconcileConditions),
2094+
"Conditions should remain stable after multiple reconciles")
2095+
2096+
for i, secondCond := range secondReconcileConditions {
2097+
thirdCond := thirdReconcileConditions[i]
2098+
require.Equal(t, secondCond.LastTransitionTime, thirdCond.LastTransitionTime,
2099+
"lastTransitionTime should remain stable across reconciles")
2100+
}
2101+
})
2102+
}
2103+
}
2104+
2105+
// TestSetDeprecationStatus_StatusChangesOnlyWhenNeeded verifies that calling SetDeprecationStatus
2106+
// only modifies the status when actual deprecation state changes, not on every reconcile.
2107+
func TestSetDeprecationStatus_StatusChangesOnlyWhenNeeded(t *testing.T) {
2108+
ext := &ocv1.ClusterExtension{
2109+
ObjectMeta: metav1.ObjectMeta{
2110+
Generation: 1,
2111+
},
2112+
Status: ocv1.ClusterExtensionStatus{
2113+
Conditions: []metav1.Condition{},
2114+
},
2115+
}
2116+
2117+
// Scenario 1: Package becomes deprecated
2118+
deprecation := &declcfg.Deprecation{
2119+
Entries: []declcfg.DeprecationEntry{
2120+
{
2121+
Reference: declcfg.PackageScopedReference{Schema: declcfg.SchemaPackage},
2122+
Message: "package is deprecated",
2123+
},
2124+
},
2125+
}
2126+
2127+
// First reconcile: add deprecation condition
2128+
controllers.SetDeprecationStatus(ext, "test.v1.0.0", deprecation, true)
2129+
statusAfterFirstReconcile := ext.Status.DeepCopy()
2130+
2131+
// Second reconcile: same deprecation state
2132+
controllers.SetDeprecationStatus(ext, "test.v1.0.0", deprecation, true)
2133+
statusAfterSecondReconcile := ext.Status.DeepCopy()
2134+
2135+
// Status should be semantically equal (DeepEqual would return true)
2136+
require.True(t, equality.Semantic.DeepEqual(statusAfterFirstReconcile, statusAfterSecondReconcile),
2137+
"Status should not change when deprecation state is unchanged")
2138+
2139+
// Scenario 2: Deprecation is resolved (package no longer deprecated)
2140+
controllers.SetDeprecationStatus(ext, "test.v1.0.0", nil, true)
2141+
statusAfterResolution := ext.Status.DeepCopy()
2142+
2143+
// Status should have changed (conditions removed)
2144+
require.False(t, equality.Semantic.DeepEqual(statusAfterSecondReconcile, statusAfterResolution),
2145+
"Status should change when deprecation is resolved")
2146+
2147+
// Scenario 3: Verify resolution is stable
2148+
controllers.SetDeprecationStatus(ext, "test.v1.0.0", nil, true)
2149+
statusAfterFourthReconcile := ext.Status.DeepCopy()
2150+
2151+
require.True(t, equality.Semantic.DeepEqual(statusAfterResolution, statusAfterFourthReconcile),
2152+
"Status should remain stable after deprecation is resolved")
2153+
}
2154+
2155+
// filterDeprecationConditions returns only the deprecation-related conditions
2156+
func filterDeprecationConditions(conditions []metav1.Condition) []metav1.Condition {
2157+
var result []metav1.Condition
2158+
for _, cond := range conditions {
2159+
switch cond.Type {
2160+
case ocv1.TypeDeprecated, ocv1.TypePackageDeprecated, ocv1.TypeChannelDeprecated, ocv1.TypeBundleDeprecated:
2161+
result = append(result, cond)
2162+
}
2163+
}
2164+
return result
2165+
}
2166+
19632167
type MockActionGetter struct {
19642168
description string
19652169
rels []*release.Release

0 commit comments

Comments
 (0)