Skip to content

Commit a6db608

Browse files
CSPL-4239: Stabilize MC Error State (#1631)
* stabilize MC errors by re-fetching statefulset * update golang.org/xcrypto * stabilize unit test cases * stabilize MC checks after other CRs are Ready
1 parent 3314a75 commit a6db608

File tree

10 files changed

+54
-25
lines changed

10 files changed

+54
-25
lines changed

go.mod

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -126,16 +126,16 @@ require (
126126
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
127127
go.uber.org/automaxprocs v1.6.0 // indirect
128128
go.uber.org/multierr v1.11.0 // indirect
129-
golang.org/x/crypto v0.43.0 // indirect
129+
golang.org/x/crypto v0.45.0 // indirect
130130
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
131-
golang.org/x/net v0.45.0 // indirect
131+
golang.org/x/net v0.47.0 // indirect
132132
golang.org/x/oauth2 v0.27.0 // indirect
133-
golang.org/x/sync v0.17.0 // indirect
134-
golang.org/x/sys v0.37.0 // indirect
135-
golang.org/x/term v0.36.0 // indirect
136-
golang.org/x/text v0.30.0 // indirect
133+
golang.org/x/sync v0.18.0 // indirect
134+
golang.org/x/sys v0.38.0 // indirect
135+
golang.org/x/term v0.37.0 // indirect
136+
golang.org/x/text v0.31.0 // indirect
137137
golang.org/x/time v0.6.0 // indirect
138-
golang.org/x/tools v0.37.0 // indirect
138+
golang.org/x/tools v0.38.0 // indirect
139139
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
140140
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
141141
google.golang.org/appengine v1.6.7 // indirect

go.sum

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,8 @@ golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
343343
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
344344
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
345345
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
346+
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
347+
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
346348
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
347349
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
348350
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
@@ -372,6 +374,8 @@ golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
372374
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
373375
golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
374376
golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
377+
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
378+
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
375379
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
376380
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
377381
golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
@@ -389,6 +393,8 @@ golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
389393
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
390394
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
391395
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
396+
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
397+
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
392398
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
393399
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
394400
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -407,13 +413,17 @@ golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
407413
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
408414
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
409415
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
416+
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
417+
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
410418
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
411419
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
412420
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
413421
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
414422
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
415423
golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
416424
golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
425+
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
426+
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
417427
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
418428
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
419429
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -424,6 +434,8 @@ golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
424434
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
425435
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
426436
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
437+
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
438+
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
427439
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
428440
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
429441
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -440,6 +452,8 @@ golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
440452
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
441453
golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
442454
golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
455+
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
456+
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
443457
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
444458
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
445459
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/splunk/enterprise/clustermanager_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) {
574574
{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"},
575575
{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"},
576576
{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"},
577+
{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"},
577578
{MetaName: "*v1.Pod-test-splunk-stack1-cluster-manager-0"},
578579
{MetaName: "*v1.StatefulSet-test-splunk-test-monitoring-console"},
579580
{MetaName: "*v4.ClusterManager-test-stack1"},

pkg/splunk/enterprise/clustermaster_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) {
263263
{MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"},
264264
{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
265265
{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
266+
{MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"},
266267
{MetaName: "*v1.Pod-test-splunk-stack1-cluster-master-0"},
267268
{MetaName: "*v1.StatefulSet-test-splunk-test-monitoring-console"},
268269
{MetaName: "*v3.ClusterMaster-test-stack1"},

pkg/splunk/enterprise/indexercluster_test.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -769,7 +769,7 @@ func TestIndexerClusterPodManager(t *testing.T) {
769769
listmockCall := []spltest.MockFuncCall{
770770
{ListOpts: listOpts}}
771771

772-
wantCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}}
772+
wantCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}}
773773

774774
// test 1 ready pod
775775
mockHandlers := []spltest.MockHTTPHandler{
@@ -822,6 +822,7 @@ func TestIndexerClusterPodManager(t *testing.T) {
822822
{MetaName: "*v1.Secret-test-splunk-test-secret"},
823823
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
824824
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
825+
{MetaName: "*v1.StatefulSet-test-splunk-stack1"},
825826
{MetaName: "*v1.Pod-test-splunk-stack1-0"},
826827
{MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"},
827828
{MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"},
@@ -836,6 +837,7 @@ func TestIndexerClusterPodManager(t *testing.T) {
836837
{MetaName: "*v1.Secret-test-splunk-test-secret"},
837838
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
838839
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
840+
{MetaName: "*v1.StatefulSet-test-splunk-stack1"},
839841
{MetaName: "*v1.Pod-test-splunk-stack1-0"},
840842
}
841843
mockHandlers = []spltest.MockHTTPHandler{mockHandlers[0], mockHandlers[1]}
@@ -862,7 +864,7 @@ func TestIndexerClusterPodManager(t *testing.T) {
862864
statefulSet.Status.Replicas = 2
863865
statefulSet.Status.ReadyReplicas = 2
864866
statefulSet.Status.UpdatedReplicas = 2
865-
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4]}, "Create": {funcCalls[1]}}
867+
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[0]}, "Create": {funcCalls[1]}}
866868
method = "indexerClusterPodManager.Update(Pod Not Found)"
867869
indexerClusterPodManagerUpdateTester(t, method, mockHandlers, 1, enterpriseApi.PhaseScalingDown, statefulSet, wantCalls, nil, statefulSet, pod)
868870

@@ -885,6 +887,7 @@ func TestIndexerClusterPodManager(t *testing.T) {
885887
{MetaName: "*v1.Secret-test-splunk-test-secret"},
886888
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
887889
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
890+
{MetaName: "*v1.StatefulSet-test-splunk-stack1"},
888891
{MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"},
889892
{MetaName: "*v1.PersistentVolumeClaim-test-pvc-etc-splunk-stack1-1"},
890893
{MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"},

pkg/splunk/enterprise/searchheadcluster_test.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) {
317317
},
318318
}
319319
method = "searchHeadClusterPodManager.Update(All pods ready)"
320-
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}}
320+
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}}
321321
searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseReady, statefulSet, wantCalls, nil, statefulSet, pod)
322322

323323
// test pod needs update => transition to detention
@@ -337,21 +337,21 @@ func TestSearchHeadClusterPodManager(t *testing.T) {
337337
)
338338
pod.ObjectMeta.Labels["controller-revision-hash"] = "v0"
339339
method = "searchHeadClusterPodManager.Update(Quarantine Pod)"
340-
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5], funcCalls[2], funcCalls[2]}, "Create": {funcCalls[1]}}
340+
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5], funcCalls[2], funcCalls[2]}, "Create": {funcCalls[1]}}
341341
searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod)
342342

343343
// test pod needs update => wait for searches to drain
344344
mockHandlers = []spltest.MockHTTPHandler{mockHandlers[0], mockHandlers[1]}
345345
mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"status":"Up"`, `"status":"ManualDetention"`, 1)
346346
mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"active_historical_search_count":0`, `"active_historical_search_count":1`, 1)
347347
method = "searchHeadClusterPodManager.Update(Draining Searches)"
348-
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}}
348+
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}}
349349
searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod)
350350

351351
// test pod needs update => delete pod
352352
mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"active_historical_search_count":1`, `"active_historical_search_count":0`, 1)
353353
method = "searchHeadClusterPodManager.Update(Delete Pod)"
354-
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "Delete": {funcCalls[5]}}
354+
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}, "Delete": {funcCalls[5]}}
355355
searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod)
356356

357357
// test pod update finished => release from detention
@@ -364,7 +364,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) {
364364
Body: ``,
365365
})
366366
method = "searchHeadClusterPodManager.Update(Release Quarantine)"
367-
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5], funcCalls[2]}, "Create": {funcCalls[1]}}
367+
wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5], funcCalls[2]}, "Create": {funcCalls[1]}}
368368
searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod)
369369

370370
// test scale down => remove member
@@ -394,6 +394,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) {
394394
{MetaName: "*v1.Pod-test-splunk-stack1-search-head-0"},
395395
{MetaName: "*v1.Pod-test-splunk-stack1-search-head-0"},
396396
{MetaName: "*v1.Pod-test-splunk-stack1-search-head-1"},
397+
{MetaName: "*v1.StatefulSet-test-splunk-stack1"},
397398
{MetaName: "*v1.Pod-test-splunk-stack1-search-head-1"},
398399
{MetaName: "*v1.PersistentVolumeClaim-test-pvc-etc-splunk-stack1-1"},
399400
{MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"},

pkg/splunk/splkcontroller/statefulset.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,17 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st
118118
"name", statefulSet.GetObjectMeta().GetName(),
119119
"namespace", statefulSet.GetObjectMeta().GetNamespace())
120120

121+
// Re-fetch the StatefulSet to ensure we have the latest status, especially UpdateRevision.
122+
// This addresses a race condition where the StatefulSet controller may not have updated
123+
// Status.UpdateRevision yet after a spec change was applied. Without this re-fetch,
124+
// we might incorrectly report PhaseReady when pods actually need to be recycled.
125+
namespacedName := types.NamespacedName{Namespace: statefulSet.GetNamespace(), Name: statefulSet.GetName()}
126+
err := c.Get(ctx, namespacedName, statefulSet)
127+
if err != nil {
128+
scopedLog.Error(err, "Unable to re-fetch StatefulSet for latest status")
129+
return enterpriseApi.PhaseError, err
130+
}
131+
121132
// wait for all replicas ready
122133
replicas := *statefulSet.Spec.Replicas
123134
readyReplicas := statefulSet.Status.ReadyReplicas
@@ -249,7 +260,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st
249260
}
250261

251262
// Remove unwanted owner references
252-
err := splutil.RemoveUnwantedSecrets(ctx, c, statefulSet.GetName(), statefulSet.GetNamespace())
263+
err = splutil.RemoveUnwantedSecrets(ctx, c, statefulSet.GetName(), statefulSet.GetNamespace())
253264
if err != nil {
254265
return enterpriseApi.PhaseReady, err
255266
}

pkg/splunk/splkcontroller/statefulset_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,8 @@ func TestUpdateStatefulSetPods(t *testing.T) {
246246
},
247247
}
248248
statefulSet.Status.ReadyReplicas = 3
249+
// Add statefulSet to mock client so UpdateStatefulSetPods can re-fetch it
250+
c.AddObject(statefulSet)
249251
c.InduceErrorKind[splcommon.MockClientInduceErrorUpdate] = rerr
250252
_, err = UpdateStatefulSetPods(ctx, c, statefulSet, &mgr, 1)
251253
if err == nil {

pkg/splunk/test/controller.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM
794794
// test scale up (zero ready so far; wait for ready)
795795
revised = current.DeepCopy()
796796
current.Status.ReadyReplicas = 0
797-
scaleUpCalls := map[string][]MockFuncCall{"Get": {funcCalls[0]}}
797+
scaleUpCalls := map[string][]MockFuncCall{"Get": {funcCalls[0], funcCalls[0]}}
798798
methodPlus = fmt.Sprintf("%s(%s)", method, "ScalingUp, 0 ready")
799799
PodManagerUpdateTester(t, methodPlus, mgr, 1, enterpriseApi.PhasePending, revised, scaleUpCalls, nil, current)
800800

@@ -809,7 +809,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM
809809
replicas = 1
810810
current.Status.Replicas = 1
811811
current.Status.ReadyReplicas = 1
812-
updateCalls = map[string][]MockFuncCall{"Get": {funcCalls[0]}, "Update": {funcCalls[0]}}
812+
updateCalls = map[string][]MockFuncCall{"Get": {funcCalls[0], funcCalls[0]}, "Update": {funcCalls[0]}}
813813
methodPlus = fmt.Sprintf("%s(%s)", method, "ScalingUp, Update Replicas 1=>2")
814814
PodManagerUpdateTester(t, methodPlus, mgr, 2, enterpriseApi.PhaseScalingUp, revised, updateCalls, nil, current, pod)
815815

@@ -827,7 +827,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM
827827
{MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"},
828828
}
829829
scaleDownCalls := map[string][]MockFuncCall{
830-
"Get": {funcCalls[0], pvcCalls[0], pvcCalls[1]},
830+
"Get": {funcCalls[0], funcCalls[0], pvcCalls[0], pvcCalls[1]},
831831
"Update": {funcCalls[0]},
832832
"Delete": pvcCalls,
833833
}
@@ -845,7 +845,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM
845845
replicas = 1
846846
current.Status.Replicas = 1
847847
current.Status.ReadyReplicas = 1
848-
podCalls := []MockFuncCall{funcCalls[0], {MetaName: "*v1.Pod-test-splunk-stack1-0"}}
848+
podCalls := []MockFuncCall{funcCalls[0], funcCalls[0], {MetaName: "*v1.Pod-test-splunk-stack1-0"}}
849849
getPodCalls := map[string][]MockFuncCall{"Get": podCalls}
850850
//getPodCalls := map[string][]MockFuncCall{}
851851
methodPlus = fmt.Sprintf("%s(%s)", method, "Pod not found")

test/appframework_aws/c3/manager_appframework_test.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -363,9 +363,6 @@ var _ = Describe("c3appfw test", func() {
363363
// Ensure Cluster Manager goes to Ready phase
364364
testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst)
365365

366-
// Ensure Monitoring Console goes to Ready phase
367-
testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst)
368-
369366
// // Ensure Search Head Cluster go to Ready phase
370367
testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst)
371368

@@ -439,15 +436,14 @@ var _ = Describe("c3appfw test", func() {
439436
// Wait for License Manager to be in READY phase
440437
testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst)
441438

442-
// // Verify Monitoring Console is ready and stays in ready state
443-
testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst)
444-
445439
// // Ensure Search Head Cluster go to Ready phase
446440
testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst)
447441

448442
// // Ensure Indexers go to Ready phase
449443
testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst)
450444

445+
// // Verify Monitoring Console is ready and stays in ready state
446+
testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst)
451447
})
452448
})
453449

0 commit comments

Comments
 (0)