Skip to content

Commit 320fd17

Browse files
kasiakoziolkasiakoziol
committed
CSPL-4360 Fix failing integ and helm tests
1 parent 155b21a commit 320fd17

File tree

16 files changed

+143
-134
lines changed

16 files changed

+143
-134
lines changed

‎api/v4/objectstorage_types.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ type S3Spec struct {
5555

5656
// ObjectStorageStatus defines the observed state of ObjectStorage.
5757
type ObjectStorageStatus struct {
58-
// Phase of the large message store
58+
// Phase of the object storage
5959
Phase Phase `json:"phase"`
6060

6161
// Resource revision tracker

‎config/crd/bases/enterprise.splunk.com_objectstorages.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ spec:
8787
description: Auxillary message describing CR status
8888
type: string
8989
phase:
90-
description: Phase of the large message store
90+
description: Phase of the object storage
9191
enum:
9292
- Pending
9393
- Ready

‎docs/CustomResources.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -404,21 +404,21 @@ spec:
404404
endpoint: https://s3.us-west-2.amazonaws.com
405405
```
406406

407-
ObjectStorage inputs can be found in the table below. As of now, only S3 provider of large message store is supported.
407+
ObjectStorage inputs can be found in the table below. As of now, only S3 provider of object storage is supported.
408408

409409
| Key | Type | Description |
410410
| ---------- | ------- | ------------------------------------------------- |
411-
| provider | string | [Required] Provider of large message store (Allowed values: s3) |
412-
| s3 | S3 | [Required if provider=s3] S3 large message store inputs |
411+
| provider | string | [Required] Provider of object storage (Allowed values: s3) |
412+
| s3 | S3 | [Required if provider=s3] S3 object storage inputs |
413413

414-
S3 large message store inputs can be found in the table below.
414+
S3 object storage inputs can be found in the table below.
415415

416416
| Key | Type | Description |
417417
| ---------- | ------- | ------------------------------------------------- |
418418
| path | string | [Required] Remote storage location for messages that are larger than the underlying maximum message size |
419419
| endpoint | string | [Optional, if not provided formed based on region] S3-compatible service endpoint
420420

421-
Change of any of the large message queue inputs triggers the restart of Splunk so that appropriate .conf files are correctly refreshed and consumed.
421+
Change of any of the object storage inputs triggers the restart of Splunk so that appropriate .conf files are correctly refreshed and consumed.
422422

423423
## MonitoringConsole Resource Spec Parameters
424424

‎docs/IndexIngestionSeparation.md‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ SQS message queue inputs can be found in the table below.
4444
| endpoint | string | [Optional, if not provided formed based on region] AWS SQS Service endpoint
4545
| dlq | string | [Required] Name of the dead letter queue |
4646

47-
**First provisioning or update of any of the bus inputs requires Ingestor Cluster and Indexer Cluster Splunkd restart, but this restart is implemented automatically and done by SOK.**
47+
**First provisioning or update of any of the queue inputs requires Ingestor Cluster and Indexer Cluster Splunkd restart, but this restart is implemented automatically and done by SOK.**
4848

4949
## Example
5050
```
@@ -67,21 +67,21 @@ ObjectStorage is introduced to store large message (messages that exceed the siz
6767

6868
## Spec
6969

70-
ObjectStorage inputs can be found in the table below. As of now, only S3 provider of large message store is supported.
70+
ObjectStorage inputs can be found in the table below. As of now, only S3 provider of object storage is supported.
7171

7272
| Key | Type | Description |
7373
| ---------- | ------- | ------------------------------------------------- |
74-
| provider | string | [Required] Provider of large message store (Allowed values: s3) |
75-
| s3 | S3 | [Required if provider=s3] S3 large message store inputs |
74+
| provider | string | [Required] Provider of object storage (Allowed values: s3) |
75+
| s3 | S3 | [Required if provider=s3] S3 object storage inputs |
7676

77-
S3 large message store inputs can be found in the table below.
77+
S3 object storage inputs can be found in the table below.
7878

7979
| Key | Type | Description |
8080
| ---------- | ------- | ------------------------------------------------- |
8181
| path | string | [Required] Remote storage location for messages that are larger than the underlying maximum message size |
8282
| endpoint | string | [Optional, if not provided formed based on region] S3-compatible service endpoint
8383

84-
Change of any of the large message queue inputs triggers the restart of Splunk so that appropriate .conf files are correctly refreshed and consumed.
84+
Change of any of the object storage inputs triggers the restart of Splunk so that appropriate .conf files are correctly refreshed and consumed.
8585

8686
## Example
8787
```
@@ -108,13 +108,13 @@ In addition to common spec inputs, the IngestorCluster resource provides the fol
108108
| ---------- | ------- | ------------------------------------------------- |
109109
| replicas | integer | The number of replicas (defaults to 3) |
110110
| queueRef | corev1.ObjectReference | Message queue reference |
111-
| objectStorageRef | corev1.ObjectReference | Large message store reference |
111+
| objectStorageRef | corev1.ObjectReference | Object storage reference |
112112

113113
## Example
114114

115115
The example presented below configures IngestorCluster named ingestor with Splunk ${SPLUNK_IMAGE_VERSION} image that resides in a default namespace and is scaled to 3 replicas that serve the ingestion traffic. This IngestorCluster custom resource is set up with the service account named ingestor-sa allowing it to perform SQS and S3 operations. Queue and ObjectStorage references allow the user to specify queue and bucket settings for the ingestion process.
116116

117-
In this case, the setup uses the SQS and S3 based configuration where the messages are stored in sqs-test queue in us-west-2 region with dead letter queue set to sqs-dlq-test queue. The large message store is set to ingestion bucket in smartbus-test directory. Based on these inputs, default-mode.conf and outputs.conf files are configured accordingly.
117+
In this case, the setup uses the SQS and S3 based configuration where the messages are stored in sqs-test queue in us-west-2 region with dead letter queue set to sqs-dlq-test queue. The object storage is set to ingestion bucket in smartbus-test directory. Based on these inputs, default-mode.conf and outputs.conf files are configured accordingly.
118118

119119
```
120120
apiVersion: enterprise.splunk.com/v4
@@ -145,13 +145,13 @@ In addition to common spec inputs, the IndexerCluster resource provides the foll
145145
| ---------- | ------- | ------------------------------------------------- |
146146
| replicas | integer | The number of replicas (defaults to 3) |
147147
| queueRef | corev1.ObjectReference | Message queue reference |
148-
| objectStorageRef | corev1.ObjectReference | Large message store reference |
148+
| objectStorageRef | corev1.ObjectReference | Object storage reference |
149149

150150
## Example
151151

152152
The example presented below configures IndexerCluster named indexer with Splunk ${SPLUNK_IMAGE_VERSION} image that resides in a default namespace and is scaled to 3 replicas that serve the indexing traffic. This IndexerCluster custom resource is set up with the service account named ingestor-sa allowing it to perform SQS and S3 operations. Queue and ObjectStorage references allow the user to specify queue and bucket settings for the indexing process.
153153

154-
In this case, the setup uses the SQS and S3 based configuration where the messages are stored in and retrieved from sqs-test queue in us-west-2 region with dead letter queue set to sqs-dlq-test queue. The large message store is set to ingestion bucket in smartbus-test directory. Based on these inputs, default-mode.conf, inputs.conf and outputs.conf files are configured accordingly.
154+
In this case, the setup uses the SQS and S3 based configuration where the messages are stored in and retrieved from sqs-test queue in us-west-2 region with dead letter queue set to sqs-dlq-test queue. The object storage is set to ingestion bucket in smartbus-test directory. Based on these inputs, default-mode.conf, inputs.conf and outputs.conf files are configured accordingly.
155155

156156
```
157157
apiVersion: enterprise.splunk.com/v4
@@ -717,7 +717,7 @@ Spec:
717717
Name: queue
718718
Namespace: default
719719
Image: splunk/splunk:${SPLUNK_IMAGE_VERSION}
720-
Large Message Store Ref:
720+
Object Storage Ref:
721721
Name: os
722722
Namespace: default
723723
Replicas: 3
@@ -741,7 +741,7 @@ Status:
741741
Endpoint: https://sqs.us-west-2.amazonaws.com
742742
Name: sqs-test
743743
Provider: sqs
744-
Large Message Store:
744+
Object Storage:
745745
S3:
746746
Endpoint: https://s3.us-west-2.amazonaws.com
747747
Path: s3://ingestion/smartbus-test

‎helm-chart/splunk-enterprise/templates/enterprise_v4_indexercluster.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,8 @@ items:
170170
namespace: {{ .namespace }}
171171
{{- end }}
172172
{{- end }}
173-
{{- with $.Values.indexerCluster.objectStoreRef }}
174-
objectStoreRef:
173+
{{- with $.Values.indexerCluster.objectStorageRef }}
174+
objectStorageRef:
175175
name: {{ .name }}
176176
{{- if .namespace }}
177177
namespace: {{ .namespace }}

‎helm-chart/splunk-enterprise/templates/enterprise_v4_queues.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ spec:
2626
{{- if .name }}
2727
name: {{ .name | quote }}
2828
{{- end }}
29-
{{- if .region }}
30-
region: {{ .region | quote }}
29+
{{- if .authRegion }}
30+
authRegion: {{ .authRegion | quote }}
3131
{{- end }}
3232
{{- if .volumes }}
3333
volumes:

‎kuttl/tests/helm/index-and-ingest-separation/02-assert.yaml‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,30 @@
11
---
2-
# assert for bus custom resource to be ready
2+
# assert for queue custom resource to be ready
33
apiVersion: enterprise.splunk.com/v4
4-
kind: Bus
4+
kind: Queue
55
metadata:
6-
name: bus
6+
name: queue
77
spec:
88
provider: sqs
99
sqs:
10-
name: sqs-test
11-
region: us-west-2
10+
name: index-ingest-separation-test-q
11+
authRegion: us-west-2
1212
endpoint: https://sqs.us-west-2.amazonaws.com
13-
dlq: sqs-dlq-test
13+
dlq: index-ingest-separation-test-dlq
1414
status:
1515
phase: Ready
1616

1717
---
18-
# assert for large message store custom resource to be ready
18+
# assert for object storage custom resource to be ready
1919
apiVersion: enterprise.splunk.com/v4
20-
kind: LargeMessageStore
20+
kind: ObjectStorage
2121
metadata:
22-
name: lms
22+
name: os
2323
spec:
2424
provider: s3
2525
s3:
2626
endpoint: https://s3.us-west-2.amazonaws.com
27-
path: s3://ingestion/smartbus-test
27+
path: s3://index-ingest-separation-test-bucket/smartbus-test
2828
status:
2929
phase: Ready
3030

@@ -61,24 +61,24 @@ metadata:
6161
name: indexer
6262
spec:
6363
replicas: 3
64-
busRef:
65-
name: bus
66-
largeMessageStoreRef:
67-
name: lms
64+
queueRef:
65+
name: queue
66+
objectStorageRef:
67+
name: os
6868
status:
6969
phase: Ready
70-
bus:
70+
queue:
7171
provider: sqs
7272
sqs:
73-
name: sqs-test
74-
region: us-west-2
73+
name: index-ingest-separation-test-q
74+
authRegion: us-west-2
7575
endpoint: https://sqs.us-west-2.amazonaws.com
76-
dlq: sqs-dlq-test
77-
largeMessageStore:
76+
dlq: index-ingest-separation-test-dlq
77+
objectStorage:
7878
provider: s3
7979
s3:
8080
endpoint: https://s3.us-west-2.amazonaws.com
81-
path: s3://ingestion/smartbus-test
81+
path: s3://index-ingest-separation-test-bucket/smartbus-test
8282

8383
---
8484
# check for stateful set and replicas as configured
@@ -103,7 +103,7 @@ kind: IngestorCluster
103103
metadata:
104104
name: ingestor
105105
spec:
106-
replicas: 4
106+
replicas: 3
107107
queueRef:
108108
name: queue
109109
objectStorageRef:
@@ -113,15 +113,15 @@ status:
113113
queue:
114114
provider: sqs
115115
sqs:
116-
name: sqs-test
117-
region: us-west-2
116+
name: index-ingest-separation-test-q
117+
authRegion: us-west-2
118118
endpoint: https://sqs.us-west-2.amazonaws.com
119-
dlq: sqs-dlq-test
119+
dlq: index-ingest-separation-test-dlq
120120
objectStorage:
121121
provider: s3
122122
s3:
123123
endpoint: https://s3.us-west-2.amazonaws.com
124-
path: s3://ingestion/smartbus-test
124+
path: s3://index-ingest-separation-test-bucket/smartbus-test
125125

126126
---
127127
# check for stateful set and replicas as configured

‎kuttl/tests/helm/index-and-ingest-separation/03-assert.yaml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,24 @@ metadata:
66
name: ingestor
77
spec:
88
replicas: 4
9-
busRef:
10-
name: bus
11-
largeMessageStoreRef:
12-
name: lms
9+
queueRef:
10+
name: queue
11+
objectStorageRef:
12+
name: os
1313
status:
1414
phase: Ready
15-
bus:
15+
queue:
1616
provider: sqs
1717
sqs:
18-
name: sqs-test
19-
region: us-west-2
18+
name: index-ingest-separation-test-q
19+
authRegion: us-west-2
2020
endpoint: https://sqs.us-west-2.amazonaws.com
21-
dlq: sqs-dlq-test
22-
largeMessageStore:
21+
dlq: index-ingest-separation-test-dlq
22+
objectStorage:
2323
provider: s3
2424
s3:
2525
endpoint: https://s3.us-west-2.amazonaws.com
26-
path: s3://ingestion/smartbus-test
26+
path: s3://index-ingest-separation-test-bucket/smartbus-test
2727

2828
---
2929
# check for stateful sets and replicas updated

‎kuttl/tests/helm/index-and-ingest-separation/splunk_index_ingest_sep.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ queue:
1010
name: queue
1111
provider: sqs
1212
sqs:
13-
name: sqs-test
14-
region: us-west-2
13+
name: index-ingest-separation-test-q
14+
authRegion: us-west-2
1515
endpoint: https://sqs.us-west-2.amazonaws.com
16-
dlq: sqs-dlq-test
16+
dlq: index-ingest-separation-test-dlq
1717
volumes:
1818
- name: helm-bus-secret-ref-test
1919
secretRef: s3-secret
@@ -24,7 +24,7 @@ objectStorage:
2424
provider: s3
2525
s3:
2626
endpoint: https://s3.us-west-2.amazonaws.com
27-
path: s3://ingestion/smartbus-test
27+
path: s3://index-ingest-separation-test-bucket/smartbus-test
2828

2929
ingestorCluster:
3030
enabled: true

‎pkg/splunk/enterprise/indexercluster.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller
7676

7777
// updates status after function completes
7878
cr.Status.ClusterManagerPhase = enterpriseApi.PhaseError
79+
if cr.Status.Replicas < cr.Spec.Replicas {
80+
cr.Status.Queue = nil
81+
}
7982
cr.Status.Replicas = cr.Spec.Replicas
8083
cr.Status.Selector = fmt.Sprintf("app.kubernetes.io/instance=splunk-%s-indexer", cr.GetName())
8184
if cr.Status.Peers == nil {
@@ -265,7 +268,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller
265268
}
266269
}
267270

268-
// Large Message Store
271+
// Object Storage
269272
os := enterpriseApi.ObjectStorage{}
270273
if cr.Spec.ObjectStorageRef.Name != "" {
271274
ns := cr.GetNamespace()
@@ -281,15 +284,15 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller
281284
}
282285
}
283286

284-
// Can not override original large message store spec due to comparison in the later code
287+
// Can not override original object storage spec due to comparison in the later code
285288
osCopy := os
286289
if osCopy.Spec.Provider == "s3" {
287290
if osCopy.Spec.S3.Endpoint == "" && queueCopy.Spec.SQS.AuthRegion != "" {
288291
osCopy.Spec.S3.Endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com", queueCopy.Spec.SQS.AuthRegion)
289292
}
290293
}
291294

292-
// If bus is updated
295+
// If queue is updated
293296
if cr.Spec.QueueRef.Name != "" {
294297
if cr.Status.Queue == nil || cr.Status.ObjectStorage == nil || !reflect.DeepEqual(*cr.Status.Queue, queue.Spec) || !reflect.DeepEqual(*cr.Status.ObjectStorage, os.Spec) {
295298
mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client)
@@ -402,6 +405,9 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient,
402405
// updates status after function completes
403406
cr.Status.Phase = enterpriseApi.PhaseError
404407
cr.Status.ClusterMasterPhase = enterpriseApi.PhaseError
408+
if cr.Status.Replicas < cr.Spec.Replicas {
409+
cr.Status.Queue = nil
410+
}
405411
cr.Status.Replicas = cr.Spec.Replicas
406412
cr.Status.Selector = fmt.Sprintf("app.kubernetes.io/instance=splunk-%s-indexer", cr.GetName())
407413
if cr.Status.Peers == nil {
@@ -594,7 +600,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient,
594600
}
595601
}
596602

597-
// Large Message Store
603+
// Object Storage
598604
os := enterpriseApi.ObjectStorage{}
599605
if cr.Spec.ObjectStorageRef.Name != "" {
600606
ns := cr.GetNamespace()
@@ -618,7 +624,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient,
618624
}
619625
}
620626

621-
// If bus is updated
627+
// If queue is updated
622628
if cr.Spec.QueueRef.Name != "" {
623629
if cr.Status.Queue == nil || cr.Status.ObjectStorage == nil || !reflect.DeepEqual(*cr.Status.Queue, queue.Spec) || !reflect.DeepEqual(*cr.Status.ObjectStorage, os.Spec) {
624630
mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client)

0 commit comments

Comments
 (0)