Skip to content

Commit 607632f

Browse files
kasiakoziolkasiakoziol
committed
CSPL-4358 Making region authRegion and optional, simplifying endpoint
1 parent b6f5b0b commit 607632f

14 files changed

+119
-122
lines changed

‎api/v4/objectstorage_types.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ type ObjectStorageSpec struct {
4343

4444
type S3Spec struct {
4545
// +optional
46-
// +kubebuilder:validation:Pattern=`^https://s3(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$`
46+
// +kubebuilder:validation:Pattern=`^https?://[^\s/$.?#].[^\s]*$`
4747
// S3-compatible Service endpoint
4848
Endpoint string `json:"endpoint"`
4949

‎api/v4/queue_types.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,18 @@ type SQSSpec struct {
4747
// Name of the queue
4848
Name string `json:"name"`
4949

50-
// +kubebuilder:validation:Required
50+
// +optional
5151
// +kubebuilder:validation:Pattern=`^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$`
52-
// Region of the resources
53-
Region string `json:"region"`
52+
// Auth Region of the resources
53+
AuthRegion string `json:"authRegion"`
5454

5555
// +kubebuilder:validation:Required
5656
// +kubebuilder:validation:MinLength=1
5757
// Name of the dead letter queue resource
5858
DLQ string `json:"dlq"`
5959

6060
// +optional
61-
// +kubebuilder:validation:Pattern=`^https://sqs(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$`
61+
// +kubebuilder:validation:Pattern=`^https?://[^\s/$.?#].[^\s]*$`
6262
// Amazon SQS Service endpoint
6363
Endpoint string `json:"endpoint"`
6464
}

‎config/crd/bases/enterprise.splunk.com_indexerclusters.yaml‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8396,7 +8396,7 @@ spec:
83968396
properties:
83978397
endpoint:
83988398
description: S3-compatible Service endpoint
8399-
pattern: ^https://s3(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$
8399+
pattern: ^https?://[^\s/$.?#].[^\s]*$
84008400
type: string
84018401
path:
84028402
description: S3 bucket path
@@ -8464,26 +8464,25 @@ spec:
84648464
sqs:
84658465
description: sqs specific inputs
84668466
properties:
8467+
authRegion:
8468+
description: Auth Region of the resources
8469+
pattern: ^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$
8470+
type: string
84678471
dlq:
84688472
description: Name of the dead letter queue resource
84698473
minLength: 1
84708474
type: string
84718475
endpoint:
84728476
description: Amazon SQS Service endpoint
8473-
pattern: ^https://sqs(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$
8477+
pattern: ^https?://[^\s/$.?#].[^\s]*$
84748478
type: string
84758479
name:
84768480
description: Name of the queue
84778481
minLength: 1
84788482
type: string
8479-
region:
8480-
description: Region of the resources
8481-
pattern: ^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$
8482-
type: string
84838483
required:
84848484
- dlq
84858485
- name
8486-
- region
84878486
type: object
84888487
required:
84898488
- provider

‎config/crd/bases/enterprise.splunk.com_ingestorclusters.yaml‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4607,7 +4607,7 @@ spec:
46074607
properties:
46084608
endpoint:
46094609
description: S3-compatible Service endpoint
4610-
pattern: ^https://s3(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$
4610+
pattern: ^https?://[^\s/$.?#].[^\s]*$
46114611
type: string
46124612
path:
46134613
description: S3 bucket path
@@ -4645,26 +4645,25 @@ spec:
46454645
sqs:
46464646
description: sqs specific inputs
46474647
properties:
4648+
authRegion:
4649+
description: Auth Region of the resources
4650+
pattern: ^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$
4651+
type: string
46484652
dlq:
46494653
description: Name of the dead letter queue resource
46504654
minLength: 1
46514655
type: string
46524656
endpoint:
46534657
description: Amazon SQS Service endpoint
4654-
pattern: ^https://sqs(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$
4658+
pattern: ^https?://[^\s/$.?#].[^\s]*$
46554659
type: string
46564660
name:
46574661
description: Name of the queue
46584662
minLength: 1
46594663
type: string
4660-
region:
4661-
description: Region of the resources
4662-
pattern: ^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$
4663-
type: string
46644664
required:
46654665
- dlq
46664666
- name
4667-
- region
46684667
type: object
46694668
required:
46704669
- provider

‎config/crd/bases/enterprise.splunk.com_objectstorages.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ spec:
6464
properties:
6565
endpoint:
6666
description: S3-compatible Service endpoint
67-
pattern: ^https://s3(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$
67+
pattern: ^https?://[^\s/$.?#].[^\s]*$
6868
type: string
6969
path:
7070
description: S3 bucket path

‎config/crd/bases/enterprise.splunk.com_queues.yaml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,26 +62,25 @@ spec:
6262
sqs:
6363
description: sqs specific inputs
6464
properties:
65+
authRegion:
66+
description: Auth Region of the resources
67+
pattern: ^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$
68+
type: string
6569
dlq:
6670
description: Name of the dead letter queue resource
6771
minLength: 1
6872
type: string
6973
endpoint:
7074
description: Amazon SQS Service endpoint
71-
pattern: ^https://sqs(?:-fips)?\.[a-z]+-[a-z]+(?:-[a-z]+)?-\d+\.amazonaws\.com(?:\.cn)?(?:/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)?$
75+
pattern: ^https?://[^\s/$.?#].[^\s]*$
7276
type: string
7377
name:
7478
description: Name of the queue
7579
minLength: 1
7680
type: string
77-
region:
78-
description: Region of the resources
79-
pattern: ^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$
80-
type: string
8181
required:
8282
- dlq
8383
- name
84-
- region
8584
type: object
8685
required:
8786
- provider

‎internal/controller/ingestorcluster_controller_test.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,10 @@ var _ = Describe("IngestorCluster Controller", func() {
7979
Spec: enterpriseApi.QueueSpec{
8080
Provider: "sqs",
8181
SQS: enterpriseApi.SQSSpec{
82-
Name: "smartbus-queue",
83-
Region: "us-west-2",
84-
DLQ: "smartbus-dlq",
85-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
82+
Name: "smartbus-queue",
83+
AuthRegion: "us-west-2",
84+
DLQ: "smartbus-dlq",
85+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
8686
},
8787
},
8888
}
@@ -127,10 +127,10 @@ var _ = Describe("IngestorCluster Controller", func() {
127127
Spec: enterpriseApi.QueueSpec{
128128
Provider: "sqs",
129129
SQS: enterpriseApi.SQSSpec{
130-
Name: "smartbus-queue",
131-
Region: "us-west-2",
132-
DLQ: "smartbus-dlq",
133-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
130+
Name: "smartbus-queue",
131+
AuthRegion: "us-west-2",
132+
DLQ: "smartbus-dlq",
133+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
134134
},
135135
},
136136
}

‎internal/controller/queue_controller_test.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,10 @@ var _ = Describe("Queue Controller", func() {
7373
spec := enterpriseApi.QueueSpec{
7474
Provider: "sqs",
7575
SQS: enterpriseApi.SQSSpec{
76-
Name: "smartbus-queue",
77-
Region: "us-west-2",
78-
DLQ: "smartbus-dlq",
79-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
76+
Name: "smartbus-queue",
77+
AuthRegion: "us-west-2",
78+
DLQ: "smartbus-dlq",
79+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
8080
},
8181
}
8282
CreateQueue("test", nsSpecs.Name, annotations, enterpriseApi.PhaseReady, spec)
@@ -102,10 +102,10 @@ var _ = Describe("Queue Controller", func() {
102102
spec := enterpriseApi.QueueSpec{
103103
Provider: "sqs",
104104
SQS: enterpriseApi.SQSSpec{
105-
Name: "smartbus-queue",
106-
Region: "us-west-2",
107-
DLQ: "smartbus-dlq",
108-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
105+
Name: "smartbus-queue",
106+
AuthRegion: "us-west-2",
107+
DLQ: "smartbus-dlq",
108+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
109109
},
110110
}
111111
CreateQueue("test", nsSpecs.Name, annotations, enterpriseApi.PhaseReady, spec)
@@ -141,10 +141,10 @@ var _ = Describe("Queue Controller", func() {
141141
spec := enterpriseApi.QueueSpec{
142142
Provider: "sqs",
143143
SQS: enterpriseApi.SQSSpec{
144-
Name: "smartbus-queue",
145-
Region: "us-west-2",
146-
DLQ: "smartbus-dlq",
147-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
144+
Name: "smartbus-queue",
145+
AuthRegion: "us-west-2",
146+
DLQ: "smartbus-dlq",
147+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
148148
},
149149
}
150150
bcSpec := testutils.NewQueue("test", namespace, spec)

‎pkg/splunk/enterprise/indexercluster.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller
263263
// Can not override original queue spec due to comparison in the later code
264264
queueCopy := queue
265265
if queueCopy.Spec.Provider == "sqs" {
266-
if queueCopy.Spec.SQS.Endpoint == "" {
267-
queueCopy.Spec.SQS.Endpoint = fmt.Sprintf("https://sqs.%s.amazonaws.com", queueCopy.Spec.SQS.Region)
266+
if queueCopy.Spec.SQS.Endpoint == "" && queueCopy.Spec.SQS.AuthRegion != "" {
267+
queueCopy.Spec.SQS.Endpoint = fmt.Sprintf("https://sqs.%s.amazonaws.com", queueCopy.Spec.SQS.AuthRegion)
268268
}
269269
}
270270

@@ -287,8 +287,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller
287287
// Can not override original large message store spec due to comparison in the later code
288288
osCopy := os
289289
if osCopy.Spec.Provider == "s3" {
290-
if osCopy.Spec.S3.Endpoint == "" {
291-
osCopy.Spec.S3.Endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com", queueCopy.Spec.SQS.Region)
290+
if osCopy.Spec.S3.Endpoint == "" && queueCopy.Spec.SQS.AuthRegion != "" {
291+
osCopy.Spec.S3.Endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com", queueCopy.Spec.SQS.AuthRegion)
292292
}
293293
}
294294

@@ -586,8 +586,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient,
586586
// Can not override original queue spec due to comparison in the later code
587587
queueCopy := queue
588588
if queueCopy.Spec.Provider == "sqs" {
589-
if queueCopy.Spec.SQS.Endpoint == "" {
590-
queueCopy.Spec.SQS.Endpoint = fmt.Sprintf("https://sqs.%s.amazonaws.com", queueCopy.Spec.SQS.Region)
589+
if queueCopy.Spec.SQS.Endpoint == "" && queueCopy.Spec.SQS.AuthRegion != "" {
590+
queueCopy.Spec.SQS.Endpoint = fmt.Sprintf("https://sqs.%s.amazonaws.com", queueCopy.Spec.SQS.AuthRegion)
591591
}
592592
}
593593

@@ -610,8 +610,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient,
610610
// Can not override original queue spec due to comparison in the later code
611611
osCopy := os
612612
if osCopy.Spec.Provider == "s3" {
613-
if osCopy.Spec.S3.Endpoint == "" {
614-
osCopy.Spec.S3.Endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com", queueCopy.Spec.SQS.Region)
613+
if osCopy.Spec.S3.Endpoint == "" && queueCopy.Spec.SQS.AuthRegion != "" {
614+
osCopy.Spec.S3.Endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com", queueCopy.Spec.SQS.AuthRegion)
615615
}
616616
}
617617

@@ -1391,21 +1391,21 @@ func pullQueueChanged(oldQueue, newQueue *enterpriseApi.QueueSpec, oldOS, newOS
13911391
if newQueue.Provider == "sqs" {
13921392
queueProvider = "sqs_smartbus"
13931393
}
1394-
osProvider := ""
1394+
osProvider := ""
13951395
if newOS.Provider == "s3" {
13961396
osProvider = "sqs_smartbus"
13971397
}
13981398

13991399
if oldQueue.Provider != newQueue.Provider || afterDelete {
14001400
inputs = append(inputs, []string{"remote_queue.type", queueProvider})
14011401
}
1402-
if oldQueue.SQS.Region != newQueue.SQS.Region || afterDelete {
1403-
inputs = append(inputs, []string{fmt.Sprintf("remote_queue.%s.auth_region", queueProvider), newQueue.SQS.Region})
1402+
if newQueue.SQS.AuthRegion != "" &&(oldQueue.SQS.AuthRegion != newQueue.SQS.AuthRegion || afterDelete) {
1403+
inputs = append(inputs, []string{fmt.Sprintf("remote_queue.%s.auth_region", queueProvider), newQueue.SQS.AuthRegion})
14041404
}
1405-
if oldQueue.SQS.Endpoint != newQueue.SQS.Endpoint || afterDelete {
1405+
if newQueue.SQS.Endpoint != "" && (oldQueue.SQS.Endpoint != newQueue.SQS.Endpoint || afterDelete) {
14061406
inputs = append(inputs, []string{fmt.Sprintf("remote_queue.%s.endpoint", queueProvider), newQueue.SQS.Endpoint})
14071407
}
1408-
if oldOS.S3.Endpoint != newOS.S3.Endpoint || afterDelete {
1408+
if newOS.S3.Endpoint != "" && (oldOS.S3.Endpoint != newOS.S3.Endpoint || afterDelete) {
14091409
inputs = append(inputs, []string{fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", osProvider), newOS.S3.Endpoint})
14101410
}
14111411
if oldOS.S3.Path != newOS.S3.Path || afterDelete {

‎pkg/splunk/enterprise/indexercluster_test.go‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,10 +1355,10 @@ func TestGetIndexerStatefulSet(t *testing.T) {
13551355
Spec: enterpriseApi.QueueSpec{
13561356
Provider: "sqs",
13571357
SQS: enterpriseApi.SQSSpec{
1358-
Name: "test-queue",
1359-
Region: "us-west-2",
1360-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
1361-
DLQ: "sqs-dlq-test",
1358+
Name: "test-queue",
1359+
AuthRegion: "us-west-2",
1360+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
1361+
DLQ: "sqs-dlq-test",
13621362
},
13631363
},
13641364
}
@@ -2059,10 +2059,10 @@ func TestGetChangedQueueFieldsForIndexer(t *testing.T) {
20592059
Spec: enterpriseApi.QueueSpec{
20602060
Provider: "sqs",
20612061
SQS: enterpriseApi.SQSSpec{
2062-
Name: "test-queue",
2063-
Region: "us-west-2",
2064-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
2065-
DLQ: "sqs-dlq-test",
2062+
Name: "test-queue",
2063+
AuthRegion: "us-west-2",
2064+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
2065+
DLQ: "sqs-dlq-test",
20662066
},
20672067
},
20682068
}
@@ -2099,7 +2099,7 @@ func TestGetChangedQueueFieldsForIndexer(t *testing.T) {
20992099
assert.Equal(t, 8, len(queueChangedFieldsInputs))
21002100
assert.Equal(t, [][]string{
21012101
{"remote_queue.type", provider},
2102-
{fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.Region},
2102+
{fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion},
21032103
{fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint},
21042104
{fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint},
21052105
{fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), os.Spec.S3.Path},
@@ -2111,7 +2111,7 @@ func TestGetChangedQueueFieldsForIndexer(t *testing.T) {
21112111
assert.Equal(t, 10, len(queueChangedFieldsOutputs))
21122112
assert.Equal(t, [][]string{
21132113
{"remote_queue.type", provider},
2114-
{fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.Region},
2114+
{fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion},
21152115
{fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint},
21162116
{fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint},
21172117
{fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), os.Spec.S3.Path},
@@ -2148,10 +2148,10 @@ func TestHandlePullQueueChange(t *testing.T) {
21482148
Spec: enterpriseApi.QueueSpec{
21492149
Provider: "sqs",
21502150
SQS: enterpriseApi.SQSSpec{
2151-
Name: "test-queue",
2152-
Region: "us-west-2",
2153-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
2154-
DLQ: "sqs-dlq-test",
2151+
Name: "test-queue",
2152+
AuthRegion: "us-west-2",
2153+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
2154+
DLQ: "sqs-dlq-test",
21552155
},
21562156
},
21572157
}
@@ -2192,8 +2192,8 @@ func TestHandlePullQueueChange(t *testing.T) {
21922192
},
21932193
},
21942194
Status: enterpriseApi.IndexerClusterStatus{
2195-
ReadyReplicas: 3,
2196-
Queue: &enterpriseApi.QueueSpec{},
2195+
ReadyReplicas: 3,
2196+
Queue: &enterpriseApi.QueueSpec{},
21972197
ObjectStorage: &enterpriseApi.ObjectStorageSpec{},
21982198
},
21992199
}
@@ -2276,7 +2276,7 @@ func TestHandlePullQueueChange(t *testing.T) {
22762276

22772277
// outputs.conf
22782278
propertyKVList := [][]string{
2279-
{fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.Region},
2279+
{fmt.Sprintf("remote_queue.%s.auth_region", provider), queue.Spec.SQS.AuthRegion},
22802280
{fmt.Sprintf("remote_queue.%s.endpoint", provider), queue.Spec.SQS.Endpoint},
22812281
{fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", provider), os.Spec.S3.Endpoint},
22822282
{fmt.Sprintf("remote_queue.%s.large_message_store.path", provider), os.Spec.S3.Path},
@@ -2407,10 +2407,10 @@ func TestApplyIndexerClusterManager_Queue_Success(t *testing.T) {
24072407
Spec: enterpriseApi.QueueSpec{
24082408
Provider: "sqs",
24092409
SQS: enterpriseApi.SQSSpec{
2410-
Name: "test-queue",
2411-
Region: "us-west-2",
2412-
Endpoint: "https://sqs.us-west-2.amazonaws.com",
2413-
DLQ: "sqs-dlq-test",
2410+
Name: "test-queue",
2411+
AuthRegion: "us-west-2",
2412+
Endpoint: "https://sqs.us-west-2.amazonaws.com",
2413+
DLQ: "sqs-dlq-test",
24142414
},
24152415
},
24162416
}

0 commit comments

Comments
 (0)