Skip to content

Commit 94d3632

Browse files
idanovindaidanovinda
authored
stream: slot and FES should not be created if the publication creation fails (#2704)
* slot should not be created if the publication creation fails * not create FES resource when slot doesn't exist
1 parent 31f474a commit 94d3632

File tree

3 files changed

+56
-15
lines changed

3 files changed

+56
-15
lines changed

‎e2e/tests/test_e2e.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2041,6 +2041,20 @@ def test_stream_resources(self):
20412041
"recoveryEventType": "test-event-dlq"
20422042
}
20432043
}
2044+
},
2045+
{
2046+
"applicationId": "test-app2",
2047+
"batchSize": 100,
2048+
"database": "foo",
2049+
"enableRecovery": True,
2050+
"tables": {
2051+
"test_non_exist_table": {
2052+
"eventType": "test-event",
2053+
"idColumn": "id",
2054+
"payloadColumn": "payload",
2055+
"recoveryEventType": "test-event-dlq"
2056+
}
2057+
}
20442058
}
20452059
]
20462060
}
@@ -2064,6 +2078,18 @@ def test_stream_resources(self):
20642078
"zalando.org", "v1", "default", "fabriceventstreams", label_selector="cluster-name=acid-minimal-cluster")["items"]), 1,
20652079
"Could not find Fabric Event Stream resource", 10, 5)
20662080

2081+
# check if the non-existing table in the stream section does not create a publication and slot
2082+
get_publication_query_not_exist_table = """
2083+
SELECT * FROM pg_publication WHERE pubname = 'fes_foo_test_app2';
2084+
"""
2085+
get_slot_query_not_exist_table = """
2086+
SELECT * FROM pg_replication_slots WHERE slot_name = 'fes_foo_test_app2';
2087+
"""
2088+
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_publication_query_not_exist_table)), 0,
2089+
"Publication is created for non-existing tables", 10, 5)
2090+
self.eventuallyEqual(lambda: len(self.query_database(leader.metadata.name, "foo", get_slot_query_not_exist_table)), 0,
2091+
"Replication slot is created for non-existing tables", 10, 5)
2092+
20672093
# grant create and ownership of test_table to foo_user, reset search path to default
20682094
grant_permission_foo_user = """
20692095
GRANT CREATE ON DATABASE foo TO foo_user;

‎pkg/cluster/streams.go

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,6 @@ func (c *Cluster) syncPublication(dbName string, databaseSlotsList map[string]za
134134
} else if currentTables != tableList {
135135
alterPublications[slotName] = tableList
136136
}
137-
(*slotsToSync)[slotName] = slotAndPublication.Slot
138137
}
139138

140139
// check if there is any deletion
@@ -148,24 +147,30 @@ func (c *Cluster) syncPublication(dbName string, databaseSlotsList map[string]za
148147
return nil
149148
}
150149

150+
var errorMessage error = nil
151151
for publicationName, tables := range createPublications {
152152
if err = c.executeCreatePublication(publicationName, tables); err != nil {
153-
return fmt.Errorf("creation of publication %q failed: %v", publicationName, err)
153+
errorMessage = fmt.Errorf("creation of publication %q failed: %v", publicationName, err)
154+
continue
154155
}
156+
(*slotsToSync)[publicationName] = databaseSlotsList[publicationName].Slot
155157
}
156158
for publicationName, tables := range alterPublications {
157159
if err = c.executeAlterPublication(publicationName, tables); err != nil {
158-
return fmt.Errorf("update of publication %q failed: %v", publicationName, err)
160+
errorMessage = fmt.Errorf("update of publication %q failed: %v", publicationName, err)
161+
continue
159162
}
163+
(*slotsToSync)[publicationName] = databaseSlotsList[publicationName].Slot
160164
}
161165
for _, publicationName := range deletePublications {
162-
(*slotsToSync)[publicationName] = nil
163166
if err = c.executeDropPublication(publicationName); err != nil {
164-
return fmt.Errorf("deletion of publication %q failed: %v", publicationName, err)
167+
errorMessage = fmt.Errorf("deletion of publication %q failed: %v", publicationName, err)
168+
continue
165169
}
170+
(*slotsToSync)[publicationName] = nil
166171
}
167172

168-
return nil
173+
return errorMessage
169174
}
170175

171176
func (c *Cluster) generateFabricEventStream(appId string) *zalandov1.FabricEventStream {
@@ -390,15 +395,15 @@ func (c *Cluster) syncStreams() error {
390395
}
391396

392397
// finally sync stream CRDs
393-
err = c.createOrUpdateStreams()
398+
err = c.createOrUpdateStreams(slotsToSync)
394399
if err != nil {
395400
return err
396401
}
397402

398403
return nil
399404
}
400405

401-
func (c *Cluster) createOrUpdateStreams() error {
406+
func (c *Cluster) createOrUpdateStreams(createdSlots map[string]map[string]string) error {
402407

403408
// fetch different application IDs from streams section
404409
// there will be a separate event stream resource for each ID
@@ -413,7 +418,7 @@ func (c *Cluster) createOrUpdateStreams() error {
413418
return fmt.Errorf("could not list of FabricEventStreams: %v", err)
414419
}
415420

416-
for _, appId := range appIds {
421+
for idx, appId := range appIds {
417422
streamExists := false
418423

419424
// update stream when it exists and EventStreams array differs
@@ -435,6 +440,12 @@ func (c *Cluster) createOrUpdateStreams() error {
435440
}
436441

437442
if !streamExists {
443+
// check if there is any slot with the applicationId
444+
slotName := getSlotName(c.Spec.Streams[idx].Database, appId)
445+
if _, exists := createdSlots[slotName]; !exists {
446+
c.logger.Warningf("no slot %s with applicationId %s exists, skipping event stream creation", slotName, appId)
447+
continue
448+
}
438449
c.logger.Infof("event streams with applicationId %s do not exist, create it", appId)
439450
streamCRD, err := c.createStreams(appId)
440451
if err != nil {

‎pkg/cluster/streams_test.go

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ var (
4141
fesUser string = fmt.Sprintf("%s%s", constants.EventStreamSourceSlotPrefix, constants.UserRoleNameSuffix)
4242
slotName string = fmt.Sprintf("%s_%s_%s", constants.EventStreamSourceSlotPrefix, dbName, strings.Replace(appId, "-", "_", -1))
4343

44+
fakeCreatedSlots map[string]map[string]string = map[string]map[string]string{
45+
slotName: {},
46+
}
47+
4448
pg = acidv1.Postgresql{
4549
TypeMeta: metav1.TypeMeta{
4650
Kind: "Postgresql",
@@ -222,7 +226,7 @@ func TestGenerateFabricEventStream(t *testing.T) {
222226
assert.NoError(t, err)
223227

224228
// create the streams
225-
err = cluster.createOrUpdateStreams()
229+
err = cluster.createOrUpdateStreams(fakeCreatedSlots)
226230
assert.NoError(t, err)
227231

228232
// compare generated stream with expected stream
@@ -248,7 +252,7 @@ func TestGenerateFabricEventStream(t *testing.T) {
248252
}
249253

250254
// sync streams once again
251-
err = cluster.createOrUpdateStreams()
255+
err = cluster.createOrUpdateStreams(fakeCreatedSlots)
252256
assert.NoError(t, err)
253257

254258
streams, err = cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
@@ -397,7 +401,7 @@ func TestUpdateFabricEventStream(t *testing.T) {
397401
assert.NoError(t, err)
398402

399403
// now create the stream
400-
err = cluster.createOrUpdateStreams()
404+
err = cluster.createOrUpdateStreams(fakeCreatedSlots)
401405
assert.NoError(t, err)
402406

403407
// change specs of streams and patch CRD
@@ -419,7 +423,7 @@ func TestUpdateFabricEventStream(t *testing.T) {
419423
assert.NoError(t, err)
420424

421425
cluster.Postgresql.Spec = pgPatched.Spec
422-
err = cluster.createOrUpdateStreams()
426+
err = cluster.createOrUpdateStreams(fakeCreatedSlots)
423427
assert.NoError(t, err)
424428

425429
// compare stream returned from API with expected stream
@@ -448,7 +452,7 @@ func TestUpdateFabricEventStream(t *testing.T) {
448452
assert.NoError(t, err)
449453

450454
cluster.Postgresql.Spec = pgPatched.Spec
451-
err = cluster.createOrUpdateStreams()
455+
err = cluster.createOrUpdateStreams(fakeCreatedSlots)
452456
assert.NoError(t, err)
453457

454458
result = cluster.generateFabricEventStream(appId)
@@ -466,7 +470,7 @@ func TestUpdateFabricEventStream(t *testing.T) {
466470
assert.NoError(t, err)
467471

468472
cluster.Postgresql.Spec = pgUpdated.Spec
469-
cluster.createOrUpdateStreams()
473+
cluster.createOrUpdateStreams(fakeCreatedSlots)
470474

471475
streamList, err := cluster.KubeClient.FabricEventStreams(namespace).List(context.TODO(), listOptions)
472476
if len(streamList.Items) > 0 || err != nil {

0 commit comments

Comments
 (0)