Skip to content

Commit 28c6b5d

Browse files
committed
fix: improve yaml load for sinks with active_backfill and max_retry_count set to nil
As handling `active_backfill` seems to be entirely unsupported we are ignoring that field for now until we have a proper implementation
1 parent a8c62d6 commit 28c6b5d

File tree

2 files changed

+70
-1
lines changed

2 files changed

+70
-1
lines changed

lib/sequin/transforms/transforms.ex

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,9 @@ defmodule Sequin.Transforms do
597597
"max_retry_count" when is_integer(value) and value >= 0 ->
598598
{:cont, {:ok, Map.put(acc, :max_retry_count, value)}}
599599

600+
"max_retry_count" when is_nil(value) ->
601+
{:cont, {:ok, acc}}
602+
600603
"max_retry_count" ->
601604
{:halt, {:error, Error.validation(summary: "max_retry_count must be a non-negative integer")}}
602605

@@ -612,7 +615,7 @@ defmodule Sequin.Transforms do
612615
{:cont, {:ok, acc}}
613616

614617
# Ignore internal fields that might be present in the external data
615-
ignored when ignored in ~w(id inserted_at updated_at account_id replication_slot_id sequence_id) ->
618+
ignored when ignored in ~w(id inserted_at updated_at account_id replication_slot_id sequence_id active_backfill) ->
616619
{:cont, {:ok, acc}}
617620

618621
# Unknown field

test/sequin_web/controllers/yaml_controller_test.exs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,72 @@ defmodule SequinWeb.YamlControllerTest do
108108
"summary" => "Error creating database 'test-db': - database: can't be blank"
109109
}
110110
end
111+
112+
test "successfully plans configuration with wider set of fields", %{conn: conn} do
113+
yaml = """
114+
change_retentions:
115+
- name: test_retention
116+
filters: []
117+
destination_database: sequin_test_two
118+
source_database: sequin_test_two
119+
actions:
120+
- insert
121+
- update
122+
- delete
123+
source_table_name: accounts
124+
source_table_schema: public
125+
destination_table_name: sequin_events
126+
destination_table_schema: public
127+
databases:
128+
- name: sequin_test_two
129+
port: 5432
130+
ssl: false
131+
ipv6: false
132+
hostname: localhost
133+
pool_size: 10
134+
username: postgres
135+
password: '********'
136+
database: sequin_example
137+
slot_name: sequin_slot
138+
use_local_tunnel: false
139+
publication_name: sequin_pub
140+
sinks:
141+
- name: accounts_sink
142+
status: active
143+
table: public.accounts
144+
filters: []
145+
destination:
146+
port: 4222
147+
type: nats
148+
host: localhost
149+
tls: false
150+
database: sequin_test_two
151+
transform: record-transform
152+
active_backfill:
153+
batch_size: 1
154+
load_shedding_policy: pause_on_full
155+
max_retry_count:
156+
timestamp_format: iso8601
157+
actions:
158+
- insert
159+
- update
160+
- delete
161+
group_column_names:
162+
- user_id
163+
transforms:
164+
- name: record-transform
165+
type: path
166+
path: record
167+
description: Extracts just the record from the Sequin message shape.
168+
"""
169+
170+
conn = post(conn, ~p"/api/config/plan", %{yaml: yaml})
171+
172+
response = json_response(conn, 200)
173+
assert %{"changes" => changes} = response
174+
assert is_list(changes)
175+
end
176+
111177
end
112178

113179
describe "apply/2" do

0 commit comments

Comments
 (0)