fix: improve yaml load

As handling `active_backfill` seems to be entirely unsupported we are ignoring
that field for now until we have a proper implementation

Author: davoclavo

lib/sequin/transforms/transforms.ex

@@ -432,10 +432,7 @@ defmodule Sequin.Transforms do
           end
 
         "encrypted_headers" ->
-          case parse_headers(value) do
-            {:ok, headers} -> {:cont, {:ok, Map.put(acc, :encrypted_headers, headers)}}
-            {:error, error} -> {:halt, {:error, error}}
-          end
+          {:cont, {:ok, acc}}
 
         "webhook.site" ->
           if value in [true, "true"] do
@@ -597,6 +594,9 @@ defmodule Sequin.Transforms do
         "max_retry_count" when is_integer(value) and value >= 0 ->
           {:cont, {:ok, Map.put(acc, :max_retry_count, value)}}
 
+        "max_retry_count" when is_nil(value) ->
+          {:cont, {:ok, acc}}
+
         "max_retry_count" ->
           {:halt, {:error, Error.validation(summary: "max_retry_count must be a non-negative integer")}}
 
@@ -612,7 +612,7 @@ defmodule Sequin.Transforms do
           {:cont, {:ok, acc}}
 
         # Ignore internal fields that might be present in the external data
-        ignored when ignored in ~w(id inserted_at updated_at account_id replication_slot_id sequence_id) ->
+        ignored when ignored in ~w(id inserted_at updated_at account_id replication_slot_id sequence_id active_backfill) ->
           {:cont, {:ok, acc}}
 
         # Unknown field

test/sequin_web/controllers/yaml_controller_test.exs

@@ -108,6 +108,77 @@ defmodule SequinWeb.YamlControllerTest do
                "summary" => "Error creating database 'test-db': - database: can't be blank"
              }
     end
+
+    test "successfully plans configuration with wider set of fields", %{conn: conn} do
+      yaml = """
+      change_retentions:
+        - name: test_retention
+          filters: []
+          destination_database: sequin_test
+          source_database: sequin_test
+          actions:
+            - insert
+            - update
+            - delete
+          source_table_name: accounts
+          source_table_schema: public
+          destination_table_name: sequin_events
+          destination_table_schema: public
+      databases:
+        - name: sequin_test
+          port: 5432
+          ssl: false
+          ipv6: false
+          hostname: localhost
+          pool_size: 10
+          username: postgres
+          password: '********'
+          database: sequin_example
+          slot_name: sequin_slot
+          use_local_tunnel: false
+          publication_name: sequin_pub
+      http_endpoints:
+        - name: test_http_endpoint
+          url: http://localhost:4000/something
+          headers: {}
+          encrypted_headers: '(0 encrypted header(s)) - sha256sum: b4a8f200'
+      sinks:
+        - name: accounts_sink
+          status: active
+          table: public.accounts
+          filters: []
+          destination:
+            port: 4222
+            type: nats
+            host: localhost
+            tls: false
+          database: sequin_test
+          transform: record-transform
+          active_backfill:
+          batch_size: 1
+          load_shedding_policy: pause_on_full
+          max_retry_count:
+          timestamp_format: iso8601
+          actions:
+            - insert
+            - update
+            - delete
+          group_column_names:
+            - user_id
+      transforms:
+        - name: record-transform
+          type: path
+          path: record
+          description: Extracts just the record from the Sequin message shape.
+      """
+
+      conn = post(conn, ~p"/api/config/plan", %{yaml: yaml})
+
+      response = json_response(conn, 200)
+      assert %{"changes" => changes} = response
+      assert is_list(changes)
+    end
+
   end
 
   describe "apply/2" do