fix: improve yaml load for sinks.max_retry_count, sinks.active_backfill and http_endpoints.encrypted_headers

lib/sequin/transforms/transforms.ex

@@ -403,6 +403,10 @@ defmodule Sequin.Transforms do
 
   defp encrypted_headers(%HttpEndpoint{encrypted_headers: nil}), do: %{}
 
+  defp encrypted_headers(%HttpEndpoint{encrypted_headers: encrypted_headers})
+       when is_map(encrypted_headers) and map_size(encrypted_headers) == 0,
+       do: %{}
+
   defp encrypted_headers(%HttpEndpoint{encrypted_headers: encrypted_headers}) do
     "(#{map_size(encrypted_headers)} encrypted header(s)) - sha256sum: #{sha256sum(encrypted_headers)}"
   end
@@ -597,6 +601,9 @@ defmodule Sequin.Transforms do
         "max_retry_count" when is_integer(value) and value >= 0 ->
           {:cont, {:ok, Map.put(acc, :max_retry_count, value)}}
 
+        "max_retry_count" when is_nil(value) ->
+          {:cont, {:ok, acc}}
+
         "max_retry_count" ->
           {:halt, {:error, Error.validation(summary: "max_retry_count must be a non-negative integer")}}
 
@@ -611,6 +618,10 @@ defmodule Sequin.Transforms do
         "consumer_start" ->
           {:cont, {:ok, acc}}
 
+        # Ignore until it is properly implemented
+        "active_backfill" ->
+          {:cont, {:ok, acc}}
+
         # Ignore internal fields that might be present in the external data
         ignored when ignored in ~w(id inserted_at updated_at account_id replication_slot_id sequence_id) ->
           {:cont, {:ok, acc}}

test/sequin_web/controllers/yaml_controller_test.exs

@@ -4,6 +4,7 @@
   alias Sequin.Databases.PostgresDatabase
   alias Sequin.Databases.Sequence
   alias Sequin.Test.UnboxedRepo
+  alias Sequin.TestSupport.Models.Character
   alias Sequin.TestSupport.ReplicationSlots
 
   @moduletag :unboxed
@@ -108,6 +109,75 @@
                "summary" => "Error creating database 'test-db': - database: can't be blank"
              }
     end
+
+    test "successfully plans configuration with wider set of fields", %{conn: conn} do
+      yaml = """
+      change_retentions:
+        - name: test_retention
+          filters: []
+          destination_database: sequin_test
+          source_database: sequin_test
+          actions:
+            - insert
+            - update
+            - delete
+          source_table_name: #{Character.table_name()}
+          source_table_schema: public
+          destination_table_name: sequin_events
+          destination_table_schema: public
+      databases:
+        - name: sequin_test
+          port: 5432
+          ssl: false
+          ipv6: false
+          hostname: localhost
+          pool_size: 10
+          username: postgres
+          password: '********'
+          database: sequin_test
+          slot_name: sequin_slot
+          use_local_tunnel: false
+          publication_name: characters_publication
+      http_endpoints:
+        - name: test_http_endpoint
+          url: http://localhost:4000/something
+          headers: {}
+      sinks:
+        - name: accounts_sink
+          status: active
+          table: public.#{Character.table_name()}
+          filters: []
+          destination:
+            port: 4222
+            type: nats
+            host: localhost
+            tls: false
+          database: sequin_test
+          transform: record-transform
+          active_backfill:
+          batch_size: 1
+          load_shedding_policy: pause_on_full
+          max_retry_count:
+          timestamp_format: iso8601
+          actions:
+            - insert
+            - update
+            - delete
+          group_column_names:
+            - id
+      transforms:
+        - name: record-transform
+          type: path
+          path: record
+          description: Extracts just the record from the Sequin message shape.
+      """
+
+      conn = post(conn, ~p"/api/config/plan", %{yaml: yaml})
+
+      response = json_response(conn, 200)
+      assert %{"changes" => changes} = response
+      assert is_list(changes)
+    end
   end
 
   describe "apply/2" do

test/support/models/character.ex

@@ -17,6 +17,10 @@ defmodule Sequin.TestSupport.Models.Character do
     timestamps(type: :naive_datetime_usec)
   end
 
+  def table_name do
+    "Characters"
+  end
+
   def quoted_table_name do
     ~s{"Characters"}
   end