fix: improve yaml load for sinks.max_retry_count, sinks.active_backfill and http_endpoints.encrypted_headers

lib/sequin/transforms/transforms.ex

@@ -403,6 +403,10 @@ defmodule Sequin.Transforms do
 
   defp encrypted_headers(%HttpEndpoint{encrypted_headers: nil}), do: %{}
 
+  defp encrypted_headers(%HttpEndpoint{encrypted_headers: encrypted_headers})
+       when is_map(encrypted_headers) and map_size(encrypted_headers) == 0,
+       do: %{}
+
   defp encrypted_headers(%HttpEndpoint{encrypted_headers: encrypted_headers}) do
     "(#{map_size(encrypted_headers)} encrypted header(s)) - sha256sum: #{sha256sum(encrypted_headers)}"
   end
@@ -431,11 +435,9 @@ defmodule Sequin.Transforms do
             {:error, error} -> {:halt, {:error, error}}
           end
 
+        # Ignore encrypted_headers until encryption/decryption works
         "encrypted_headers" ->
-          case parse_headers(value) do
-            {:ok, headers} -> {:cont, {:ok, Map.put(acc, :encrypted_headers, headers)}}
-            {:error, error} -> {:halt, {:error, error}}
-          end
+          {:cont, {:ok, acc}}
 
         "webhook.site" ->
           if value in [true, "true"] do
@@ -597,6 +599,9 @@ defmodule Sequin.Transforms do
         "max_retry_count" when is_integer(value) and value >= 0 ->
           {:cont, {:ok, Map.put(acc, :max_retry_count, value)}}
 
+        "max_retry_count" when is_nil(value) ->
+          {:cont, {:ok, acc}}
+
         "max_retry_count" ->
           {:halt, {:error, Error.validation(summary: "max_retry_count must be a non-negative integer")}}
 
@@ -611,6 +616,10 @@ defmodule Sequin.Transforms do
         "consumer_start" ->
           {:cont, {:ok, acc}}
 
+        # Ignore until it is properly implemented
+        "active_backfill" ->
+          {:cont, {:ok, acc}}
+
         # Ignore internal fields that might be present in the external data
         ignored when ignored in ~w(id inserted_at updated_at account_id replication_slot_id sequence_id) ->
           {:cont, {:ok, acc}}

test/sequin_web/controllers/yaml_controller_test.exs

@@ -108,6 +108,76 @@
                "summary" => "Error creating database 'test-db': - database: can't be blank"
              }
     end
+
+    test "successfully plans configuration with wider set of fields", %{conn: conn} do
+      yaml = """
+      change_retentions:
+        - name: test_retention
+          filters: []
+          destination_database: sequin_test
+          source_database: sequin_test
+          actions:
+            - insert
+            - update
+            - delete
+          source_table_name: accounts
+          source_table_schema: public
+          destination_table_name: sequin_events
+          destination_table_schema: public
+      databases:
+        - name: sequin_test
+          port: 5432
+          ssl: false
+          ipv6: false
+          hostname: localhost
+          pool_size: 10
+          username: postgres
+          password: '********'
+          database: sequin_example
+          slot_name: sequin_slot
+          use_local_tunnel: false
+          publication_name: sequin_pub
+      http_endpoints:
+        - name: test_http_endpoint
+          url: http://localhost:4000/something
+          headers: {}
+          encrypted_headers: '(0 encrypted header(s)) - sha256sum: b4a8f200'
+      sinks:
+        - name: accounts_sink
+          status: active
+          table: public.accounts
+          filters: []
+          destination:
+            port: 4222
+            type: nats
+            host: localhost
+            tls: false
+          database: sequin_test
+          transform: record-transform
+          active_backfill:
+          batch_size: 1
+          load_shedding_policy: pause_on_full
+          max_retry_count:
+          timestamp_format: iso8601
+          actions:
+            - insert
+            - update
+            - delete
+          group_column_names:
+            - user_id
+      transforms:
+        - name: record-transform
+          type: path
+          path: record
+          description: Extracts just the record from the Sequin message shape.
+      """
+
+      conn = post(conn, ~p"/api/config/plan", %{yaml: yaml})
+
+      response = json_response(conn, 200)
+      assert %{"changes" => changes} = response
+      assert is_list(changes)
+    end
   end
 
   describe "apply/2" do