From 6f21e83301826c997b63740eba1321180c4c5e64 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Fri, 25 Jul 2025 21:20:51 -0400 Subject: [PATCH 01/30] feat: add mysql support to sink consumer --- examples/mysql/README.md | 91 ++++++++++ examples/mysql/docker-compose.yml | 21 +++ examples/mysql/init.sql | 25 +++ lib/sequin/consumers/mysql_sink.ex | 77 +++++++++ lib/sequin/consumers/sink_consumer.ex | 3 + lib/sequin/runtime/mysql_pipeline.ex | 129 ++++++++++++++ lib/sequin/runtime/sink_pipeline.ex | 1 + lib/sequin/sinks/mysql/client.ex | 231 ++++++++++++++++++++++++++ lib/sequin/transforms/transforms.ex | 35 ++++ mix.exs | 1 + mix.lock | 1 + test/sequin/mysql_client_test.exs | 25 +++ test/sequin/mysql_sink_test.exs | 153 +++++++++++++++++ 13 files changed, 793 insertions(+) create mode 100644 examples/mysql/README.md create mode 100644 examples/mysql/docker-compose.yml create mode 100644 examples/mysql/init.sql create mode 100644 lib/sequin/consumers/mysql_sink.ex create mode 100644 lib/sequin/runtime/mysql_pipeline.ex create mode 100644 lib/sequin/sinks/mysql/client.ex create mode 100644 test/sequin/mysql_client_test.exs create mode 100644 test/sequin/mysql_sink_test.exs diff --git a/examples/mysql/README.md b/examples/mysql/README.md new file mode 100644 index 000000000..b0500ece8 --- /dev/null +++ b/examples/mysql/README.md @@ -0,0 +1,91 @@ +# MySQL Sink Example + +This example demonstrates how to set up Sequin to stream Postgres changes to a MySQL database. + +## Prerequisites + +- A running Postgres database with logical replication enabled +- A running MySQL database +- Sequin installed and configured + +## Setup + +1. **Create a MySQL table** to receive the data: + +```sql +CREATE DATABASE sequin_test; +USE sequin_test; + +CREATE TABLE products ( + id INT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + price DECIMAL(10,2), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); +``` + +2. **Configure the MySQL Sink** in Sequin: + +```yaml +name: products-to-mysql +source: + database: my_postgres_db + table: products + actions: [insert, update, delete] + +destination: + type: mysql + host: localhost + port: 3306 + database: sequin_test + table_name: products + username: mysql_user + password: mysql_password + ssl: false + batch_size: 100 + timeout_seconds: 30 + upsert_on_duplicate: true + +transform: | + def transform(action, record, changes, metadata) do + # Map the Postgres record to MySQL-compatible format + %{ + id: record["id"], + name: record["name"], + price: record["price"] + } + end +``` + +## Features + +- **Upsert support**: Uses MySQL's `ON DUPLICATE KEY UPDATE` for handling updates +- **Batch processing**: Efficiently processes multiple records in batches +- **SSL support**: Can connect to MySQL over SSL +- **Type handling**: Automatically handles different data types and JSON encoding for complex values +- **Error handling**: Comprehensive error handling with detailed error messages + +## Usage + +Once configured, Sequin will: + +1. Capture changes from your Postgres table +2. Transform the data using your transform function +3. Insert/update records in MySQL using batch operations +4. Handle deletes by removing records from MySQL + +The sink supports both insert-only mode and upsert mode depending on your `upsert_on_duplicate` setting. + +## Connection Options + +- `host`: MySQL server hostname +- `port`: MySQL server port (default: 3306) +- `database`: Target database name +- `table_name`: Target table name +- `username`: MySQL username +- `password`: MySQL password +- `ssl`: Enable SSL connection (default: false) +- `batch_size`: Number of records to process in each batch (default: 100) +- `timeout_seconds`: Connection timeout in seconds (default: 30) +- `upsert_on_duplicate`: Use upsert instead of insert-only (default: true) \ No newline at end of file diff --git a/examples/mysql/docker-compose.yml b/examples/mysql/docker-compose.yml new file mode 100644 index 000000000..9379cb9a3 --- /dev/null +++ b/examples/mysql/docker-compose.yml @@ -0,0 +1,21 @@ +services: + mysql: + image: mysql:8.0 + container_name: sequin-mysql-example + environment: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_DATABASE: sequin_test + MYSQL_USER: sequin_user + MYSQL_PASSWORD: sequin_password + ports: + - "3306:3306" + volumes: + - mysql_data:/var/lib/mysql + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + timeout: 20s + retries: 10 + +volumes: + mysql_data: \ No newline at end of file diff --git a/examples/mysql/init.sql b/examples/mysql/init.sql new file mode 100644 index 000000000..32dcc9b21 --- /dev/null +++ b/examples/mysql/init.sql @@ -0,0 +1,25 @@ +-- Create the products table for testing Sequin MySQL sink +CREATE TABLE IF NOT EXISTS products ( + id INT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + price DECIMAL(10,2), + description TEXT, + category VARCHAR(100), + in_stock BOOLEAN DEFAULT TRUE, + metadata JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); + +-- Create an index on commonly queried fields +CREATE INDEX idx_products_category ON products(category); +CREATE INDEX idx_products_in_stock ON products(in_stock); + +-- Insert some sample data for testing +INSERT INTO products (id, name, price, description, category, in_stock) VALUES +(1, 'Sample Product 1', 19.99, 'This is a sample product for testing', 'electronics', TRUE), +(2, 'Sample Product 2', 29.99, 'Another sample product', 'books', TRUE), +(3, 'Sample Product 3', 39.99, 'Third sample product', 'clothing', FALSE); + +-- Show the created table structure +DESCRIBE products; \ No newline at end of file diff --git a/lib/sequin/consumers/mysql_sink.ex b/lib/sequin/consumers/mysql_sink.ex new file mode 100644 index 000000000..a1a4ad845 --- /dev/null +++ b/lib/sequin/consumers/mysql_sink.ex @@ -0,0 +1,77 @@ +defmodule Sequin.Consumers.MysqlSink do + @moduledoc false + use Ecto.Schema + use TypedEctoSchema + + import Ecto.Changeset + + alias Sequin.Encrypted + + @derive {Jason.Encoder, only: [:host, :port, :database, :table_name, :username, :ssl]} + @derive {Inspect, except: [:password]} + + @primary_key false + typed_embedded_schema do + field(:type, Ecto.Enum, values: [:mysql], default: :mysql) + field(:host, :string) + field(:port, :integer, default: 3306) + field(:database, :string) + field(:table_name, :string) + field(:username, :string) + field(:password, Encrypted.Field) + field(:ssl, :boolean, default: false) + field(:batch_size, :integer, default: 100) + field(:timeout_seconds, :integer, default: 30) + field(:upsert_on_duplicate, :boolean, default: true) + end + + def changeset(struct, params) do + struct + |> cast(params, [ + :host, + :port, + :database, + :table_name, + :username, + :password, + :ssl, + :batch_size, + :timeout_seconds, + :upsert_on_duplicate + ]) + |> validate_required([:host, :database, :table_name, :username, :password]) + |> validate_number(:port, greater_than: 0, less_than_or_equal_to: 65535) + |> validate_number(:batch_size, greater_than: 0, less_than_or_equal_to: 10_000) + |> validate_number(:timeout_seconds, greater_than: 0, less_than_or_equal_to: 300) + |> validate_length(:host, max: 255) + |> validate_length(:database, max: 64) + |> validate_length(:table_name, max: 64) + |> validate_length(:username, max: 32) + |> validate_table_name() + end + + defp validate_table_name(changeset) do + changeset + |> validate_format(:table_name, ~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, + message: "must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)" + ) + end + + def connection_opts(%__MODULE__{} = sink) do + opts = [ + hostname: sink.host, + port: sink.port, + database: sink.database, + username: sink.username, + password: sink.password, + timeout: :timer.seconds(sink.timeout_seconds), + pool_size: 10 + ] + + if sink.ssl do + Keyword.put(opts, :ssl, true) + else + opts + end + end +end diff --git a/lib/sequin/consumers/sink_consumer.ex b/lib/sequin/consumers/sink_consumer.ex index a61b71b2b..3e28a8fe9 100644 --- a/lib/sequin/consumers/sink_consumer.ex +++ b/lib/sequin/consumers/sink_consumer.ex @@ -18,6 +18,7 @@ defmodule Sequin.Consumers.SinkConsumer do alias Sequin.Consumers.KafkaSink alias Sequin.Consumers.KinesisSink alias Sequin.Consumers.MeilisearchSink + alias Sequin.Consumers.MysqlSink alias Sequin.Consumers.NatsSink alias Sequin.Consumers.RabbitMqSink alias Sequin.Consumers.RedisStreamSink @@ -50,6 +51,7 @@ defmodule Sequin.Consumers.SinkConsumer do :azure_event_hub, :typesense, :meilisearch, + :mysql, :sns, :elasticsearch ] @@ -136,6 +138,7 @@ defmodule Sequin.Consumers.SinkConsumer do azure_event_hub: AzureEventHubSink, typesense: TypesenseSink, meilisearch: MeilisearchSink, + mysql: MysqlSink, elasticsearch: ElasticsearchSink ], on_replace: :update, diff --git a/lib/sequin/runtime/mysql_pipeline.ex b/lib/sequin/runtime/mysql_pipeline.ex new file mode 100644 index 000000000..448577fcf --- /dev/null +++ b/lib/sequin/runtime/mysql_pipeline.ex @@ -0,0 +1,129 @@ +defmodule Sequin.Runtime.MysqlPipeline do + @moduledoc false + @behaviour Sequin.Runtime.SinkPipeline + + alias Sequin.Consumers.SinkConsumer + alias Sequin.Runtime.SinkPipeline + alias Sequin.Runtime.Trace + alias Sequin.Sinks.Mysql.Client + alias Sequin.Transforms.Message + + @impl SinkPipeline + def init(context, _opts) do + context + end + + @impl SinkPipeline + def batchers_config(consumer) do + concurrency = min(System.schedulers_online() * 2, 20) + + [ + default: [ + concurrency: concurrency, + batch_size: consumer.sink.batch_size, + batch_timeout: 1000 + ], + delete: [ + concurrency: concurrency, + batch_size: consumer.sink.batch_size, + batch_timeout: 1000 + ] + ] + end + + @impl SinkPipeline + def handle_message(message, context) do + batcher = + case message.data.data.action do + :delete -> :delete + _ -> :default + end + + {:ok, Broadway.Message.put_batcher(message, batcher), context} + end + + @impl SinkPipeline + def handle_batch(:default, messages, _batch_info, context) do + %{ + consumer: %SinkConsumer{sink: sink} = consumer, + test_pid: test_pid + } = context + + setup_allowances(test_pid) + + records = + Enum.map(messages, fn %{data: message} -> + consumer + |> Message.to_external(message) + |> ensure_string_keys() + end) + + case Client.upsert_records(sink, records) do + {:ok} -> + Trace.info(consumer.id, %Trace.Event{ + message: "Upserted records to MySQL table \"#{sink.table_name}\"" + }) + + {:ok, messages, context} + + {:error, error} -> + Trace.error(consumer.id, %Trace.Event{ + message: "Failed to upsert records to MySQL table \"#{sink.table_name}\"", + error: error + }) + + {:error, error} + end + end + + @impl SinkPipeline + def handle_batch(:delete, messages, _batch_info, context) do + %{ + consumer: %SinkConsumer{sink: sink} = consumer, + test_pid: test_pid + } = context + + setup_allowances(test_pid) + + record_pks = + Enum.flat_map(messages, fn %{data: message} -> message.record_pks end) + + case Client.delete_records(sink, record_pks) do + {:ok} -> + Trace.info(consumer.id, %Trace.Event{ + message: "Deleted records from MySQL table \"#{sink.table_name}\"", + extra: %{record_pks: record_pks} + }) + + {:ok, messages, context} + + {:error, error} -> + Trace.error(consumer.id, %Trace.Event{ + message: "Failed to delete records from MySQL table \"#{sink.table_name}\"", + error: error, + extra: %{record_pks: record_pks} + }) + + {:error, error} + end + end + + # Helper functions + + # Ensure all keys in the record are strings for MySQL column compatibility + defp ensure_string_keys(record) when is_map(record) do + Map.new(record, fn + {key, value} when is_atom(key) -> {Atom.to_string(key), value} + {key, value} -> {key, value} + end) + end + + defp ensure_string_keys(record), do: record + + defp setup_allowances(nil), do: :ok + + defp setup_allowances(test_pid) do + Req.Test.allow(Client, test_pid, self()) + Mox.allow(Sequin.TestSupport.DateTimeMock, test_pid, self()) + end +end diff --git a/lib/sequin/runtime/sink_pipeline.ex b/lib/sequin/runtime/sink_pipeline.ex index 4feb7550d..6772336d3 100644 --- a/lib/sequin/runtime/sink_pipeline.ex +++ b/lib/sequin/runtime/sink_pipeline.ex @@ -428,6 +428,7 @@ defmodule Sequin.Runtime.SinkPipeline do :s2 -> Sequin.Runtime.S2Pipeline :typesense -> Sequin.Runtime.TypesensePipeline :meilisearch -> Sequin.Runtime.MeilisearchPipeline + :mysql -> Sequin.Runtime.MysqlPipeline :sns -> Sequin.Runtime.SnsPipeline end end diff --git a/lib/sequin/sinks/mysql/client.ex b/lib/sequin/sinks/mysql/client.ex new file mode 100644 index 000000000..795516169 --- /dev/null +++ b/lib/sequin/sinks/mysql/client.ex @@ -0,0 +1,231 @@ +defmodule Sequin.Sinks.Mysql.Client do + @moduledoc """ + Client for interacting with MySQL databases using MyXQL. + """ + + alias Sequin.Consumers.MysqlSink + alias Sequin.Error + + require Logger + + @doc """ + Test the connection to the MySQL database. + """ + def test_connection(%MysqlSink{} = sink) do + case start_connection(sink) do + {:ok, pid} -> + try do + case MyXQL.query(pid, "SELECT 1", []) do + {:ok, _result} -> :ok + {:error, error} -> {:error, format_error(error)} + end + after + GenServer.stop(pid) + end + + {:error, error} -> + {:error, format_error(error)} + end + end + + @doc """ + Insert or update records in batch. + """ + def upsert_records(%MysqlSink{} = sink, records) when is_list(records) do + if Enum.empty?(records) do + {:ok} + else + case start_connection(sink) do + {:ok, pid} -> + try do + result = + if sink.upsert_on_duplicate do + insert_or_update_records(pid, sink, records) + else + insert_records(pid, sink, records) + end + + GenServer.stop(pid) + result + rescue + error -> + GenServer.stop(pid) + {:error, format_error(error)} + end + + {:error, error} -> + {:error, format_error(error)} + end + end + end + + @doc """ + Delete records by their primary keys. + """ + def delete_records(%MysqlSink{} = sink, record_pks) when is_list(record_pks) do + if Enum.empty?(record_pks) do + {:ok} + else + case start_connection(sink) do + {:ok, pid} -> + try do + # Assume primary key is 'id' for simplicity + placeholders = Enum.map(record_pks, fn _ -> "?" end) |> Enum.join(", ") + delete_sql = "DELETE FROM `#{sink.table_name}` WHERE `id` IN (#{placeholders})" + + case MyXQL.query(pid, delete_sql, record_pks) do + {:ok, %MyXQL.Result{num_rows: num_rows}} -> + Logger.debug("[MySQL] Deleted #{num_rows} records from #{sink.table_name}") + {:ok} + + {:error, error} -> + {:error, format_error(error)} + end + after + GenServer.stop(pid) + end + + {:error, error} -> + {:error, format_error(error)} + end + end + end + + # Private functions + + defp start_connection(%MysqlSink{} = sink) do + MyXQL.start_link(MysqlSink.connection_opts(sink)) + end + + defp insert_or_update_records(pid, %MysqlSink{} = sink, records) do + case build_upsert_query(sink, records) do + {:ok, {sql, params}} -> + case MyXQL.query(pid, sql, params) do + {:ok, %MyXQL.Result{num_rows: num_rows}} -> + Logger.debug("[MySQL] Upserted #{num_rows} records to #{sink.table_name}") + {:ok} + + {:error, error} -> + {:error, format_error(error)} + end + + {:error, error} -> + {:error, error} + end + end + + defp insert_records(pid, %MysqlSink{} = sink, records) do + case build_insert_query(sink, records) do + {:ok, {sql, params}} -> + case MyXQL.query(pid, sql, params) do + {:ok, %MyXQL.Result{num_rows: num_rows}} -> + Logger.debug("[MySQL] Inserted #{num_rows} records to #{sink.table_name}") + {:ok} + + {:error, error} -> + {:error, format_error(error)} + end + + {:error, error} -> + {:error, error} + end + end + + defp build_upsert_query(%MysqlSink{} = sink, records) do + case extract_columns_and_values(records) do + {:ok, {columns, values}} -> + column_list = Enum.map(columns, &"`#{&1}`") |> Enum.join(", ") + placeholders = Enum.map(columns, fn _ -> "?" end) |> Enum.join(", ") + + # Build ON DUPLICATE KEY UPDATE clause + update_clause = + columns + |> Enum.map(&"`#{&1}` = VALUES(`#{&1}`)") + |> Enum.join(", ") + + sql = """ + INSERT INTO `#{sink.table_name}` (#{column_list}) + VALUES #{Enum.map(values, fn _ -> "(#{placeholders})" end) |> Enum.join(", ")} + ON DUPLICATE KEY UPDATE #{update_clause} + """ + + # Flatten all values for parameters + params = Enum.flat_map(values, & &1) + + {:ok, {sql, params}} + + {:error, error} -> + {:error, error} + end + end + + defp build_insert_query(%MysqlSink{} = sink, records) do + case extract_columns_and_values(records) do + {:ok, {columns, values}} -> + column_list = Enum.map(columns, &"`#{&1}`") |> Enum.join(", ") + placeholders = Enum.map(columns, fn _ -> "?" end) |> Enum.join(", ") + + sql = """ + INSERT INTO `#{sink.table_name}` (#{column_list}) + VALUES #{Enum.map(values, fn _ -> "(#{placeholders})" end) |> Enum.join(", ")} + """ + + # Flatten all values for parameters + params = Enum.flat_map(values, & &1) + + {:ok, {sql, params}} + + {:error, error} -> + {:error, error} + end + end + + defp extract_columns_and_values(records) do + if Enum.empty?(records) do + {:error, Error.service(service: :mysql, message: "No records provided")} + else + # Get all unique columns from all records + all_columns = + records + |> Enum.flat_map(&Map.keys/1) + |> Enum.uniq() + |> Enum.sort() + + # Extract values for each record, filling nil for missing columns + values = + Enum.map(records, fn record -> + Enum.map(all_columns, fn column -> + case Map.get(record, column) do + nil -> nil + value when is_binary(value) -> value + value when is_number(value) -> value + value when is_boolean(value) -> value + value -> Jason.encode!(value) # JSON encode complex values + end + end) + end) + + {:ok, {all_columns, values}} + end + end + + defp format_error(%MyXQL.Error{} = error) do + Error.service( + service: :mysql, + message: Exception.message(error), + details: %{mysql_error: error} + ) + end + + defp format_error(error) when is_binary(error) do + Error.service(service: :mysql, message: error) + end + + defp format_error(error) do + Error.service( + service: :mysql, + message: "MySQL error: #{inspect(error)}", + details: %{original_error: error} + ) + end +end diff --git a/lib/sequin/transforms/transforms.ex b/lib/sequin/transforms/transforms.ex index cda95b793..1e236f135 100644 --- a/lib/sequin/transforms/transforms.ex +++ b/lib/sequin/transforms/transforms.ex @@ -15,6 +15,7 @@ defmodule Sequin.Transforms do alias Sequin.Consumers.KafkaSink alias Sequin.Consumers.KinesisSink alias Sequin.Consumers.MeilisearchSink + alias Sequin.Consumers.MysqlSink alias Sequin.Consumers.NatsSink alias Sequin.Consumers.PathFunction alias Sequin.Consumers.RabbitMqSink @@ -420,6 +421,22 @@ defmodule Sequin.Transforms do }) end + def to_external(%MysqlSink{} = sink, show_sensitive) do + reject_nil_values(%{ + type: "mysql", + host: sink.host, + port: sink.port, + database: sink.database, + table_name: sink.table_name, + username: sink.username, + password: SensitiveValue.new(sink.password, show_sensitive), + ssl: sink.ssl, + batch_size: sink.batch_size, + timeout_seconds: sink.timeout_seconds, + upsert_on_duplicate: sink.upsert_on_duplicate + }) + end + def to_external(%ElasticsearchSink{} = sink, show_sensitive) do reject_nil_values(%{ type: "elasticsearch", @@ -1282,6 +1299,24 @@ defmodule Sequin.Transforms do }} end + # Add parse_sink for mysql type + defp parse_sink(%{"type" => "mysql"} = attrs, _resources) do + {:ok, + %{ + type: :mysql, + host: attrs["host"], + port: attrs["port"], + database: attrs["database"], + table_name: attrs["table_name"], + username: attrs["username"], + password: attrs["password"], + ssl: attrs["ssl"], + batch_size: attrs["batch_size"], + timeout_seconds: attrs["timeout_seconds"], + upsert_on_duplicate: attrs["upsert_on_duplicate"] + }} + end + # Add parse_sink for elasticsearch type defp parse_sink(%{"type" => "elasticsearch"} = attrs, _resources) do {:ok, diff --git a/mix.exs b/mix.exs index 5ae3d4099..ddaa014fc 100644 --- a/mix.exs +++ b/mix.exs @@ -52,6 +52,7 @@ defmodule Sequin.MixProject do # Database and Ecto {:ecto_sql, "~> 3.10"}, {:postgrex, ">= 0.0.0"}, + {:myxql, "~> 0.8.0"}, {:polymorphic_embed, "~> 4.1.1"}, {:typed_ecto_schema, "~> 0.4.1"}, {:cloak_ecto, "~> 1.3.0"}, diff --git a/mix.lock b/mix.lock index 1d7eef294..dcb3235ee 100644 --- a/mix.lock +++ b/mix.lock @@ -77,6 +77,7 @@ "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mix_test_interactive": {:hex, :mix_test_interactive, "2.0.4", "f17643b2f441da63aba1ba8e75b9fd407c7358abb3fb559e6abb90ea74e61416", [:mix], [{:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "69e2b663dc532514fcd2caa50a8793565b64463b57c8c42593bd462646dd68d8"}, "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, + "myxql": {:hex, :myxql, "0.8.0", "60c60e87c7320d2f5759416aa1758c8e7534efbae07b192861977f8455e35acd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:geo, "~> 3.4 or ~> 4.0", [hex: :geo, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "1ec0ceb26fb3cd0f8756519cf4f0e4f9348177a020705223bdf4742a2c44d774"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, diff --git a/test/sequin/mysql_client_test.exs b/test/sequin/mysql_client_test.exs new file mode 100644 index 000000000..dcfa45a75 --- /dev/null +++ b/test/sequin/mysql_client_test.exs @@ -0,0 +1,25 @@ +defmodule Sequin.Sinks.Mysql.ClientTest do + use Sequin.Case, async: true + + alias Sequin.Consumers.MysqlSink + alias Sequin.Factory.SinkFactory + alias Sequin.Sinks.Mysql.Client + + @sink %MysqlSink{ + type: :mysql, + host: "localhost", + port: 3306, + database: "test_db", + table_name: "test_table", + username: "test_user", + password: "test_password", + timeout_seconds: 30 + } + + # Integration tests would be needed to test the client functions + # against a real MySQL database, but that's beyond the scope of unit testing +end + +# Note: This test file tests public functions only. +# Private function testing would require exposing them in the main module +# or using more advanced testing techniques. diff --git a/test/sequin/mysql_sink_test.exs b/test/sequin/mysql_sink_test.exs new file mode 100644 index 000000000..8496efbf7 --- /dev/null +++ b/test/sequin/mysql_sink_test.exs @@ -0,0 +1,153 @@ +defmodule Sequin.Consumers.MysqlSinkTest do + use Sequin.Case, async: true + + alias Sequin.Consumers.MysqlSink + + describe "changeset/2" do + setup do + %{ + valid_params: %{ + host: "localhost", + port: 3306, + database: "test_db", + table_name: "test_table", + username: "test_user", + password: "test_password" + } + } + end + + test "valid params have no errors", %{valid_params: params} do + changeset = MysqlSink.changeset(%MysqlSink{}, params) + assert Sequin.Error.errors_on(changeset) == %{} + end + + test "validates required fields", %{valid_params: params} do + changeset = MysqlSink.changeset(%MysqlSink{}, %{}) + errors = Sequin.Error.errors_on(changeset) + + assert errors[:host] == ["can't be blank"] + assert errors[:database] == ["can't be blank"] + assert errors[:table_name] == ["can't be blank"] + assert errors[:username] == ["can't be blank"] + assert errors[:password] == ["can't be blank"] + end + + test "validates port range", %{valid_params: params} do + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | port: 0}) + assert Sequin.Error.errors_on(changeset)[:port] == ["must be greater than 0"] + + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | port: 70000}) + assert Sequin.Error.errors_on(changeset)[:port] == ["must be less than or equal to 65535"] + end + + test "validates batch_size range", %{valid_params: params} do + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | batch_size: 0}) + assert Sequin.Error.errors_on(changeset)[:batch_size] == ["must be greater than 0"] + + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | batch_size: 20000}) + assert Sequin.Error.errors_on(changeset)[:batch_size] == ["must be less than or equal to 10000"] + end + + test "validates timeout_seconds range", %{valid_params: params} do + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | timeout_seconds: 0}) + assert Sequin.Error.errors_on(changeset)[:timeout_seconds] == ["must be greater than 0"] + + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | timeout_seconds: 400}) + assert Sequin.Error.errors_on(changeset)[:timeout_seconds] == ["must be less than or equal to 300"] + end + + test "validates table_name format", %{valid_params: params} do + # Valid table names + valid_names = ["table", "user_data", "Table123", "_private"] + for name <- valid_names do + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: name}) + assert Sequin.Error.errors_on(changeset)[:table_name] == nil, "#{name} should be valid" + end + + # Invalid table names + invalid_names = ["123table", "table-name", "table space", ""] + for name <- invalid_names do + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: name}) + assert Sequin.Error.errors_on(changeset)[:table_name] == ["must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)"], "#{name} should be invalid" + end + end + + test "validates string field lengths", %{valid_params: params} do + # Test host length + long_host = String.duplicate("a", 256) + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | host: long_host}) + assert Sequin.Error.errors_on(changeset)[:host] == ["should be at most 255 character(s)"] + + # Test database length + long_database = String.duplicate("a", 65) + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | database: long_database}) + assert Sequin.Error.errors_on(changeset)[:database] == ["should be at most 64 character(s)"] + + # Test table_name length + long_table = String.duplicate("a", 65) + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: long_table}) + assert Sequin.Error.errors_on(changeset)[:table_name] == ["should be at most 64 character(s)"] + + # Test username length + long_username = String.duplicate("a", 33) + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | username: long_username}) + assert Sequin.Error.errors_on(changeset)[:username] == ["should be at most 32 character(s)"] + end + end + + describe "connection_opts/1" do + test "generates correct connection options without SSL" do + sink = %MysqlSink{ + host: "localhost", + port: 3306, + database: "test_db", + username: "test_user", + password: "test_password", + ssl: false, + timeout_seconds: 30 + } + + opts = MysqlSink.connection_opts(sink) + + expected = [ + hostname: "localhost", + port: 3306, + database: "test_db", + username: "test_user", + password: "test_password", + timeout: 30_000, + pool_size: 10 + ] + + assert opts == expected + end + + test "generates correct connection options with SSL" do + sink = %MysqlSink{ + host: "mysql.example.com", + port: 3306, + database: "prod_db", + username: "prod_user", + password: "prod_password", + ssl: true, + timeout_seconds: 60 + } + + opts = MysqlSink.connection_opts(sink) + + expected = [ + hostname: "mysql.example.com", + port: 3306, + database: "prod_db", + username: "prod_user", + password: "prod_password", + timeout: 60_000, + pool_size: 10, + ssl: true + ] + + assert opts == expected + end + end +end From 354b80f82205aac97c9069092c07f2e9ddb185f3 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Fri, 25 Jul 2025 21:39:07 -0400 Subject: [PATCH 02/30] feat: implement dynamic routing for MySQL sink consumer --- examples/mysql/README.md | 54 +++++++++++++++++-- examples/mysql/init.sql | 41 +++++++++++++- lib/sequin/consumers/mysql_sink.ex | 4 +- lib/sequin/runtime/mysql_pipeline.ex | 34 +++++++++--- lib/sequin/runtime/routing/consumers/mysql.ex | 49 +++++++++++++++++ lib/sequin/runtime/routing/routing.ex | 1 + lib/sequin/transforms/transforms.ex | 11 +++- 7 files changed, 178 insertions(+), 16 deletions(-) create mode 100644 lib/sequin/runtime/routing/consumers/mysql.ex diff --git a/examples/mysql/README.md b/examples/mysql/README.md index b0500ece8..03a30d5b3 100644 --- a/examples/mysql/README.md +++ b/examples/mysql/README.md @@ -39,13 +39,14 @@ destination: host: localhost port: 3306 database: sequin_test - table_name: products + table_name: products # Used when routing_mode is static username: mysql_user password: mysql_password ssl: false batch_size: 100 timeout_seconds: 30 upsert_on_duplicate: true + routing_mode: static # Can be 'static' or 'dynamic' transform: | def transform(action, record, changes, metadata) do @@ -58,8 +59,54 @@ transform: | end ``` +## Dynamic Table Routing + +You can use routing functions to dynamically choose which MySQL table to route data to based on the record content: + +```yaml +name: multi-table-mysql-sink +source: + database: my_postgres_db + table: events + actions: [insert, update, delete] + +destination: + type: mysql + host: localhost + port: 3306 + database: sequin_test + table_name: default_events # Fallback table name + username: mysql_user + password: mysql_password + routing_mode: dynamic # Enable dynamic routing + +routing: | + def route(action, record, changes, metadata) do + # Route based on event type + table_name = case record["event_type"] do + "user_signup" -> "user_events" + "purchase" -> "purchase_events" + "analytics" -> "analytics_events" + _ -> "other_events" + end + + %{table_name: table_name} + end + +transform: | + def transform(action, record, changes, metadata) do + %{ + id: record["id"], + event_type: record["event_type"], + data: record["data"], + created_at: record["created_at"] + } + end +``` + ## Features +- **Dynamic routing**: Route to different MySQL tables based on record content - **Upsert support**: Uses MySQL's `ON DUPLICATE KEY UPDATE` for handling updates - **Batch processing**: Efficiently processes multiple records in batches - **SSL support**: Can connect to MySQL over SSL @@ -82,10 +129,11 @@ The sink supports both insert-only mode and upsert mode depending on your `upser - `host`: MySQL server hostname - `port`: MySQL server port (default: 3306) - `database`: Target database name -- `table_name`: Target table name +- `table_name`: Target table name (used as fallback when routing_mode is dynamic) - `username`: MySQL username - `password`: MySQL password - `ssl`: Enable SSL connection (default: false) - `batch_size`: Number of records to process in each batch (default: 100) - `timeout_seconds`: Connection timeout in seconds (default: 30) -- `upsert_on_duplicate`: Use upsert instead of insert-only (default: true) \ No newline at end of file +- `upsert_on_duplicate`: Use upsert instead of insert-only (default: true) +- `routing_mode`: Set to "dynamic" to enable routing functions, "static" for fixed table (default: static) \ No newline at end of file diff --git a/examples/mysql/init.sql b/examples/mysql/init.sql index 32dcc9b21..eb8615a9c 100644 --- a/examples/mysql/init.sql +++ b/examples/mysql/init.sql @@ -21,5 +21,42 @@ INSERT INTO products (id, name, price, description, category, in_stock) VALUES (2, 'Sample Product 2', 29.99, 'Another sample product', 'books', TRUE), (3, 'Sample Product 3', 39.99, 'Third sample product', 'clothing', FALSE); --- Show the created table structure -DESCRIBE products; \ No newline at end of file +-- Create additional tables for routing examples +CREATE TABLE IF NOT EXISTS user_events ( + id INT PRIMARY KEY, + event_type VARCHAR(100) NOT NULL, + user_id INT, + data JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS purchase_events ( + id INT PRIMARY KEY, + event_type VARCHAR(100) NOT NULL, + user_id INT, + product_id INT, + amount DECIMAL(10,2), + data JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS analytics_events ( + id INT PRIMARY KEY, + event_type VARCHAR(100) NOT NULL, + session_id VARCHAR(255), + page_url VARCHAR(500), + data JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS other_events ( + id INT PRIMARY KEY, + event_type VARCHAR(100) NOT NULL, + data JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Show the created table structures +DESCRIBE products; +DESCRIBE user_events; +DESCRIBE purchase_events; \ No newline at end of file diff --git a/lib/sequin/consumers/mysql_sink.ex b/lib/sequin/consumers/mysql_sink.ex index a1a4ad845..88cb44618 100644 --- a/lib/sequin/consumers/mysql_sink.ex +++ b/lib/sequin/consumers/mysql_sink.ex @@ -23,6 +23,7 @@ defmodule Sequin.Consumers.MysqlSink do field(:batch_size, :integer, default: 100) field(:timeout_seconds, :integer, default: 30) field(:upsert_on_duplicate, :boolean, default: true) + field(:routing_mode, Ecto.Enum, values: [:dynamic, :static], default: :static) end def changeset(struct, params) do @@ -37,7 +38,8 @@ defmodule Sequin.Consumers.MysqlSink do :ssl, :batch_size, :timeout_seconds, - :upsert_on_duplicate + :upsert_on_duplicate, + :routing_mode ]) |> validate_required([:host, :database, :table_name, :username, :password]) |> validate_number(:port, greater_than: 0, less_than_or_equal_to: 65535) diff --git a/lib/sequin/runtime/mysql_pipeline.ex b/lib/sequin/runtime/mysql_pipeline.ex index 448577fcf..4aa5bf5e9 100644 --- a/lib/sequin/runtime/mysql_pipeline.ex +++ b/lib/sequin/runtime/mysql_pipeline.ex @@ -3,6 +3,7 @@ defmodule Sequin.Runtime.MysqlPipeline do @behaviour Sequin.Runtime.SinkPipeline alias Sequin.Consumers.SinkConsumer + alias Sequin.Runtime.Routing alias Sequin.Runtime.SinkPipeline alias Sequin.Runtime.Trace alias Sequin.Sinks.Mysql.Client @@ -33,17 +34,22 @@ defmodule Sequin.Runtime.MysqlPipeline do @impl SinkPipeline def handle_message(message, context) do + %{consumer: consumer} = context + + %Routing.Consumers.Mysql{table_name: table_name} = Routing.route_message(consumer, message.data) + batcher = case message.data.data.action do :delete -> :delete _ -> :default end + message = Broadway.Message.put_batch_key(message, table_name) {:ok, Broadway.Message.put_batcher(message, batcher), context} end @impl SinkPipeline - def handle_batch(:default, messages, _batch_info, context) do + def handle_batch(:default, messages, batch_info, context) do %{ consumer: %SinkConsumer{sink: sink} = consumer, test_pid: test_pid @@ -51,6 +57,9 @@ defmodule Sequin.Runtime.MysqlPipeline do setup_allowances(test_pid) + # Get the table name from the batch key (set by routing) + table_name = Map.get(batch_info, :batch_key, sink.table_name) + records = Enum.map(messages, fn %{data: message} -> consumer @@ -58,17 +67,20 @@ defmodule Sequin.Runtime.MysqlPipeline do |> ensure_string_keys() end) - case Client.upsert_records(sink, records) do + # Create a temporary sink with the routed table name + routed_sink = %{sink | table_name: table_name} + + case Client.upsert_records(routed_sink, records) do {:ok} -> Trace.info(consumer.id, %Trace.Event{ - message: "Upserted records to MySQL table \"#{sink.table_name}\"" + message: "Upserted records to MySQL table \"#{table_name}\"" }) {:ok, messages, context} {:error, error} -> Trace.error(consumer.id, %Trace.Event{ - message: "Failed to upsert records to MySQL table \"#{sink.table_name}\"", + message: "Failed to upsert records to MySQL table \"#{table_name}\"", error: error }) @@ -77,7 +89,7 @@ defmodule Sequin.Runtime.MysqlPipeline do end @impl SinkPipeline - def handle_batch(:delete, messages, _batch_info, context) do + def handle_batch(:delete, messages, batch_info, context) do %{ consumer: %SinkConsumer{sink: sink} = consumer, test_pid: test_pid @@ -85,13 +97,19 @@ defmodule Sequin.Runtime.MysqlPipeline do setup_allowances(test_pid) + # Get the table name from the batch key (set by routing) + table_name = Map.get(batch_info, :batch_key, sink.table_name) + record_pks = Enum.flat_map(messages, fn %{data: message} -> message.record_pks end) - case Client.delete_records(sink, record_pks) do + # Create a temporary sink with the routed table name + routed_sink = %{sink | table_name: table_name} + + case Client.delete_records(routed_sink, record_pks) do {:ok} -> Trace.info(consumer.id, %Trace.Event{ - message: "Deleted records from MySQL table \"#{sink.table_name}\"", + message: "Deleted records from MySQL table \"#{table_name}\"", extra: %{record_pks: record_pks} }) @@ -99,7 +117,7 @@ defmodule Sequin.Runtime.MysqlPipeline do {:error, error} -> Trace.error(consumer.id, %Trace.Event{ - message: "Failed to delete records from MySQL table \"#{sink.table_name}\"", + message: "Failed to delete records from MySQL table \"#{table_name}\"", error: error, extra: %{record_pks: record_pks} }) diff --git a/lib/sequin/runtime/routing/consumers/mysql.ex b/lib/sequin/runtime/routing/consumers/mysql.ex new file mode 100644 index 000000000..25697a45a --- /dev/null +++ b/lib/sequin/runtime/routing/consumers/mysql.ex @@ -0,0 +1,49 @@ +defmodule Sequin.Runtime.Routing.Consumers.Mysql do + @moduledoc false + use Sequin.Runtime.Routing.RoutedConsumer + + alias Sequin.Runtime.Routing + + @primary_key false + @derive {Jason.Encoder, only: [:table_name]} + typed_embedded_schema do + field :table_name, :string + end + + def changeset(struct, params) do + allowed_keys = [:table_name] + + struct + |> cast(params, allowed_keys, empty_values: []) + |> Routing.Helpers.validate_no_extra_keys(params, allowed_keys) + |> validate_required([:table_name]) + |> validate_length(:table_name, min: 1, max: 64) + |> validate_format(:table_name, ~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, + message: "must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)" + ) + end + + def route(_action, _record, _changes, metadata) do + # Default routing: use the source table name, but replace schema separator + table_name = + if metadata.table_schema do + "#{metadata.table_schema}_#{metadata.table_name}" + else + metadata.table_name + end + + %{table_name: sanitize_table_name(table_name)} + end + + def route_consumer(%Sequin.Consumers.SinkConsumer{sink: sink}) do + %{table_name: sink.table_name} + end + + # Private helper to sanitize table names for MySQL + defp sanitize_table_name(name) do + name + |> String.replace(~r/[^a-zA-Z0-9_]/, "_") + |> String.replace(~r/^[0-9]/, "_\\0") # Ensure it doesn't start with a number + |> String.slice(0, 64) # MySQL table name limit + end +end diff --git a/lib/sequin/runtime/routing/routing.ex b/lib/sequin/runtime/routing/routing.ex index 15b4c3479..d86954022 100644 --- a/lib/sequin/runtime/routing/routing.ex +++ b/lib/sequin/runtime/routing/routing.ex @@ -95,6 +95,7 @@ defmodule Sequin.Runtime.Routing do :kinesis -> Sequin.Runtime.Routing.Consumers.Kinesis :typesense -> Sequin.Runtime.Routing.Consumers.Typesense :meilisearch -> Sequin.Runtime.Routing.Consumers.Meilisearch + :mysql -> Sequin.Runtime.Routing.Consumers.Mysql :elasticsearch -> Sequin.Runtime.Routing.Consumers.Elasticsearch :rabbitmq -> Sequin.Runtime.Routing.Consumers.Rabbitmq :sqs -> Sequin.Runtime.Routing.Consumers.Sqs diff --git a/lib/sequin/transforms/transforms.ex b/lib/sequin/transforms/transforms.ex index 1e236f135..7a0f64261 100644 --- a/lib/sequin/transforms/transforms.ex +++ b/lib/sequin/transforms/transforms.ex @@ -433,7 +433,8 @@ defmodule Sequin.Transforms do ssl: sink.ssl, batch_size: sink.batch_size, timeout_seconds: sink.timeout_seconds, - upsert_on_duplicate: sink.upsert_on_duplicate + upsert_on_duplicate: sink.upsert_on_duplicate, + routing_mode: sink.routing_mode }) end @@ -1313,7 +1314,8 @@ defmodule Sequin.Transforms do ssl: attrs["ssl"], batch_size: attrs["batch_size"], timeout_seconds: attrs["timeout_seconds"], - upsert_on_duplicate: attrs["upsert_on_duplicate"] + upsert_on_duplicate: attrs["upsert_on_duplicate"], + routing_mode: parse_routing_mode(attrs["routing_mode"]) }} end @@ -1475,4 +1477,9 @@ defmodule Sequin.Transforms do defp parse_auth_type("basic"), do: :basic defp parse_auth_type("bearer"), do: :bearer defp parse_auth_type(nil), do: :api_key + + # Helper to parse routing_mode + defp parse_routing_mode("dynamic"), do: :dynamic + defp parse_routing_mode("static"), do: :static + defp parse_routing_mode(nil), do: :static end From 2892c00e81ddc969c2bde428135accdf4ed0608c Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Tue, 29 Jul 2025 14:50:39 -0400 Subject: [PATCH 03/30] feat: add MySQL sink support and connection handling --- assets/svelte/sinks/mysql/MysqlIcon.svelte | 23 ++ .../svelte/sinks/mysql/MysqlSinkCard.svelte | 123 +++++++++ .../svelte/sinks/mysql/MysqlSinkForm.svelte | 243 ++++++++++++++++++ docs/docs.json | 1 + docs/reference/sequin-yaml.mdx | 18 ++ docs/reference/sinks/mysql.mdx | 183 +++++++++++++ lib/sequin/application.ex | 1 + lib/sequin/consumers/mysql_sink.ex | 8 + lib/sequin/runtime/mysql_pipeline.ex | 8 +- lib/sequin/sinks/mysql.ex | 28 ++ lib/sequin/sinks/mysql/client.ex | 45 +--- lib/sequin/sinks/mysql/connection_cache.ex | 234 +++++++++++++++++ test/sequin/mysql_pipeline_test.exs | 217 ++++++++++++++++ test/support/mocks.ex | 4 + 14 files changed, 1101 insertions(+), 35 deletions(-) create mode 100644 assets/svelte/sinks/mysql/MysqlIcon.svelte create mode 100644 assets/svelte/sinks/mysql/MysqlSinkCard.svelte create mode 100644 assets/svelte/sinks/mysql/MysqlSinkForm.svelte create mode 100644 docs/reference/sinks/mysql.mdx create mode 100644 lib/sequin/sinks/mysql.ex create mode 100644 lib/sequin/sinks/mysql/connection_cache.ex create mode 100644 test/sequin/mysql_pipeline_test.exs diff --git a/assets/svelte/sinks/mysql/MysqlIcon.svelte b/assets/svelte/sinks/mysql/MysqlIcon.svelte new file mode 100644 index 000000000..db2421ac0 --- /dev/null +++ b/assets/svelte/sinks/mysql/MysqlIcon.svelte @@ -0,0 +1,23 @@ + + + + + + + diff --git a/assets/svelte/sinks/mysql/MysqlSinkCard.svelte b/assets/svelte/sinks/mysql/MysqlSinkCard.svelte new file mode 100644 index 000000000..2d263cc11 --- /dev/null +++ b/assets/svelte/sinks/mysql/MysqlSinkCard.svelte @@ -0,0 +1,123 @@ + + + + + MySQL Configuration + + +
+
+ Host +
+
+ {consumer.sink.host}:{consumer.sink.port} +
+
+
+ +
+ Database +
+ + {consumer.sink.database} + +
+
+ +
+ Username +
+ + {consumer.sink.username} + +
+
+ +
+ Connection +
+ + {consumer.sink.ssl ? "SSL Enabled" : "SSL Disabled"} + +
+
+ +
+ Batch Size +
+ + {consumer.sink.batch_size} + +
+
+ +
+ Upsert Mode +
+ + {consumer.sink.upsert_on_duplicate ? "Enabled" : "Disabled"} + +
+
+
+ + + + + + Routing + + +
+
+ Table Name +
+ + {#if consumer.routing_id} + determined-by-router + {:else} + {consumer.sink.table_name} + {/if} + +
+
+
+ + {#if consumer.routing} +
+ Router +
+ 
{consumer.routing.function.code}
+
+
+ {/if} + + diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte new file mode 100644 index 000000000..7d6ba66cf --- /dev/null +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -0,0 +1,243 @@ + + + + + MySQL configuration + + + + Transform requirements + +

+ Your transform + must return data matching your MySQL table schema. +

+

+ Ensure your transform returns key-value pairs where keys match your + table's column names and values are appropriately typed. +

+ + + +
+
+ + + {#if errors.sink?.host} +

{errors.sink.host}

+ {/if} +

+ MySQL server hostname or IP address +

+
+ +
+ + + {#if errors.sink?.port} +

{errors.sink.port}

+ {/if} +

+ MySQL server port (default: 3306) +

+
+
+ +
+ + + {#if errors.sink?.database} +

{errors.sink.database}

+ {/if} +

+ Name of the target MySQL database +

+
+ +
+
+ + + {#if errors.sink?.username} +

{errors.sink.username}

+ {/if} +
+ +
+ +
+ + +
+ {#if errors.sink?.password} +

{errors.sink.password}

+ {/if} +
+
+ +
+
+ + +
+ {#if errors.sink?.ssl} +

{errors.sink.ssl}

+ {/if} +
+ +
+
+ + +
+ {#if errors.sink?.upsert_on_duplicate} +

+ {errors.sink.upsert_on_duplicate} +

+ {/if} +

+ When enabled, uses MySQL's ON DUPLICATE KEY UPDATE to handle existing + records. When disabled, attempts to insert records directly (may fail on + duplicates). +

+
+ +
+
+ + + {#if errors.sink?.batch_size} +

{errors.sink.batch_size}

+ {/if} +

+ Number of records to process in each batch (1-10,000) +

+
+ +
+ + + {#if errors.sink?.timeout_seconds} +

{errors.sink.timeout_seconds}

+ {/if} +

+ Connection timeout in seconds (1-300) +

+
+
+ + + + + + Routing + + + + + {#if !selectedDynamic} +
+ + + {#if errors.sink?.table_name} +

{errors.sink.table_name}

+ {/if} +

+ Name of the target MySQL table. Must be a valid MySQL identifier. +

+
+ {/if} + + diff --git a/docs/docs.json b/docs/docs.json index 877ea52c8..bce5950fc 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -114,6 +114,7 @@ "reference/sinks/gcp-pubsub", "reference/sinks/kafka", "reference/sinks/meilisearch", + "reference/sinks/mysql", "reference/sinks/nats", "reference/sinks/rabbitmq", "reference/sinks/redis-string", diff --git a/docs/reference/sequin-yaml.mdx b/docs/reference/sequin-yaml.mdx index f21a8cd71..4df1c8330 100644 --- a/docs/reference/sequin-yaml.mdx +++ b/docs/reference/sequin-yaml.mdx @@ -304,6 +304,24 @@ destination: batch_size: 100 # Optional, messages per batch, default: 100, max: 10000 ``` +#### MySQL sink +For writing data to MySQL tables: +```yaml +destination: + type: "mysql" # Required + host: "localhost" # Required, MySQL server hostname + port: 3306 # Optional, default: 3306 + database: "my_database" # Required + table_name: "products" # Required (used as fallback when routing_mode is dynamic) + username: "mysql_user" # Required + password: "mysql_password" # Required + ssl: false # Optional, enable SSL connection, default: false + batch_size: 100 # Optional, records per batch, default: 100, max: 10000 + timeout_seconds: 30 # Optional, connection timeout, default: 30, max: 300 + upsert_on_duplicate: true # Optional, use ON DUPLICATE KEY UPDATE, default: true + routing_mode: "static" # Optional: static or dynamic, default: static +``` + #### Sequin Stream sink [Sequin Stream](/reference/sinks/sequin-stream) is a durable, scalable, and fault-tolerant message stream that you can use with Sequin in place of additional infrastructure like Kafka or SQS. diff --git a/docs/reference/sinks/mysql.mdx b/docs/reference/sinks/mysql.mdx new file mode 100644 index 000000000..075f99254 --- /dev/null +++ b/docs/reference/sinks/mysql.mdx @@ -0,0 +1,183 @@ +--- +title: "MySQL sink" +sidebarTitle: "MySQL sink" +description: "Stream Postgres changes directly to MySQL with Sequin's MySQL sink." +--- + +The **MySQL sink** writes change data into MySQL tables using direct SQL operations. + + + This is the reference for the MySQL sink. See the [how-to guide](/how-to/stream-postgres-to-mysql) for a step-by-step walkthrough. + + +## Configuration + +- **Host** + + Hostname or IP address of your MySQL server (for example, `localhost` or `mysql.example.com`). + +- **Port** *(optional)* + + Port number of your MySQL server (default `3306`). + +- **Database** + + Name of the MySQL database to connect to. + +- **Table name** + + Name of the MySQL table to write to. The table **must exist** before Sequin can stream data. + +- **Username** + + MySQL username for authentication. + +- **Password** + + MySQL password for authentication. + +- **SSL/TLS** *(optional)* + + Enable SSL/TLS encryption for the connection (default `false`). + +- **Upsert on duplicate** *(optional)* + + When enabled, uses MySQL's `ON DUPLICATE KEY UPDATE` to handle existing records. When disabled, attempts direct inserts (default `true`). + +- **Batch size** *(optional)* + + Maximum number of records per batch operation (default `100`, maximum `10,000`). + +- **Timeout** *(optional)* + + Connection timeout in seconds (default `30`, maximum `300`). + +## Transform requirements + +Your [transform](/reference/transforms) must return a map where keys correspond to MySQL table column names and values are properly typed for the target columns. + +For complex data types like JSON objects or arrays, Sequin will automatically encode them as JSON strings when inserting into MySQL. + +### Example transforms + +Insert the full record: +```elixir +def transform(_action, record, _changes, _metadata) do + record +end +``` + +Map specific fields and rename columns: +```elixir +def transform(_action, record, _changes, _metadata) do + %{ + id: record["product_id"], + name: record["product_name"], + description: record["product_description"], + updated_at: record["modified_at"] + } +end +``` + +Handle JSON data: +```elixir +def transform(_action, record, _changes, _metadata) do + %{ + id: record["id"], + name: record["name"], + metadata: %{ + tags: record["tags"], + attributes: record["attributes"] + } + } +end +``` + +## SQL operations + +Sequin uses the following SQL operations based on the change action: + +| Change action | SQL operation | Behavior | +|---------------|--------------|----------| +| `INSERT`, `UPDATE`, `READ` | `INSERT ... ON DUPLICATE KEY UPDATE` (if upsert enabled) or `INSERT` | Creates or updates the record | +| `DELETE` | `DELETE` | Removes the record by primary key | + +### Upsert behavior + +When **upsert on duplicate** is enabled (default), Sequin uses MySQL's `ON DUPLICATE KEY UPDATE` clause: + +```sql +INSERT INTO products (id, name, price) +VALUES (1, 'Product Name', 19.99) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + price = VALUES(price) +``` + +When disabled, Sequin performs direct inserts, which may fail if records with duplicate primary keys exist. + +## Connection management + +Sequin maintains a connection pool to your MySQL server for optimal performance. Connections are cached and reused across multiple operations to minimize connection overhead. + +## Error handling + +Common errors include: + +- Connection failures (network, authentication) +- SQL syntax errors (usually from transform output) +- Constraint violations (foreign key, unique constraints) +- Data type mismatches +- Table or column not found + +Errors are displayed in the Sequin console's **Messages** tab with detailed MySQL error information. + +## Limits + +- **Batch size** is limited to `10,000` records by Sequin +- **Timeout** is limited to `300` seconds (5 minutes) +- One sink targets one table; multiple tables require multiple sinks (or dynamic routing) +- Table and column names must be valid MySQL identifiers + +## Routing + +The MySQL sink supports dynamic routing of the `table_name` with [routing functions](/reference/routing). + +Example routing function to route to different tables based on record type: + +```elixir +def route(action, record, changes, metadata) do + # Route based on record type + table_name = case record["event_type"] do + "user_signup" -> "user_events" + "purchase" -> "purchase_events" + "analytics" -> "analytics_events" + _ -> "other_events" + end + + %{table_name: table_name} +end +``` + +Route based on source table with schema prefix: + +```elixir +def route(action, record, changes, metadata) do + table_name = if metadata.table_schema do + "#{metadata.table_schema}_#{metadata.table_name}" + else + metadata.table_name + end + + %{table_name: table_name} +end +``` + +When not using a routing function, messages will be inserted into the statically configured table. + +## Requirements + +- MySQL 5.7 or later +- Target tables must exist before streaming begins +- Appropriate permissions for `INSERT`, `UPDATE`, `DELETE` operations on target tables +- For SSL connections, ensure your MySQL server supports SSL/TLS \ No newline at end of file diff --git a/lib/sequin/application.ex b/lib/sequin/application.ex index 43ec602de..a5843f0da 100644 --- a/lib/sequin/application.ex +++ b/lib/sequin/application.ex @@ -88,6 +88,7 @@ defmodule Sequin.Application do Sequin.Databases.ConnectionCache, Sequin.Sinks.Redis.ConnectionCache, Sequin.Sinks.Kafka.ConnectionCache, + Sequin.Sinks.Mysql.ConnectionCache, Sequin.Sinks.Nats.ConnectionCache, Sequin.Sinks.RabbitMq.ConnectionCache, SequinWeb.Presence, diff --git a/lib/sequin/consumers/mysql_sink.ex b/lib/sequin/consumers/mysql_sink.ex index 88cb44618..6cebbc2bb 100644 --- a/lib/sequin/consumers/mysql_sink.ex +++ b/lib/sequin/consumers/mysql_sink.ex @@ -24,6 +24,7 @@ defmodule Sequin.Consumers.MysqlSink do field(:timeout_seconds, :integer, default: 30) field(:upsert_on_duplicate, :boolean, default: true) field(:routing_mode, Ecto.Enum, values: [:dynamic, :static], default: :static) + field(:connection_id, :string, virtual: true) end def changeset(struct, params) do @@ -59,6 +60,13 @@ defmodule Sequin.Consumers.MysqlSink do ) end + @doc """ + Generate a unique connection ID for caching purposes. + """ + def connection_id(%__MODULE__{} = sink) do + "#{sink.host}:#{sink.port}/#{sink.database}@#{sink.username}" + end + def connection_opts(%__MODULE__{} = sink) do opts = [ hostname: sink.host, diff --git a/lib/sequin/runtime/mysql_pipeline.ex b/lib/sequin/runtime/mysql_pipeline.ex index 4aa5bf5e9..ac9f48e86 100644 --- a/lib/sequin/runtime/mysql_pipeline.ex +++ b/lib/sequin/runtime/mysql_pipeline.ex @@ -6,7 +6,7 @@ defmodule Sequin.Runtime.MysqlPipeline do alias Sequin.Runtime.Routing alias Sequin.Runtime.SinkPipeline alias Sequin.Runtime.Trace - alias Sequin.Sinks.Mysql.Client + alias Sequin.Sinks.Mysql alias Sequin.Transforms.Message @impl SinkPipeline @@ -70,7 +70,7 @@ defmodule Sequin.Runtime.MysqlPipeline do # Create a temporary sink with the routed table name routed_sink = %{sink | table_name: table_name} - case Client.upsert_records(routed_sink, records) do + case Mysql.upsert_records(routed_sink, records) do {:ok} -> Trace.info(consumer.id, %Trace.Event{ message: "Upserted records to MySQL table \"#{table_name}\"" @@ -106,7 +106,7 @@ defmodule Sequin.Runtime.MysqlPipeline do # Create a temporary sink with the routed table name routed_sink = %{sink | table_name: table_name} - case Client.delete_records(routed_sink, record_pks) do + case Mysql.delete_records(routed_sink, record_pks) do {:ok} -> Trace.info(consumer.id, %Trace.Event{ message: "Deleted records from MySQL table \"#{table_name}\"", @@ -141,7 +141,7 @@ defmodule Sequin.Runtime.MysqlPipeline do defp setup_allowances(nil), do: :ok defp setup_allowances(test_pid) do - Req.Test.allow(Client, test_pid, self()) + Mox.allow(Sequin.Sinks.MysqlMock, test_pid, self()) Mox.allow(Sequin.TestSupport.DateTimeMock, test_pid, self()) end end diff --git a/lib/sequin/sinks/mysql.ex b/lib/sequin/sinks/mysql.ex new file mode 100644 index 000000000..4c2ded26a --- /dev/null +++ b/lib/sequin/sinks/mysql.ex @@ -0,0 +1,28 @@ +defmodule Sequin.Sinks.Mysql do + @moduledoc false + alias Sequin.Consumers.MysqlSink + alias Sequin.Error + + @callback test_connection(MysqlSink.t()) :: :ok | {:error, Error.t()} + @callback upsert_records(MysqlSink.t(), [map()]) :: :ok | {:error, Error.t()} + @callback delete_records(MysqlSink.t(), [any()]) :: :ok | {:error, Error.t()} + + @spec test_connection(MysqlSink.t()) :: :ok | {:error, Error.t()} + def test_connection(%MysqlSink{} = sink) do + impl().test_connection(sink) + end + + @spec upsert_records(MysqlSink.t(), [map()]) :: :ok | {:error, Error.t()} + def upsert_records(%MysqlSink{} = sink, records) when is_list(records) do + impl().upsert_records(sink, records) + end + + @spec delete_records(MysqlSink.t(), [any()]) :: :ok | {:error, Error.t()} + def delete_records(%MysqlSink{} = sink, record_pks) when is_list(record_pks) do + impl().delete_records(sink, record_pks) + end + + defp impl do + Application.get_env(:sequin, :mysql_module, Sequin.Sinks.Mysql.Client) + end +end diff --git a/lib/sequin/sinks/mysql/client.ex b/lib/sequin/sinks/mysql/client.ex index 795516169..1ab0395d5 100644 --- a/lib/sequin/sinks/mysql/client.ex +++ b/lib/sequin/sinks/mysql/client.ex @@ -5,6 +5,7 @@ defmodule Sequin.Sinks.Mysql.Client do alias Sequin.Consumers.MysqlSink alias Sequin.Error + alias Sequin.Sinks.Mysql.ConnectionCache require Logger @@ -12,19 +13,9 @@ defmodule Sequin.Sinks.Mysql.Client do Test the connection to the MySQL database. """ def test_connection(%MysqlSink{} = sink) do - case start_connection(sink) do - {:ok, pid} -> - try do - case MyXQL.query(pid, "SELECT 1", []) do - {:ok, _result} -> :ok - {:error, error} -> {:error, format_error(error)} - end - after - GenServer.stop(pid) - end - - {:error, error} -> - {:error, format_error(error)} + case ConnectionCache.test_connection(sink) do + :ok -> :ok + {:error, error} -> {:error, format_error(error)} end end @@ -35,21 +26,16 @@ defmodule Sequin.Sinks.Mysql.Client do if Enum.empty?(records) do {:ok} else - case start_connection(sink) do + case ConnectionCache.get_connection(sink) do {:ok, pid} -> try do - result = - if sink.upsert_on_duplicate do - insert_or_update_records(pid, sink, records) - else - insert_records(pid, sink, records) - end - - GenServer.stop(pid) - result + if sink.upsert_on_duplicate do + insert_or_update_records(pid, sink, records) + else + insert_records(pid, sink, records) + end rescue error -> - GenServer.stop(pid) {:error, format_error(error)} end @@ -66,7 +52,7 @@ defmodule Sequin.Sinks.Mysql.Client do if Enum.empty?(record_pks) do {:ok} else - case start_connection(sink) do + case ConnectionCache.get_connection(sink) do {:ok, pid} -> try do # Assume primary key is 'id' for simplicity @@ -81,8 +67,9 @@ defmodule Sequin.Sinks.Mysql.Client do {:error, error} -> {:error, format_error(error)} end - after - GenServer.stop(pid) + rescue + error -> + {:error, format_error(error)} end {:error, error} -> @@ -93,10 +80,6 @@ defmodule Sequin.Sinks.Mysql.Client do # Private functions - defp start_connection(%MysqlSink{} = sink) do - MyXQL.start_link(MysqlSink.connection_opts(sink)) - end - defp insert_or_update_records(pid, %MysqlSink{} = sink, records) do case build_upsert_query(sink, records) do {:ok, {sql, params}} -> diff --git a/lib/sequin/sinks/mysql/connection_cache.ex b/lib/sequin/sinks/mysql/connection_cache.ex new file mode 100644 index 000000000..af4e4d1ac --- /dev/null +++ b/lib/sequin/sinks/mysql/connection_cache.ex @@ -0,0 +1,234 @@ +defmodule Sequin.Sinks.Mysql.ConnectionCache do + @moduledoc """ + Cache connections to customer MySQL instances. + + By caching these connections, we can avoid paying a significant startup + penalty when performing multiple operations on the same MySQL instance. + + Each `Sequin.Consumers.MysqlSink` gets its own connection in the cache. + + The cache takes ownership of the MySQL connections and is responsible for + closing them when they are invalidated (or when the cache is stopped). Thus, + callers should not call `GenServer.stop/1` on these connections. + + Cached connections are invalidated and recreated when their MySQL sink's + connection options change. + + The cache will detect dead connections and create new ones as needed. + """ + + use GenServer + + alias Sequin.Consumers.MysqlSink + alias Sequin.Error.NotFoundError + + require Logger + + defmodule Cache do + @moduledoc false + + @type sink :: MysqlSink.t() + @type entry :: %{ + conn: pid(), + options_hash: binary() + } + @type t :: %{binary() => entry()} + + @spec new :: t() + def new, do: %{} + + @spec each(t(), (pid() -> any())) :: :ok + def each(cache, function) do + Enum.each(cache, fn {_id, entry} -> function.(entry.conn) end) + end + + @spec lookup(t(), sink()) :: {:ok, pid()} | {:error, :stale} | {:error, :not_found} + def lookup(cache, sink) do + new_hash = options_hash(sink) + entry = Map.get(cache, MysqlSink.connection_id(sink)) + + cond do + is_nil(entry) -> + {:error, :not_found} + + is_pid(entry.conn) and !Process.alive?(entry.conn) -> + Logger.warning("Cached MySQL connection was dead upon lookup", sink_id: MysqlSink.connection_id(sink)) + {:error, :not_found} + + entry.options_hash != new_hash -> + Logger.info("Cached MySQL sink connection was stale", sink_id: MysqlSink.connection_id(sink)) + {:error, :stale} + + true -> + {:ok, entry.conn} + end + end + + @spec pop(t(), sink()) :: {pid() | nil, t()} + def pop(cache, sink) do + {entry, new_cache} = Map.pop(cache, MysqlSink.connection_id(sink), nil) + + if entry, do: {entry.conn, new_cache}, else: {nil, new_cache} + end + + @spec store(t(), sink(), pid()) :: t() + def store(cache, sink, conn) do + entry = %{conn: conn, options_hash: options_hash(sink)} + Map.put(cache, MysqlSink.connection_id(sink), entry) + end + + defp options_hash(%MysqlSink{} = sink) do + :erlang.phash2(MysqlSink.connection_opts(sink)) + end + end + + defmodule State do + @moduledoc false + use TypedStruct + + alias Sequin.Error + + @type sink :: MysqlSink.t() + @type opt :: {:start_fn, State.start_function()} | {:stop_fn, State.stop_function()} + @type start_function :: (sink() -> start_result()) + @type start_result :: {:ok, pid()} | {:error, Error.t()} + @type stop_function :: (pid() -> :ok) + + typedstruct do + field :cache, Cache.t(), default: Cache.new() + field :start_fn, start_function() + field :stop_fn, stop_function() + end + + @spec new([opt]) :: t() + def new(opts) do + start_fn = Keyword.get(opts, :start_fn, &default_start/1) + stop_fn = Keyword.get(opts, :stop_fn, &GenServer.stop/1) + + %__MODULE__{ + start_fn: start_fn, + stop_fn: stop_fn + } + end + + @spec find_or_create_connection(t(), sink(), boolean()) :: {:ok, pid(), t()} | {:error, term()} + def find_or_create_connection(%__MODULE__{} = state, sink, create_on_miss) do + case Cache.lookup(state.cache, sink) do + {:ok, conn} -> + {:ok, conn, state} + + {:error, :stale} -> + state + |> invalidate_connection(sink) + |> find_or_create_connection(sink, create_on_miss) + + {:error, :not_found} when create_on_miss -> + with {:ok, conn} <- state.start_fn.(sink) do + new_cache = Cache.store(state.cache, sink, conn) + new_state = %{state | cache: new_cache} + {:ok, conn, new_state} + end + + {:error, :not_found} -> + {:error, :not_found} + end + end + + @spec invalidate_all(t()) :: t() + def invalidate_all(%__MODULE__{} = state) do + Cache.each(state.cache, state.stop_fn) + + %{state | cache: Cache.new()} + end + + @spec invalidate_connection(t(), sink()) :: t() + def invalidate_connection(%__MODULE__{} = state, sink) do + {conn, new_cache} = Cache.pop(state.cache, sink) + + if conn, do: state.stop_fn.(conn) + + %{state | cache: new_cache} + end + + defp default_start(%MysqlSink{} = sink) do + MyXQL.start_link(MysqlSink.connection_opts(sink)) + end + end + + ## GenServer API + + @spec start_link(keyword()) :: GenServer.on_start() + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @spec get_connection(MysqlSink.t()) :: {:ok, pid()} | {:error, term()} + def get_connection(%MysqlSink{} = sink) do + GenServer.call(__MODULE__, {:get_connection, sink}) + end + + @spec test_connection(MysqlSink.t()) :: :ok | {:error, term()} + def test_connection(%MysqlSink{} = sink) do + GenServer.call(__MODULE__, {:test_connection, sink}) + end + + @spec invalidate(MysqlSink.t()) :: :ok + def invalidate(%MysqlSink{} = sink) do + GenServer.cast(__MODULE__, {:invalidate, sink}) + end + + @spec invalidate_all :: :ok + def invalidate_all do + GenServer.cast(__MODULE__, :invalidate_all) + end + + ## GenServer callbacks + + @impl GenServer + def init(opts) do + {:ok, State.new(opts)} + end + + @impl GenServer + def handle_call({:get_connection, sink}, _from, state) do + case State.find_or_create_connection(state, sink, true) do + {:ok, conn, new_state} -> + {:reply, {:ok, conn}, new_state} + + {:error, error} -> + Logger.error("Failed to get MySQL connection: #{inspect(error)}") + {:reply, {:error, error}, state} + end + end + + @impl GenServer + def handle_call({:test_connection, sink}, _from, state) do + case State.find_or_create_connection(state, sink, true) do + {:ok, conn, new_state} -> + case MyXQL.query(conn, "SELECT 1", []) do + {:ok, _result} -> + {:reply, :ok, new_state} + + {:error, error} -> + # Invalidate the connection on test failure + invalidated_state = State.invalidate_connection(new_state, sink) + {:reply, {:error, error}, invalidated_state} + end + + {:error, error} -> + {:reply, {:error, error}, state} + end + end + + @impl GenServer + def handle_cast({:invalidate, sink}, state) do + new_state = State.invalidate_connection(state, sink) + {:noreply, new_state} + end + + @impl GenServer + def handle_cast(:invalidate_all, state) do + new_state = State.invalidate_all(state) + {:noreply, new_state} + end +end diff --git a/test/sequin/mysql_pipeline_test.exs b/test/sequin/mysql_pipeline_test.exs new file mode 100644 index 000000000..0d7df3b52 --- /dev/null +++ b/test/sequin/mysql_pipeline_test.exs @@ -0,0 +1,217 @@ +defmodule Sequin.Runtime.MysqlPipelineTest do + use Sequin.DataCase, async: true + + import Mox + + alias Sequin.Factory.AccountsFactory + alias Sequin.Factory.ConsumersFactory + alias Sequin.Factory.FunctionsFactory + alias Sequin.Functions.MiniElixir + alias Sequin.Runtime.SinkPipeline + alias Sequin.Sinks.MysqlMock + + setup :verify_on_exit! + + describe "mysql pipeline" do + setup do + account = AccountsFactory.insert_account!() + + transform = + FunctionsFactory.insert_function!( + account_id: account.id, + function_type: :transform, + function_attrs: %{body: "record"} + ) + + MiniElixir.create(transform.id, transform.function.code) + + consumer = + ConsumersFactory.insert_sink_consumer!( + account_id: account.id, + type: :mysql, + message_kind: :event, + transform_id: transform.id + ) + + {:ok, %{consumer: consumer}} + end + + test "one message is upserted", %{consumer: consumer} do + message = + ConsumersFactory.consumer_message( + consumer_id: consumer.id, + message_kind: consumer.message_kind, + data: + ConsumersFactory.consumer_message_data( + message_kind: consumer.message_kind, + action: :insert, + record: %{"id" => 1, "name" => "test-name"} + ) + ) + + MysqlMock + |> expect(:upsert_records, fn sink, records -> + assert sink.table_name == consumer.sink.table_name + assert length(records) == 1 + assert List.first(records)["name"] == "test-name" + :ok + end) + + start_pipeline!(consumer) + + send_test_batch(consumer, [message]) + + assert_receive {:ack, _ref, [success], []}, 3000 + assert success.data == message + end + + test "multiple messages are batched and upserted", %{consumer: consumer} do + messages = + for i <- 1..3 do + ConsumersFactory.insert_consumer_message!( + consumer_id: consumer.id, + message_kind: consumer.message_kind, + data: + ConsumersFactory.consumer_message_data( + message_kind: consumer.message_kind, + action: :insert, + record: %{"id" => i, "name" => "test-name-#{i}"} + ) + ) + end + + MysqlMock + |> expect(:upsert_records, fn sink, records -> + assert sink.table_name == consumer.sink.table_name + assert length(records) == 3 + assert Enum.map(records, & &1["name"]) == ["test-name-1", "test-name-2", "test-name-3"] + :ok + end) + + start_pipeline!(consumer) + + send_test_batch(consumer, messages) + + assert_receive {:ack, _ref, successful, []}, 3000 + assert length(successful) == 3 + end + + test "delete action removes records", %{consumer: consumer} do + message = + ConsumersFactory.consumer_message( + consumer_id: consumer.id, + message_kind: consumer.message_kind, + data: + ConsumersFactory.consumer_message_data( + message_kind: consumer.message_kind, + action: :delete, + record: %{"id" => 1}, + record_pks: [1] + ) + ) + + MysqlMock + |> expect(:delete_records, fn sink, record_pks -> + assert sink.table_name == consumer.sink.table_name + assert record_pks == [1] + :ok + end) + + start_pipeline!(consumer) + + send_test_batch(consumer, [message]) + + assert_receive {:ack, _ref, [success], []}, 3000 + assert success.data == message + end + + test "handles mysql errors gracefully", %{consumer: consumer} do + message = + ConsumersFactory.consumer_message( + consumer_id: consumer.id, + message_kind: consumer.message_kind, + data: + ConsumersFactory.consumer_message_data( + message_kind: consumer.message_kind, + action: :insert, + record: %{"id" => 1, "name" => "test-name"} + ) + ) + + error = Sequin.Error.service(service: :mysql, message: "Connection failed") + + MysqlMock + |> expect(:upsert_records, fn _sink, _records -> + {:error, error} + end) + + start_pipeline!(consumer) + + send_test_batch(consumer, [message]) + + assert_receive {:failed, _ref, [failed], []}, 3000 + assert failed.data == message + end + + test "dynamic routing uses correct table name", %{consumer: consumer} do + # Update consumer to use dynamic routing + routing = + FunctionsFactory.insert_function!( + account_id: consumer.account_id, + function_type: :routing, + function_attrs: %{body: ~s|%{table_name: "dynamic_table"}|} + ) + + MiniElixir.create(routing.id, routing.function.code) + + consumer = %{consumer | routing_id: routing.id, routing: routing} + + message = + ConsumersFactory.consumer_message( + consumer_id: consumer.id, + message_kind: consumer.message_kind, + data: + ConsumersFactory.consumer_message_data( + message_kind: consumer.message_kind, + action: :insert, + record: %{"id" => 1, "name" => "test-name"} + ) + ) + + MysqlMock + |> expect(:upsert_records, fn sink, records -> + assert sink.table_name == "dynamic_table" + assert length(records) == 1 + :ok + end) + + start_pipeline!(consumer) + + send_test_batch(consumer, [message]) + + assert_receive {:ack, _ref, [success], []}, 3000 + assert success.data == message + end + end + + defp start_pipeline!(consumer) do + Application.put_env(:sequin, :mysql_module, MysqlMock) + + start_supervised!( + {SinkPipeline, + [ + consumer_id: consumer.id, + producer: Broadway.DummyProducer, + test_pid: self() + ]} + ) + end + + defp send_test_batch(consumer, events) do + Broadway.test_batch(broadway(consumer), events) + end + + defp broadway(consumer) do + SinkPipeline.via_tuple(consumer.id) + end +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index 75861732d..2716c9cf2 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -18,6 +18,10 @@ Mox.defmock(Sequin.Sinks.KafkaMock, for: Sequin.Sinks.Kafka ) +Mox.defmock(Sequin.Sinks.MysqlMock, + for: Sequin.Sinks.Mysql +) + Mox.defmock(Sequin.Runtime.TableReaderServerMock, for: Sequin.Runtime.TableReaderServer ) From 91e9aa7aed555fd0574a4776826c0553d1cf51b4 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 31 Jul 2025 16:39:51 -0400 Subject: [PATCH 04/30] feat: enhance MySQL sink integration with routing documentation and UI components --- .../svelte/consumers/SinkConsumerForm.svelte | 9 ++++++ assets/svelte/consumers/SinkIndex.svelte | 6 ++++ assets/svelte/consumers/dynamicRoutingDocs.ts | 10 ++++++ assets/svelte/consumers/types.ts | 26 +++++++++++++-- assets/svelte/sinks/mysql/MysqlIcon.svelte | 32 ++++++++++--------- lib/sequin/sinks/mysql/connection_cache.ex | 1 - 6 files changed, 66 insertions(+), 18 deletions(-) diff --git a/assets/svelte/consumers/SinkConsumerForm.svelte b/assets/svelte/consumers/SinkConsumerForm.svelte index a544d4238..4146c5d51 100644 --- a/assets/svelte/consumers/SinkConsumerForm.svelte +++ b/assets/svelte/consumers/SinkConsumerForm.svelte @@ -36,6 +36,7 @@ import TypesenseSinkForm from "$lib/sinks/typesense/TypesenseSinkForm.svelte"; import MeilisearchSinkForm from "$lib/sinks/meilisearch/MeilisearchSinkForm.svelte"; import ElasticsearchSinkForm from "$lib/sinks/elasticsearch/ElasticsearchSinkForm.svelte"; + import MysqlSinkForm from "$lib/sinks/mysql/MysqlSinkForm.svelte"; import * as Alert from "$lib/components/ui/alert/index.js"; import SchemaTableSelector from "../components/SchemaTableSelector.svelte"; import * as Tooltip from "$lib/components/ui/tooltip"; @@ -797,6 +798,14 @@ {refreshFunctions} bind:functionRefreshState /> + {:else if consumer.type === "mysql"} + {/if} diff --git a/assets/svelte/consumers/SinkIndex.svelte b/assets/svelte/consumers/SinkIndex.svelte index 9499e5a80..8454eb125 100644 --- a/assets/svelte/consumers/SinkIndex.svelte +++ b/assets/svelte/consumers/SinkIndex.svelte @@ -33,6 +33,7 @@ import TypesenseIcon from "../sinks/typesense/TypesenseIcon.svelte"; import MeilisearchIcon from "../sinks/meilisearch/MeilisearchIcon.svelte"; import ElasticsearchIcon from "../sinks/elasticsearch/ElasticsearchIcon.svelte"; + import MysqlIcon from "../sinks/mysql/MysqlIcon.svelte"; import { Badge } from "$lib/components/ui/badge"; import * as d3 from "d3"; @@ -164,6 +165,11 @@ name: "Elasticsearch", icon: ElasticsearchIcon, }, + { + id: "mysql", + name: "MySQL", + icon: MysqlIcon, + }, ]; function handleConsumerClick(id: string, type: string) { diff --git a/assets/svelte/consumers/dynamicRoutingDocs.ts b/assets/svelte/consumers/dynamicRoutingDocs.ts index aa646c966..a3274e641 100644 --- a/assets/svelte/consumers/dynamicRoutingDocs.ts +++ b/assets/svelte/consumers/dynamicRoutingDocs.ts @@ -231,4 +231,14 @@ export const routedSinkDocs: Record = { }, }, }, + mysql: { + fields: { + table_name: { + description: "MySQL table name to write records to", + staticValue: "", + staticFormField: "table_name", + dynamicDefault: "_", + }, + }, + }, }; diff --git a/assets/svelte/consumers/types.ts b/assets/svelte/consumers/types.ts index 77a482de4..8cc729d77 100644 --- a/assets/svelte/consumers/types.ts +++ b/assets/svelte/consumers/types.ts @@ -247,6 +247,24 @@ export type ElasticsearchConsumer = BaseConsumer & { }; }; +// MySQL specific sink +export type MysqlConsumer = BaseConsumer & { + sink: { + type: "mysql"; + host: string; + port: number; + database: string; + table_name: string; + username: string; + password?: string; + ssl: boolean; + batch_size: number; + timeout_seconds: number; + upsert_on_duplicate: boolean; + routing_mode: "static" | "dynamic"; + }; +}; + // Union type for all consumer types export type Consumer = | HttpPushConsumer @@ -263,6 +281,7 @@ export type Consumer = | TypesenseConsumer | SnsConsumer | ElasticsearchConsumer + | MysqlConsumer | RedisStringConsumer; export const SinkTypeValues = [ @@ -270,17 +289,19 @@ export const SinkTypeValues = [ "sqs", "sns", "kinesis", + "s2", "redis_stream", + "redis_string", "kafka", "sequin_stream", "gcp_pubsub", - "elasticsearch", "nats", "rabbitmq", + "azure_event_hub", "typesense", "meilisearch", "elasticsearch", - "redis_string", + "mysql", ] as const; export type SinkType = (typeof SinkTypeValues)[number]; @@ -301,6 +322,7 @@ export const RoutedSinkTypeValues = [ "sqs", "sns", "kinesis", + "mysql", ] as const; export type RoutedSinkType = (typeof RoutedSinkTypeValues)[number]; diff --git a/assets/svelte/sinks/mysql/MysqlIcon.svelte b/assets/svelte/sinks/mysql/MysqlIcon.svelte index db2421ac0..f7e30e656 100644 --- a/assets/svelte/sinks/mysql/MysqlIcon.svelte +++ b/assets/svelte/sinks/mysql/MysqlIcon.svelte @@ -1,23 +1,25 @@ - - - + + + diff --git a/lib/sequin/sinks/mysql/connection_cache.ex b/lib/sequin/sinks/mysql/connection_cache.ex index af4e4d1ac..f0008154f 100644 --- a/lib/sequin/sinks/mysql/connection_cache.ex +++ b/lib/sequin/sinks/mysql/connection_cache.ex @@ -20,7 +20,6 @@ defmodule Sequin.Sinks.Mysql.ConnectionCache do use GenServer alias Sequin.Consumers.MysqlSink - alias Sequin.Error.NotFoundError require Logger From 8a6c20c6b8575013359ba4db073a47701af6fb3d Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 31 Jul 2025 17:00:35 -0400 Subject: [PATCH 05/30] feat: implement MySQL connection testing and encoding/decoding for sink configuration --- .../live/components/consumer_form.ex | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/lib/sequin_web/live/components/consumer_form.ex b/lib/sequin_web/live/components/consumer_form.ex index 1420b5beb..fa0478cae 100644 --- a/lib/sequin_web/live/components/consumer_form.ex +++ b/lib/sequin_web/live/components/consumer_form.ex @@ -16,6 +16,7 @@ defmodule SequinWeb.Components.ConsumerForm do alias Sequin.Consumers.KafkaSink alias Sequin.Consumers.KinesisSink alias Sequin.Consumers.MeilisearchSink + alias Sequin.Consumers.MysqlSink alias Sequin.Consumers.NatsSink alias Sequin.Consumers.RabbitMqSink alias Sequin.Consumers.RedisStreamSink @@ -309,6 +310,12 @@ defmodule SequinWeb.Components.ConsumerForm do {:error, error} -> {:reply, %{ok: false, error: error}, socket} end + :mysql -> + case test_mysql_connection(socket) do + :ok -> {:reply, %{ok: true}, socket} + {:error, error} -> {:reply, %{ok: false, error: error}, socket} + end + :redis_string -> case test_redis_string_connection(socket) do :ok -> {:reply, %{ok: true}, socket} @@ -693,6 +700,27 @@ defmodule SequinWeb.Components.ConsumerForm do end end + defp test_mysql_connection(socket) do + sink_changeset = + socket.assigns.changeset + |> Ecto.Changeset.get_field(:sink) + |> case do + %Ecto.Changeset{} = changeset -> changeset + %MysqlSink{} = sink -> MysqlSink.changeset(sink, %{}) + end + + if sink_changeset.valid? do + sink = Ecto.Changeset.apply_changes(sink_changeset) + + case Sequin.Sinks.Mysql.test_connection(sink) do + :ok -> :ok + {:error, error} -> {:error, Exception.message(error)} + end + else + {:error, encode_errors(sink_changeset)} + end + end + defp decode_params(form, socket) do sink = decode_sink(socket.assigns.consumer.type, form["sink"]) @@ -863,6 +891,22 @@ defmodule SequinWeb.Components.ConsumerForm do } end + defp decode_sink(:mysql, sink) do + %{ + "type" => "mysql", + "host" => sink["host"], + "port" => sink["port"], + "database" => sink["database"], + "table_name" => sink["table_name"], + "username" => sink["username"], + "password" => sink["password"], + "ssl" => sink["ssl"], + "batch_size" => sink["batch_size"], + "timeout_seconds" => sink["timeout_seconds"], + "upsert_on_duplicate" => sink["upsert_on_duplicate"] + } + end + defp decode_sink(:nats, sink) do %{ "type" => "nats", @@ -1224,6 +1268,22 @@ defmodule SequinWeb.Components.ConsumerForm do } end + defp encode_sink(%MysqlSink{} = sink) do + %{ + "type" => "mysql", + "host" => sink.host, + "port" => sink.port, + "database" => sink.database, + "table_name" => sink.table_name, + "username" => sink.username, + "ssl" => sink.ssl, + "batch_size" => sink.batch_size, + "timeout_seconds" => sink.timeout_seconds, + "upsert_on_duplicate" => sink.upsert_on_duplicate, + "routing_mode" => sink.routing_mode + } + end + defp encode_errors(nil), do: %{} defp encode_errors(%Ecto.Changeset{} = changeset) do @@ -1451,6 +1511,7 @@ defmodule SequinWeb.Components.ConsumerForm do :s2 -> "S2 Sink" :sequin_stream -> "Sequin Stream Sink" :gcp_pubsub -> "GCP Pub/Sub Sink" + :mysql -> "MySQL Sink" :nats -> "NATS Sink" :rabbitmq -> "RabbitMQ Sink" :azure_event_hub -> "Azure Event Hub Sink" @@ -1489,6 +1550,7 @@ defmodule SequinWeb.Components.ConsumerForm do :typesense -> {%TypesenseSink{}, %{}} :meilisearch -> {%MeilisearchSink{}, %{}} :elasticsearch -> {%ElasticsearchSink{}, %{}} + :mysql -> {%MysqlSink{}, %{batch_size: 100}} :redis_string -> {%RedisStringSink{}, %{batch_size: 10}} end From 4f5b1cb963056c945e44ce4ee5065c40c4529843 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 31 Jul 2025 17:47:54 -0400 Subject: [PATCH 06/30] feat: add MySQL service to docker-compose and implement MyXQL error encoding --- docker-compose.yaml | 14 ++++++++++++++ lib/sequin/myxql/encoders.ex | 14 ++++++++++++++ lib/sequin_web/live/sink_consumers/show.ex | 17 +++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 lib/sequin/myxql/encoders.ex diff --git a/docker-compose.yaml b/docker-compose.yaml index 088b6f264..f5d5b7b13 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,19 @@ name: sequin-dev services: + mysql: + profiles: [databases] + image: mysql:8.0 + ports: + - "127.0.0.1:3307:3306" + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: demo + MYSQL_PASSWORD: demo + MYSQL_DATABASE: demo + volumes: + - sequin_dev_mysql:/var/lib/mysql + postgres: profiles: [databases] build: @@ -161,3 +174,4 @@ volumes: sequin_dev_elasticsearch: prometheus_data: grafana_data: + sequin_dev_mysql: \ No newline at end of file diff --git a/lib/sequin/myxql/encoders.ex b/lib/sequin/myxql/encoders.ex new file mode 100644 index 000000000..1b892380e --- /dev/null +++ b/lib/sequin/myxql/encoders.ex @@ -0,0 +1,14 @@ +defimpl Jason.Encoder, for: MyXQL.Error do + def encode(%MyXQL.Error{} = error, opts) do + Jason.Encoder.encode( + %{ + type: :mysql_error, + message: error.message, + connection_id: error.connection_id, + mysql: error.mysql, + statement: error.statement + }, + opts + ) + end +end diff --git a/lib/sequin_web/live/sink_consumers/show.ex b/lib/sequin_web/live/sink_consumers/show.ex index d562cc9b6..ea36f8fbb 100644 --- a/lib/sequin_web/live/sink_consumers/show.ex +++ b/lib/sequin_web/live/sink_consumers/show.ex @@ -21,6 +21,7 @@ defmodule SequinWeb.SinkConsumersLive.Show do alias Sequin.Consumers.KafkaSink alias Sequin.Consumers.KinesisSink alias Sequin.Consumers.MeilisearchSink + alias Sequin.Consumers.MysqlSink alias Sequin.Consumers.NatsSink alias Sequin.Consumers.PathFunction alias Sequin.Consumers.RabbitMqSink @@ -1001,6 +1002,21 @@ defmodule SequinWeb.SinkConsumersLive.Show do } end + defp encode_sink(%SinkConsumer{sink: %MysqlSink{} = sink}) do + %{ + type: :mysql, + host: sink.host, + port: sink.port, + database: sink.database, + table_name: sink.table_name, + username: sink.username, + ssl: sink.ssl, + batch_size: sink.batch_size, + timeout_seconds: sink.timeout_seconds, + upsert_on_duplicate: sink.upsert_on_duplicate + } + end + defp encode_sink(%SinkConsumer{sink: %ElasticsearchSink{} = sink}) do %{ type: :elasticsearch, @@ -1394,6 +1410,7 @@ defmodule SequinWeb.SinkConsumersLive.Show do defp consumer_title(%{sink: %{type: :sqs}}), do: "SQS Sink" defp consumer_title(%{sink: %{type: :typesense}}), do: "Typesense Sink" defp consumer_title(%{sink: %{type: :meilisearch}}), do: "Meilisearch Sink" + defp consumer_title(%{sink: %{type: :mysql}}), do: "MySQL Sink" defp put_health(%SinkConsumer{} = consumer) do with {:ok, health} <- Health.health(consumer), From 3acccc23c59a6519df41e052effbf8e76b146769 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 11:44:48 -0400 Subject: [PATCH 07/30] feat: integrate MySQL sink components into ShowSink and update MysqlSinkCard and MysqlSinkForm --- assets/svelte/consumers/ShowSink.svelte | 8 ++++++++ assets/svelte/sinks/mysql/MysqlSinkCard.svelte | 5 ++--- assets/svelte/sinks/mysql/MysqlSinkForm.svelte | 8 +++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/assets/svelte/consumers/ShowSink.svelte b/assets/svelte/consumers/ShowSink.svelte index f3c7a4447..6037a2540 100644 --- a/assets/svelte/consumers/ShowSink.svelte +++ b/assets/svelte/consumers/ShowSink.svelte @@ -27,12 +27,14 @@ ElasticsearchConsumer, RedisStringConsumer, AzureEventHubConsumer, + MysqlConsumer, } from "./types"; import AzureEventHubSinkCard from "../sinks/azure_event_hub/AzureEventHubSinkCard.svelte"; import ElasticsearchSinkCard from "../sinks/elasticsearch/ElasticsearchSinkCard.svelte"; import GcpPubsubSinkCard from "../sinks/gcp_pubsub/GcpPubsubSinkCard.svelte"; import KafkaSinkCard from "../sinks/kafka/KafkaSinkCard.svelte"; import KinesisSinkCard from "../sinks/kinesis/KinesisSinkCard.svelte"; + import MysqlSinkCard from "../sinks/mysql/MysqlSinkCard.svelte"; import NatsSinkCard from "../sinks/nats/NatsSinkCard.svelte"; import RabbitMqSinkCard from "../sinks/rabbitmq/RabbitMqSinkCard.svelte"; import RedisStreamSinkCard from "../sinks/redis-stream/RedisStreamSinkCard.svelte"; @@ -161,6 +163,10 @@ return consumer.sink.type === "rabbitmq"; } + function isMysqlConsumer(consumer: Consumer): consumer is MysqlConsumer { + return consumer.sink.type === "mysql"; + } + let chartElement; let updateChart; let resizeObserver; @@ -1254,6 +1260,8 @@ {:else if isRedisStringConsumer(consumer)} + {:else if isMysqlConsumer(consumer)} + {/if} diff --git a/assets/svelte/sinks/mysql/MysqlSinkCard.svelte b/assets/svelte/sinks/mysql/MysqlSinkCard.svelte index 2d263cc11..21ae65d2b 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkCard.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkCard.svelte @@ -1,14 +1,13 @@ diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte index 7d6ba66cf..abec06a54 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -8,15 +8,17 @@ CardTitle, } from "$lib/components/ui/card"; import { Label } from "$lib/components/ui/label"; - import { Eye, EyeOff, Info } from "lucide-svelte"; + import { Eye, EyeOff } from "lucide-svelte"; import { Checkbox } from "$lib/components/ui/checkbox"; import DynamicRoutingForm from "$lib/consumers/DynamicRoutingForm.svelte"; - export let form: any; // MysqlConsumer type would be defined elsewhere + import type { MysqlConsumer } from "../../consumers/types"; + + export let form: MysqlConsumer; export let functions: Array = []; export let refreshFunctions: () => void; export let functionRefreshState: "idle" | "refreshing" | "done" = "idle"; - let selectedDynamic = form.routingMode === "dynamic"; + let selectedDynamic = form.sink.routing_mode === "dynamic"; export let errors: any = {}; let showPassword = false; From 653ddaff1c8c045d07a9c2f4d53ff103090beca0 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 11:45:04 -0400 Subject: [PATCH 08/30] refactor: remove unnecessary whitespace in MysqlSinkForm.svelte --- assets/svelte/sinks/mysql/MysqlSinkForm.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte index abec06a54..99a18f00b 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -11,7 +11,6 @@ import { Eye, EyeOff } from "lucide-svelte"; import { Checkbox } from "$lib/components/ui/checkbox"; import DynamicRoutingForm from "$lib/consumers/DynamicRoutingForm.svelte"; - import type { MysqlConsumer } from "../../consumers/types"; export let form: MysqlConsumer; From 78a354fe524db435658454cff4fe6f6518b55602 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:05:01 -0400 Subject: [PATCH 09/30] feat: add MysqlIcon to ShowSinkHeader for MySQL sink type --- assets/svelte/consumers/ShowSinkHeader.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/assets/svelte/consumers/ShowSinkHeader.svelte b/assets/svelte/consumers/ShowSinkHeader.svelte index c000a91af..88ea65c84 100644 --- a/assets/svelte/consumers/ShowSinkHeader.svelte +++ b/assets/svelte/consumers/ShowSinkHeader.svelte @@ -32,6 +32,7 @@ import AzureEventHubIcon from "../sinks/azure_event_hub/AzureEventHubIcon.svelte"; import TypesenseIcon from "../sinks/typesense/TypesenseIcon.svelte"; import ElasticsearchIcon from "../sinks/elasticsearch/ElasticsearchIcon.svelte"; + import MysqlIcon from "../sinks/mysql/MysqlIcon.svelte"; import StopSinkModal from "./StopSinkModal.svelte"; import { Badge } from "$lib/components/ui/badge"; @@ -186,6 +187,8 @@ {:else if consumer.sink.type === "s2"} + {:else if consumer.sink.type === "mysql"} + {/if}

{consumer.name} From fe48096f31eaf1694b0ff12421245b94a1479f37 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:09:52 -0400 Subject: [PATCH 10/30] feat: add MySQL to sink consumer types --- assets/svelte/consumers/SinkIndex.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assets/svelte/consumers/SinkIndex.svelte b/assets/svelte/consumers/SinkIndex.svelte index 8454eb125..4d3318fb7 100644 --- a/assets/svelte/consumers/SinkIndex.svelte +++ b/assets/svelte/consumers/SinkIndex.svelte @@ -57,7 +57,8 @@ | "nats" | "rabbitmq" | "typesense" - | "elasticsearch"; + | "elasticsearch" + | "mysql"; status: "active" | "disabled" | "paused"; database_name: string; From 0f812444fe5c635dc2fa4eccf74aa45b442b13dc Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:20:04 -0400 Subject: [PATCH 11/30] feat: enhance MysqlSinkCard with routing code display and UI improvements --- .../svelte/sinks/mysql/MysqlSinkCard.svelte | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/assets/svelte/sinks/mysql/MysqlSinkCard.svelte b/assets/svelte/sinks/mysql/MysqlSinkCard.svelte index 21ae65d2b..ba2d75fe7 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkCard.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkCard.svelte @@ -1,4 +1,5 @@ @@ -15,9 +25,9 @@ MySQL Configuration -
+
- Host + Host
- Database + Database
{consumer.sink.database} @@ -39,10 +49,10 @@
- Username + Username
{consumer.sink.username} @@ -50,21 +60,21 @@
- Connection + SSL Enabled
- {consumer.sink.ssl ? "SSL Enabled" : "SSL Disabled"} + {consumer.sink.ssl ? "Yes" : "No"}
- Batch Size + Batch Size
{consumer.sink.batch_size} @@ -72,10 +82,10 @@
- Upsert Mode + Upsert Mode
{consumer.sink.upsert_on_duplicate ? "Enabled" : "Disabled"} @@ -90,33 +100,38 @@ Routing -
-
- Table Name -
- - {#if consumer.routing_id} - determined-by-router - {:else} - {consumer.sink.table_name} - {/if} - -
+
+ Table +
+ + {#if consumer.routing_id} + Determined by router + + {:else} + {consumer.sink.table_name} + {/if} +
- {#if consumer.routing} -
- Router + {#if getRoutingCode(consumer)}
- 
{consumer.routing.function.code}
+ Router +
+ 
{getRoutingCode(consumer)}
+
-
+ {/if} {/if} From 753497393434ccf1ca24d605e516b550c4cec62a Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:26:09 -0400 Subject: [PATCH 12/30] refactor: replace Checkbox with Switch in MysqlSinkForm and update related bindings --- .../svelte/sinks/mysql/MysqlSinkForm.svelte | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte index 99a18f00b..07f29429d 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -9,7 +9,7 @@ } from "$lib/components/ui/card"; import { Label } from "$lib/components/ui/label"; import { Eye, EyeOff } from "lucide-svelte"; - import { Checkbox } from "$lib/components/ui/checkbox"; + import { Switch } from "$lib/components/ui/switch"; import DynamicRoutingForm from "$lib/consumers/DynamicRoutingForm.svelte"; import type { MysqlConsumer } from "../../consumers/types"; @@ -17,7 +17,7 @@ export let functions: Array = []; export let refreshFunctions: () => void; export let functionRefreshState: "idle" | "refreshing" | "done" = "idle"; - let selectedDynamic = form.sink.routing_mode === "dynamic"; + let isDynamicRouting = form.sink.routing_mode === "dynamic"; export let errors: any = {}; let showPassword = false; @@ -118,7 +118,7 @@ bind:value={form.sink.password} placeholder="••••••••" data-1p-ignore - autocomplete="current-password" + autocomplete="off" />
-
-
- - +
+
+ { + form.sink.ssl = checked; + }} + /> +
{#if errors.sink?.ssl}

{errors.sink.ssl}

{/if}
-
-
- +
+ { + form.sink.upsert_on_duplicate = checked; + }} />
@@ -219,11 +228,11 @@ {functions} {refreshFunctions} bind:functionRefreshState - bind:selectedDynamic + bind:selectedDynamic={isDynamicRouting} {errors} /> - {#if !selectedDynamic} + {#if !isDynamicRouting}
Date: Wed, 6 Aug 2025 12:42:25 -0400 Subject: [PATCH 13/30] docs: add dynamic routing example and enhance MySQL sink documentation --- docs/reference/sequin-yaml.mdx | 16 ++++++++++++++++ docs/reference/sinks/mysql.mdx | 19 +++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/reference/sequin-yaml.mdx b/docs/reference/sequin-yaml.mdx index 4df1c8330..5880a92a7 100644 --- a/docs/reference/sequin-yaml.mdx +++ b/docs/reference/sequin-yaml.mdx @@ -322,6 +322,22 @@ destination: routing_mode: "static" # Optional: static or dynamic, default: static ``` +**Dynamic routing example:** +```yaml +destination: + type: "mysql" + host: "localhost" + port: 3306 + database: "analytics" + table_name: "events" # Fallback table name + username: "mysql_user" + password: "mysql_password" + routing_mode: "dynamic" # Enable dynamic routing + # ... other configuration +``` + +When using `routing_mode: "dynamic"`, attach a [routing function](#routing-function-example) to dynamically determine the target table based on your data. + #### Sequin Stream sink [Sequin Stream](/reference/sinks/sequin-stream) is a durable, scalable, and fault-tolerant message stream that you can use with Sequin in place of additional infrastructure like Kafka or SQS. diff --git a/docs/reference/sinks/mysql.mdx b/docs/reference/sinks/mysql.mdx index 075f99254..6a64e9288 100644 --- a/docs/reference/sinks/mysql.mdx +++ b/docs/reference/sinks/mysql.mdx @@ -18,7 +18,7 @@ The **MySQL sink** writes change data into MySQL tables using direct SQL operati - **Port** *(optional)* - Port number of your MySQL server (default `3306`). + Port number of your MySQL server (default `3306`, range `1-65535`). - **Database** @@ -38,7 +38,13 @@ The **MySQL sink** writes change data into MySQL tables using direct SQL operati - **SSL/TLS** *(optional)* - Enable SSL/TLS encryption for the connection (default `false`). + Enable SSL/TLS encryption for secure connection to the MySQL server (default `false`). + +- **Routing mode** *(optional)* + + Determines how table routing is handled: + - `static` (default): Uses the configured table name for all records + - `dynamic`: Uses a [routing function](/reference/routing) to dynamically determine the target table - **Upsert on duplicate** *(optional)* @@ -134,8 +140,13 @@ Errors are displayed in the Sequin console's **Messages** tab with detailed MySQ ## Limits -- **Batch size** is limited to `10,000` records by Sequin -- **Timeout** is limited to `300` seconds (5 minutes) +- **Host**: Maximum 255 characters +- **Port**: Range 1-65535 (default: 3306) +- **Database name**: Maximum 64 characters, must be a valid MySQL identifier +- **Table name**: Maximum 64 characters, must be a valid MySQL identifier +- **Username**: Maximum 32 characters +- **Batch size**: Maximum 10,000 records per batch (limited by Sequin) +- **Timeout**: Maximum 300 seconds (5 minutes) - One sink targets one table; multiple tables require multiple sinks (or dynamic routing) - Table and column names must be valid MySQL identifiers From 40c6918a288dc12db08a8be66d181a606cb32024 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:43:05 -0400 Subject: [PATCH 14/30] fix: add missing newline at end of mysql docker-compose file --- examples/mysql/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mysql/docker-compose.yml b/examples/mysql/docker-compose.yml index 9379cb9a3..20a72afd0 100644 --- a/examples/mysql/docker-compose.yml +++ b/examples/mysql/docker-compose.yml @@ -18,4 +18,4 @@ services: retries: 10 volumes: - mysql_data: \ No newline at end of file + mysql_data: From 35462d9576dac7ce007ac7751d1dc7db2f7697ee Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:43:38 -0400 Subject: [PATCH 15/30] fix: add missing newline at end of init.sql for consistent formatting --- examples/mysql/init.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mysql/init.sql b/examples/mysql/init.sql index eb8615a9c..b9590e8c6 100644 --- a/examples/mysql/init.sql +++ b/examples/mysql/init.sql @@ -59,4 +59,4 @@ CREATE TABLE IF NOT EXISTS other_events ( -- Show the created table structures DESCRIBE products; DESCRIBE user_events; -DESCRIBE purchase_events; \ No newline at end of file +DESCRIBE purchase_events; From 2655ed94c5935c8e7a37181ddc35f61a2f7caaa2 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 12:47:19 -0400 Subject: [PATCH 16/30] feat: update MySQL docker-compose and init.sql with enhanced configuration and sample data --- examples/mysql/README.md | 305 +++++++++++++++++++----------- examples/mysql/docker-compose.yml | 37 +++- examples/mysql/init.sql | 142 +++++++++++--- examples/mysql/mysql.conf | 56 ++++++ 4 files changed, 398 insertions(+), 142 deletions(-) create mode 100644 examples/mysql/mysql.conf diff --git a/examples/mysql/README.md b/examples/mysql/README.md index 03a30d5b3..43884a7b2 100644 --- a/examples/mysql/README.md +++ b/examples/mysql/README.md @@ -1,139 +1,220 @@ # MySQL Sink Example -This example demonstrates how to set up Sequin to stream Postgres changes to a MySQL database. +This example demonstrates how to set up Sequin to stream Postgres changes to a MySQL database using Sequin's MySQL sink. + +## Overview + +This example includes: +- A complete Docker Compose setup for MySQL +- Sample database schema with realistic tables +- Multiple configuration examples (static and dynamic routing) +- Transform functions for data mapping + +## Quick Start with Docker + +1. **Start MySQL using Docker Compose:** + +```bash +cd examples/mysql +docker-compose up -d +``` + +This will create a MySQL instance with: +- Database: `sequin_test` +- User: `sequin_user` / Password: `sequin_password` +- Root password: `rootpassword` +- Port: `3306` (mapped to host) + +2. **The database will be automatically initialized** with sample tables from `init.sql`. ## Prerequisites - A running Postgres database with logical replication enabled -- A running MySQL database +- Docker and Docker Compose (for the provided setup) - Sequin installed and configured -## Setup - -1. **Create a MySQL table** to receive the data: +## Configuration Examples -```sql -CREATE DATABASE sequin_test; -USE sequin_test; +### Basic Static Routing -CREATE TABLE products ( - id INT PRIMARY KEY, - name VARCHAR(255) NOT NULL, - price DECIMAL(10,2), - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP -); -``` - -2. **Configure the MySQL Sink** in Sequin: +Create a `sequin.yaml` configuration file: ```yaml -name: products-to-mysql -source: - database: my_postgres_db - table: products - actions: [insert, update, delete] - -destination: - type: mysql - host: localhost - port: 3306 - database: sequin_test - table_name: products # Used when routing_mode is static - username: mysql_user - password: mysql_password - ssl: false - batch_size: 100 - timeout_seconds: 30 - upsert_on_duplicate: true - routing_mode: static # Can be 'static' or 'dynamic' - -transform: | - def transform(action, record, changes, metadata) do - # Map the Postgres record to MySQL-compatible format - %{ - id: record["id"], - name: record["name"], - price: record["price"] - } - end +databases: + - name: "source-db" + hostname: "localhost" + database: "my_postgres_db" + username: "postgres" + password: "postgres" + slot: + name: "sequin_slot" + create_if_not_exists: true + publication: + name: "sequin_pub" + create_if_not_exists: true + +sinks: + - name: "products-to-mysql" + database: "source-db" + source: + include_tables: + - "public.products" + destination: + type: "mysql" + host: "localhost" + port: 3306 + database: "sequin_test" + table_name: "products" + username: "sequin_user" + password: "sequin_password" + ssl: false + batch_size: 100 + timeout_seconds: 30 + upsert_on_duplicate: true + routing_mode: "static" + transform: "product-transform" + +functions: + - name: "product-transform" + type: "transform" + code: |- + def transform(_action, record, _changes, _metadata) do + %{ + id: record["id"], + name: record["name"], + price: record["price"], + description: record["description"], + category: record["category"], + in_stock: record["in_stock"], + metadata: record["metadata"] + } + end ``` -## Dynamic Table Routing +### Dynamic Routing Example -You can use routing functions to dynamically choose which MySQL table to route data to based on the record content: +For routing to different tables based on record content: ```yaml -name: multi-table-mysql-sink -source: - database: my_postgres_db - table: events - actions: [insert, update, delete] - -destination: - type: mysql - host: localhost - port: 3306 - database: sequin_test - table_name: default_events # Fallback table name - username: mysql_user - password: mysql_password - routing_mode: dynamic # Enable dynamic routing - -routing: | - def route(action, record, changes, metadata) do - # Route based on event type - table_name = case record["event_type"] do - "user_signup" -> "user_events" - "purchase" -> "purchase_events" - "analytics" -> "analytics_events" - _ -> "other_events" - end - - %{table_name: table_name} - end - -transform: | - def transform(action, record, changes, metadata) do - %{ - id: record["id"], - event_type: record["event_type"], - data: record["data"], - created_at: record["created_at"] - } - end +databases: + - name: "source-db" + hostname: "localhost" + database: "my_postgres_db" + username: "postgres" + password: "postgres" + slot: + name: "sequin_slot" + create_if_not_exists: true + publication: + name: "sequin_pub" + create_if_not_exists: true + +sinks: + - name: "events-to-mysql" + database: "source-db" + source: + include_tables: + - "public.events" + destination: + type: "mysql" + host: "localhost" + port: 3306 + database: "sequin_test" + table_name: "other_events" # Fallback table + username: "sequin_user" + password: "sequin_password" + routing_mode: "dynamic" + routing: "event-router" + transform: "event-transform" + +functions: + - name: "event-router" + type: "routing" + sink_type: "mysql" + code: |- + def route(_action, record, _changes, _metadata) do + table_name = case record["event_type"] do + "user_signup" -> "user_events" + "user_login" -> "user_events" + "purchase" -> "purchase_events" + "page_view" -> "analytics_events" + "click" -> "analytics_events" + _ -> "other_events" + end + + %{table_name: table_name} + end + + - name: "event-transform" + type: "transform" + code: |- + def transform(_action, record, _changes, _metadata) do + %{ + id: record["id"], + event_type: record["event_type"], + data: record["data"], + created_at: record["created_at"] + } + end ``` -## Features +## Features Demonstrated +- **Static routing**: Direct table mapping for simple use cases - **Dynamic routing**: Route to different MySQL tables based on record content - **Upsert support**: Uses MySQL's `ON DUPLICATE KEY UPDATE` for handling updates - **Batch processing**: Efficiently processes multiple records in batches -- **SSL support**: Can connect to MySQL over SSL -- **Type handling**: Automatically handles different data types and JSON encoding for complex values -- **Error handling**: Comprehensive error handling with detailed error messages +- **SSL support**: Can connect to MySQL over SSL (set `ssl: true`) +- **Type handling**: Automatically handles JSON and complex data types +- **Transform functions**: Map Postgres data to MySQL-compatible format + +## Database Schema -## Usage +The example creates several tables to demonstrate different scenarios: + +- `products` - Basic product catalog with various data types +- `user_events` - User activity events +- `purchase_events` - E-commerce transaction events +- `analytics_events` - Web analytics data +- `other_events` - Catch-all for unmatched event types + +## Usage Flow Once configured, Sequin will: -1. Capture changes from your Postgres table -2. Transform the data using your transform function -3. Insert/update records in MySQL using batch operations -4. Handle deletes by removing records from MySQL - -The sink supports both insert-only mode and upsert mode depending on your `upsert_on_duplicate` setting. - -## Connection Options - -- `host`: MySQL server hostname -- `port`: MySQL server port (default: 3306) -- `database`: Target database name -- `table_name`: Target table name (used as fallback when routing_mode is dynamic) -- `username`: MySQL username -- `password`: MySQL password -- `ssl`: Enable SSL connection (default: false) -- `batch_size`: Number of records to process in each batch (default: 100) -- `timeout_seconds`: Connection timeout in seconds (default: 30) -- `upsert_on_duplicate`: Use upsert instead of insert-only (default: true) -- `routing_mode`: Set to "dynamic" to enable routing functions, "static" for fixed table (default: static) \ No newline at end of file +1. **Capture** changes from your Postgres tables via logical replication +2. **Transform** the data using your transform function +3. **Route** records to appropriate MySQL tables (dynamic mode) +4. **Batch** multiple records for efficient processing +5. **Upsert/Insert** data into MySQL using optimized SQL operations + +## Configuration Options + +For complete configuration reference, see the [MySQL sink documentation](/reference/sinks/mysql). + +Key MySQL sink options: +- `host` - MySQL server hostname +- `port` - MySQL server port (default: 3306) +- `database` - Target MySQL database +- `table_name` - Target table (or fallback for dynamic routing) +- `username`/`password` - Authentication credentials +- `ssl` - Enable SSL/TLS connection (default: false) +- `batch_size` - Records per batch (default: 100, max: 10,000) +- `timeout_seconds` - Connection timeout (default: 30, max: 300) +- `upsert_on_duplicate` - Use MySQL upsert mode (default: true) +- `routing_mode` - `"static"` or `"dynamic"` (default: "static") + +## Cleanup + +To stop and remove the MySQL container: + +```bash +docker-compose down -v # -v removes volumes +``` + +## Next Steps + +- Learn more about [MySQL sinks](/reference/sinks/mysql) +- Explore [routing functions](/reference/routing) for advanced table routing +- See [transform functions](/reference/transforms) for data mapping +- Check out the complete [sequin.yaml reference](/reference/sequin-yaml) \ No newline at end of file diff --git a/examples/mysql/docker-compose.yml b/examples/mysql/docker-compose.yml index 20a72afd0..48b32a7ea 100644 --- a/examples/mysql/docker-compose.yml +++ b/examples/mysql/docker-compose.yml @@ -1,21 +1,42 @@ +version: '3.8' + services: mysql: image: mysql:8.0 container_name: sequin-mysql-example + restart: unless-stopped environment: - MYSQL_ROOT_PASSWORD: rootpassword - MYSQL_DATABASE: sequin_test - MYSQL_USER: sequin_user - MYSQL_PASSWORD: sequin_password + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-rootpassword} + MYSQL_DATABASE: ${MYSQL_DATABASE:-sequin_test} + MYSQL_USER: ${MYSQL_USER:-sequin_user} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:-sequin_password} + # Additional MySQL configuration + MYSQL_CHARSET: utf8mb4 + MYSQL_COLLATION: utf8mb4_unicode_ci ports: - - "3306:3306" + - "${MYSQL_PORT:-3306}:3306" volumes: - mysql_data:/var/lib/mysql - ./init.sql:/docker-entrypoint-initdb.d/init.sql + - ./mysql.conf:/etc/mysql/conf.d/mysql.conf:ro healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] - timeout: 20s - retries: 10 + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u${MYSQL_USER:-sequin_user}", "-p${MYSQL_PASSWORD:-sequin_password}"] + timeout: 10s + retries: 5 + interval: 30s + start_period: 30s + networks: + - sequin-mysql-network + command: > + --character-set-server=utf8mb4 + --collation-server=utf8mb4_unicode_ci + --default-authentication-plugin=mysql_native_password + --log-bin-trust-function-creators=1 volumes: mysql_data: + driver: local + +networks: + sequin-mysql-network: + driver: bridge diff --git a/examples/mysql/init.sql b/examples/mysql/init.sql index b9590e8c6..c2e3fa42e 100644 --- a/examples/mysql/init.sql +++ b/examples/mysql/init.sql @@ -1,62 +1,160 @@ --- Create the products table for testing Sequin MySQL sink +-- ================================================== +-- Sequin MySQL Sink Example - Database Initialization +-- ================================================== +-- This script creates the necessary tables and sample data +-- for demonstrating Sequin's MySQL sink capabilities. + +USE sequin_test; + +-- ================================================== +-- PRODUCTS TABLE +-- ================================================== +-- Main products table demonstrating various MySQL data types +-- and how they map from Postgres sources CREATE TABLE IF NOT EXISTS products ( - id INT PRIMARY KEY, + id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(255) NOT NULL, price DECIMAL(10,2), description TEXT, category VARCHAR(100), in_stock BOOLEAN DEFAULT TRUE, metadata JSON, + tags VARCHAR(500), -- Comma-separated tags for simple arrays created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- Add some constraints for realistic data modeling + CONSTRAINT chk_price_positive CHECK (price >= 0), + INDEX idx_products_category (category), + INDEX idx_products_in_stock (in_stock), + INDEX idx_products_created_at (created_at) ); --- Create an index on commonly queried fields -CREATE INDEX idx_products_category ON products(category); -CREATE INDEX idx_products_in_stock ON products(in_stock); +-- Insert realistic sample products +INSERT INTO products (id, name, price, description, category, in_stock, metadata, tags) VALUES +(1, 'Wireless Bluetooth Headphones', 89.99, 'High-quality wireless headphones with noise cancellation', 'electronics', TRUE, '{"brand": "TechCorp", "warranty": "2 years", "features": ["wireless", "noise-cancellation", "long-battery"]}', 'audio,wireless,bluetooth'), +(2, 'Programming in Elixir', 45.50, 'Comprehensive guide to functional programming with Elixir', 'books', TRUE, '{"author": "Jane Developer", "pages": 420, "edition": "2nd", "language": "english"}', 'programming,elixir,functional'), +(3, 'Organic Cotton T-Shirt', 29.99, 'Comfortable organic cotton t-shirt in various colors', 'clothing', FALSE, '{"material": "organic cotton", "sizes": ["S", "M", "L", "XL"], "colors": ["white", "black", "gray"]}', 'organic,cotton,casual'), +(4, 'Smart Home Hub', 199.99, 'Central control unit for smart home devices', 'electronics', TRUE, '{"connectivity": ["wifi", "bluetooth", "zigbee"], "compatible_devices": 500, "voice_assistant": true}', 'smart-home,iot,automation'); --- Insert some sample data for testing -INSERT INTO products (id, name, price, description, category, in_stock) VALUES -(1, 'Sample Product 1', 19.99, 'This is a sample product for testing', 'electronics', TRUE), -(2, 'Sample Product 2', 29.99, 'Another sample product', 'books', TRUE), -(3, 'Sample Product 3', 39.99, 'Third sample product', 'clothing', FALSE); +-- ================================================== +-- EVENT TABLES FOR DYNAMIC ROUTING EXAMPLES +-- ================================================== --- Create additional tables for routing examples +-- User Events Table - for user activity tracking CREATE TABLE IF NOT EXISTS user_events ( - id INT PRIMARY KEY, + id INT AUTO_INCREMENT PRIMARY KEY, event_type VARCHAR(100) NOT NULL, user_id INT, + session_id VARCHAR(255), data JSON, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ip_address VARCHAR(45), -- Support both IPv4 and IPv6 + user_agent TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + INDEX idx_user_events_type (event_type), + INDEX idx_user_events_user_id (user_id), + INDEX idx_user_events_created_at (created_at) ); +-- Purchase Events Table - for e-commerce transactions CREATE TABLE IF NOT EXISTS purchase_events ( - id INT PRIMARY KEY, + id INT AUTO_INCREMENT PRIMARY KEY, event_type VARCHAR(100) NOT NULL, user_id INT, product_id INT, + quantity INT DEFAULT 1, amount DECIMAL(10,2), + currency VARCHAR(3) DEFAULT 'USD', + payment_method VARCHAR(50), data JSON, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT chk_quantity_positive CHECK (quantity > 0), + CONSTRAINT chk_amount_positive CHECK (amount >= 0), + INDEX idx_purchase_events_user_id (user_id), + INDEX idx_purchase_events_product_id (product_id), + INDEX idx_purchase_events_created_at (created_at) ); +-- Analytics Events Table - for web analytics and tracking CREATE TABLE IF NOT EXISTS analytics_events ( - id INT PRIMARY KEY, + id INT AUTO_INCREMENT PRIMARY KEY, event_type VARCHAR(100) NOT NULL, session_id VARCHAR(255), - page_url VARCHAR(500), + page_url VARCHAR(1000), + referrer_url VARCHAR(1000), + device_type VARCHAR(50), + browser VARCHAR(100), data JSON, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + INDEX idx_analytics_events_type (event_type), + INDEX idx_analytics_events_session_id (session_id), + INDEX idx_analytics_events_created_at (created_at) ); +-- Other Events Table - catch-all for unmatched event types CREATE TABLE IF NOT EXISTS other_events ( - id INT PRIMARY KEY, + id INT AUTO_INCREMENT PRIMARY KEY, event_type VARCHAR(100) NOT NULL, + source_system VARCHAR(100), data JSON, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + processed BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + INDEX idx_other_events_type (event_type), + INDEX idx_other_events_processed (processed), + INDEX idx_other_events_created_at (created_at) ); --- Show the created table structures +-- ================================================== +-- INSERT SAMPLE EVENT DATA +-- ================================================== + +-- Sample user events +INSERT INTO user_events (event_type, user_id, session_id, data, ip_address) VALUES +('user_signup', 101, 'sess_abc123', '{"signup_method": "email", "referrer": "google"}', '192.168.1.100'), +('user_login', 102, 'sess_def456', '{"login_method": "email", "remember_me": true}', '192.168.1.101'), +('profile_update', 101, 'sess_abc123', '{"fields_updated": ["email", "name"], "source": "settings_page"}', '192.168.1.100'); + +-- Sample purchase events +INSERT INTO purchase_events (event_type, user_id, product_id, quantity, amount, payment_method, data) VALUES +('purchase', 102, 1, 1, 89.99, 'credit_card', '{"promotion_code": "SAVE10", "shipping_method": "express"}'), +('purchase', 101, 2, 2, 91.00, 'paypal', '{"gift_message": "Happy Birthday!", "shipping_method": "standard"}'); + +-- Sample analytics events +INSERT INTO analytics_events (event_type, session_id, page_url, device_type, browser, data) VALUES +('page_view', 'sess_abc123', '/products/headphones', 'desktop', 'Chrome', '{"time_on_page": 45, "scroll_depth": 80}'), +('click', 'sess_def456', '/products/books', 'mobile', 'Safari', '{"element": "add_to_cart", "product_id": 2}'); + +-- ================================================== +-- SHOW TABLE STRUCTURES +-- ================================================== +-- Display the created table structures for reference + +SHOW TABLES; + DESCRIBE products; DESCRIBE user_events; DESCRIBE purchase_events; +DESCRIBE analytics_events; +DESCRIBE other_events; + +-- ================================================== +-- SUMMARY INFORMATION +-- ================================================== +SELECT 'Database setup complete!' as status; +SELECT 'Tables created:' as info, COUNT(*) as count FROM information_schema.tables WHERE table_schema = 'sequin_test'; + +-- Show sample data counts +SELECT 'Sample data inserted:' as info; +SELECT 'products' as table_name, COUNT(*) as row_count FROM products +UNION ALL +SELECT 'user_events', COUNT(*) FROM user_events +UNION ALL +SELECT 'purchase_events', COUNT(*) FROM purchase_events +UNION ALL +SELECT 'analytics_events', COUNT(*) FROM analytics_events +UNION ALL +SELECT 'other_events', COUNT(*) FROM other_events; diff --git a/examples/mysql/mysql.conf b/examples/mysql/mysql.conf new file mode 100644 index 000000000..fffdfa5cb --- /dev/null +++ b/examples/mysql/mysql.conf @@ -0,0 +1,56 @@ +# ============================================== +# Sequin MySQL Sink Example - Server Configuration +# ============================================== +# Optimized MySQL configuration for development and testing +# with Sequin MySQL sink. + +[mysqld] + +# Basic Settings +default-storage-engine = InnoDB +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci +default-authentication-plugin = mysql_native_password + +# Performance Settings +max_connections = 200 +innodb_buffer_pool_size = 256M +innodb_log_file_size = 64M +innodb_flush_log_at_trx_commit = 2 +query_cache_type = 1 +query_cache_size = 32M + +# Binary Logging (useful for debugging) +log-bin = mysql-bin +expire_logs_days = 7 +max_binlog_size = 100M + +# Enable function creation (required for some advanced use cases) +log-bin-trust-function-creators = 1 + +# Security Settings +local-infile = 0 + +# Connection Settings +wait_timeout = 3600 +interactive_timeout = 3600 +max_allowed_packet = 64M + +# InnoDB Settings +innodb_file_per_table = 1 +innodb_open_files = 400 +innodb_io_capacity = 400 + +# Slow Query Log (for debugging performance issues) +slow_query_log = 1 +slow_query_log_file = /var/log/mysql/slow.log +long_query_time = 2 + +# SQL Mode (compatible with common applications) +sql_mode = STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION + +[client] +default-character-set = utf8mb4 + +[mysql] +default-character-set = utf8mb4 \ No newline at end of file From 2ebc456976e55110284a2b7b010ba230a02e760f Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 14:04:35 -0400 Subject: [PATCH 17/30] feat: add mysql_module to Sequin configuration and clean up test setup --- config/test.exs | 1 + test/sequin/mysql_pipeline_test.exs | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/config/test.exs b/config/test.exs index 93f54042d..e451daf8c 100644 --- a/config/test.exs +++ b/config/test.exs @@ -109,6 +109,7 @@ config :sequin, ], redis_module: Sequin.Sinks.RedisMock, kafka_module: Sequin.Sinks.KafkaMock, + mysql_module: Sequin.Sinks.MysqlMock, nats_module: Sequin.Sinks.NatsMock, rabbitmq_module: Sequin.Sinks.RabbitMqMock, aws_module: Sequin.AwsMock, diff --git a/test/sequin/mysql_pipeline_test.exs b/test/sequin/mysql_pipeline_test.exs index 0d7df3b52..e51a7be3a 100644 --- a/test/sequin/mysql_pipeline_test.exs +++ b/test/sequin/mysql_pipeline_test.exs @@ -195,8 +195,6 @@ defmodule Sequin.Runtime.MysqlPipelineTest do end defp start_pipeline!(consumer) do - Application.put_env(:sequin, :mysql_module, MysqlMock) - start_supervised!( {SinkPipeline, [ From 6934db7d0801b44b7eafb2b6810b398101749ffd Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 14:08:40 -0400 Subject: [PATCH 18/30] feat: add mysql to routing function consumer types --- lib/sequin/consumers/routing_function.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sequin/consumers/routing_function.ex b/lib/sequin/consumers/routing_function.ex index 7c7c5ee68..2386fb9d3 100644 --- a/lib/sequin/consumers/routing_function.ex +++ b/lib/sequin/consumers/routing_function.ex @@ -29,7 +29,8 @@ defmodule Sequin.Consumers.RoutingFunction do :elasticsearch, :s2, :rabbitmq, - :sqs + :sqs, + :mysql ] field :code, :string From 187b48c757c94cb950f12b963bb99f8820d255e7 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 14:13:21 -0400 Subject: [PATCH 19/30] feat: define mysql_module in Sequin.Sinks.Mysql for improved configuration --- lib/sequin/sinks/mysql.ex | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/sequin/sinks/mysql.ex b/lib/sequin/sinks/mysql.ex index 4c2ded26a..b7487771e 100644 --- a/lib/sequin/sinks/mysql.ex +++ b/lib/sequin/sinks/mysql.ex @@ -7,22 +7,20 @@ defmodule Sequin.Sinks.Mysql do @callback upsert_records(MysqlSink.t(), [map()]) :: :ok | {:error, Error.t()} @callback delete_records(MysqlSink.t(), [any()]) :: :ok | {:error, Error.t()} + @module Application.compiled_env(:sequin, :mysql_module, Sequin.Sinks.Mysql.Client) + @spec test_connection(MysqlSink.t()) :: :ok | {:error, Error.t()} def test_connection(%MysqlSink{} = sink) do - impl().test_connection(sink) + @module.test_connection(sink) end @spec upsert_records(MysqlSink.t(), [map()]) :: :ok | {:error, Error.t()} def upsert_records(%MysqlSink{} = sink, records) when is_list(records) do - impl().upsert_records(sink, records) + @module.upsert_records(sink, records) end @spec delete_records(MysqlSink.t(), [any()]) :: :ok | {:error, Error.t()} def delete_records(%MysqlSink{} = sink, record_pks) when is_list(record_pks) do - impl().delete_records(sink, record_pks) - end - - defp impl do - Application.get_env(:sequin, :mysql_module, Sequin.Sinks.Mysql.Client) + @module.delete_records(sink, record_pks) end end From 6ef1e87c358dec45003eb4554d3f32937401a482 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Wed, 6 Aug 2025 14:25:35 -0400 Subject: [PATCH 20/30] refactor: clean up comments and improve readability in MySQL related files --- examples/mysql/init.sql | 7 ------- lib/sequin/runtime/mysql_pipeline.ex | 7 ------- lib/sequin/sinks/mysql/client.ex | 17 +++++------------ lib/sequin/sinks/mysql/connection_cache.ex | 1 - test/sequin/mysql_sink_test.exs | 6 ------ 5 files changed, 5 insertions(+), 33 deletions(-) diff --git a/examples/mysql/init.sql b/examples/mysql/init.sql index c2e3fa42e..47ee1f22d 100644 --- a/examples/mysql/init.sql +++ b/examples/mysql/init.sql @@ -1,14 +1,10 @@ --- ================================================== -- Sequin MySQL Sink Example - Database Initialization --- ================================================== -- This script creates the necessary tables and sample data -- for demonstrating Sequin's MySQL sink capabilities. USE sequin_test; --- ================================================== -- PRODUCTS TABLE --- ================================================== -- Main products table demonstrating various MySQL data types -- and how they map from Postgres sources CREATE TABLE IF NOT EXISTS products ( @@ -30,16 +26,13 @@ CREATE TABLE IF NOT EXISTS products ( INDEX idx_products_created_at (created_at) ); --- Insert realistic sample products INSERT INTO products (id, name, price, description, category, in_stock, metadata, tags) VALUES (1, 'Wireless Bluetooth Headphones', 89.99, 'High-quality wireless headphones with noise cancellation', 'electronics', TRUE, '{"brand": "TechCorp", "warranty": "2 years", "features": ["wireless", "noise-cancellation", "long-battery"]}', 'audio,wireless,bluetooth'), (2, 'Programming in Elixir', 45.50, 'Comprehensive guide to functional programming with Elixir', 'books', TRUE, '{"author": "Jane Developer", "pages": 420, "edition": "2nd", "language": "english"}', 'programming,elixir,functional'), (3, 'Organic Cotton T-Shirt', 29.99, 'Comfortable organic cotton t-shirt in various colors', 'clothing', FALSE, '{"material": "organic cotton", "sizes": ["S", "M", "L", "XL"], "colors": ["white", "black", "gray"]}', 'organic,cotton,casual'), (4, 'Smart Home Hub', 199.99, 'Central control unit for smart home devices', 'electronics', TRUE, '{"connectivity": ["wifi", "bluetooth", "zigbee"], "compatible_devices": 500, "voice_assistant": true}', 'smart-home,iot,automation'); --- ================================================== -- EVENT TABLES FOR DYNAMIC ROUTING EXAMPLES --- ================================================== -- User Events Table - for user activity tracking CREATE TABLE IF NOT EXISTS user_events ( diff --git a/lib/sequin/runtime/mysql_pipeline.ex b/lib/sequin/runtime/mysql_pipeline.ex index ac9f48e86..6843c8310 100644 --- a/lib/sequin/runtime/mysql_pipeline.ex +++ b/lib/sequin/runtime/mysql_pipeline.ex @@ -57,7 +57,6 @@ defmodule Sequin.Runtime.MysqlPipeline do setup_allowances(test_pid) - # Get the table name from the batch key (set by routing) table_name = Map.get(batch_info, :batch_key, sink.table_name) records = @@ -67,7 +66,6 @@ defmodule Sequin.Runtime.MysqlPipeline do |> ensure_string_keys() end) - # Create a temporary sink with the routed table name routed_sink = %{sink | table_name: table_name} case Mysql.upsert_records(routed_sink, records) do @@ -97,13 +95,11 @@ defmodule Sequin.Runtime.MysqlPipeline do setup_allowances(test_pid) - # Get the table name from the batch key (set by routing) table_name = Map.get(batch_info, :batch_key, sink.table_name) record_pks = Enum.flat_map(messages, fn %{data: message} -> message.record_pks end) - # Create a temporary sink with the routed table name routed_sink = %{sink | table_name: table_name} case Mysql.delete_records(routed_sink, record_pks) do @@ -126,9 +122,6 @@ defmodule Sequin.Runtime.MysqlPipeline do end end - # Helper functions - - # Ensure all keys in the record are strings for MySQL column compatibility defp ensure_string_keys(record) when is_map(record) do Map.new(record, fn {key, value} when is_atom(key) -> {Atom.to_string(key), value} diff --git a/lib/sequin/sinks/mysql/client.ex b/lib/sequin/sinks/mysql/client.ex index 1ab0395d5..6584f075a 100644 --- a/lib/sequin/sinks/mysql/client.ex +++ b/lib/sequin/sinks/mysql/client.ex @@ -53,10 +53,10 @@ defmodule Sequin.Sinks.Mysql.Client do {:ok} else case ConnectionCache.get_connection(sink) do - {:ok, pid} -> - try do - # Assume primary key is 'id' for simplicity - placeholders = Enum.map(record_pks, fn _ -> "?" end) |> Enum.join(", ") + {:ok, pid} -> + try do + # Assume primary key is 'id' for simplicity + placeholders = Enum.map(record_pks, fn _ -> "?" end) |> Enum.join(", ") delete_sql = "DELETE FROM `#{sink.table_name}` WHERE `id` IN (#{placeholders})" case MyXQL.query(pid, delete_sql, record_pks) do @@ -78,8 +78,6 @@ defmodule Sequin.Sinks.Mysql.Client do end end - # Private functions - defp insert_or_update_records(pid, %MysqlSink{} = sink, records) do case build_upsert_query(sink, records) do {:ok, {sql, params}} -> @@ -120,7 +118,6 @@ defmodule Sequin.Sinks.Mysql.Client do column_list = Enum.map(columns, &"`#{&1}`") |> Enum.join(", ") placeholders = Enum.map(columns, fn _ -> "?" end) |> Enum.join(", ") - # Build ON DUPLICATE KEY UPDATE clause update_clause = columns |> Enum.map(&"`#{&1}` = VALUES(`#{&1}`)") @@ -132,7 +129,6 @@ defmodule Sequin.Sinks.Mysql.Client do ON DUPLICATE KEY UPDATE #{update_clause} """ - # Flatten all values for parameters params = Enum.flat_map(values, & &1) {:ok, {sql, params}} @@ -153,7 +149,6 @@ defmodule Sequin.Sinks.Mysql.Client do VALUES #{Enum.map(values, fn _ -> "(#{placeholders})" end) |> Enum.join(", ")} """ - # Flatten all values for parameters params = Enum.flat_map(values, & &1) {:ok, {sql, params}} @@ -167,14 +162,12 @@ defmodule Sequin.Sinks.Mysql.Client do if Enum.empty?(records) do {:error, Error.service(service: :mysql, message: "No records provided")} else - # Get all unique columns from all records all_columns = records |> Enum.flat_map(&Map.keys/1) |> Enum.uniq() |> Enum.sort() - # Extract values for each record, filling nil for missing columns values = Enum.map(records, fn record -> Enum.map(all_columns, fn column -> @@ -183,7 +176,7 @@ defmodule Sequin.Sinks.Mysql.Client do value when is_binary(value) -> value value when is_number(value) -> value value when is_boolean(value) -> value - value -> Jason.encode!(value) # JSON encode complex values + value -> Jason.encode!(value) end end) end) diff --git a/lib/sequin/sinks/mysql/connection_cache.ex b/lib/sequin/sinks/mysql/connection_cache.ex index f0008154f..4e15a3f88 100644 --- a/lib/sequin/sinks/mysql/connection_cache.ex +++ b/lib/sequin/sinks/mysql/connection_cache.ex @@ -209,7 +209,6 @@ defmodule Sequin.Sinks.Mysql.ConnectionCache do {:reply, :ok, new_state} {:error, error} -> - # Invalidate the connection on test failure invalidated_state = State.invalidate_connection(new_state, sink) {:reply, {:error, error}, invalidated_state} end diff --git a/test/sequin/mysql_sink_test.exs b/test/sequin/mysql_sink_test.exs index 8496efbf7..c6ce941a6 100644 --- a/test/sequin/mysql_sink_test.exs +++ b/test/sequin/mysql_sink_test.exs @@ -58,14 +58,12 @@ defmodule Sequin.Consumers.MysqlSinkTest do end test "validates table_name format", %{valid_params: params} do - # Valid table names valid_names = ["table", "user_data", "Table123", "_private"] for name <- valid_names do changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: name}) assert Sequin.Error.errors_on(changeset)[:table_name] == nil, "#{name} should be valid" end - # Invalid table names invalid_names = ["123table", "table-name", "table space", ""] for name <- invalid_names do changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: name}) @@ -74,22 +72,18 @@ defmodule Sequin.Consumers.MysqlSinkTest do end test "validates string field lengths", %{valid_params: params} do - # Test host length long_host = String.duplicate("a", 256) changeset = MysqlSink.changeset(%MysqlSink{}, %{params | host: long_host}) assert Sequin.Error.errors_on(changeset)[:host] == ["should be at most 255 character(s)"] - # Test database length long_database = String.duplicate("a", 65) changeset = MysqlSink.changeset(%MysqlSink{}, %{params | database: long_database}) assert Sequin.Error.errors_on(changeset)[:database] == ["should be at most 64 character(s)"] - # Test table_name length long_table = String.duplicate("a", 65) changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: long_table}) assert Sequin.Error.errors_on(changeset)[:table_name] == ["should be at most 64 character(s)"] - # Test username length long_username = String.duplicate("a", 33) changeset = MysqlSink.changeset(%MysqlSink{}, %{params | username: long_username}) assert Sequin.Error.errors_on(changeset)[:username] == ["should be at most 32 character(s)"] From 79e71dc1a1eef446d9e3446c7852d768a3077d2e Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:38:58 -0400 Subject: [PATCH 21/30] refactor: simplify layout in MysqlSinkForm by removing unnecessary grid classes --- .../svelte/sinks/mysql/MysqlSinkForm.svelte | 128 +++++++++--------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte index 07f29429d..3076b0c61 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -49,35 +49,33 @@ -
-
- - - {#if errors.sink?.host} -

{errors.sink.host}

- {/if} -

- MySQL server hostname or IP address -

-
+
+ + + {#if errors.sink?.host} +

{errors.sink.host}

+ {/if} +

+ MySQL server hostname or IP address +

+
-
- - - {#if errors.sink?.port} -

{errors.sink.port}

- {/if} -

- MySQL server port (default: 3306) -

-
+
+ + + {#if errors.sink?.port} +

{errors.sink.port}

+ {/if} +

+ MySQL server port (default: 3306) +

@@ -95,47 +93,45 @@

-
-
- +
+ + + {#if errors.sink?.username} +

{errors.sink.username}

+ {/if} +
+ +
+ +
- {#if errors.sink?.username} -

{errors.sink.username}

- {/if} -
- -
- -
- - -
- {#if errors.sink?.password} -

{errors.sink.password}

- {/if} +
+ {#if errors.sink?.password} +

{errors.sink.password}

+ {/if}
From 631e7375f2f5674359a7486f8b8c83601028e37f Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:40:59 -0400 Subject: [PATCH 22/30] Signed-off-by: Yordis Prieto --- assets/svelte/sinks/mysql/MysqlSinkForm.svelte | 15 +++++++++++++++ docs/reference/sinks/mysql.mdx | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte index 3076b0c61..b747b6b79 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -171,6 +171,21 @@ records. When disabled, attempts to insert records directly (may fail on duplicates).

+
+
+
+

+ Important limitation +

+
+

+ MySQL's ON DUPLICATE KEY UPDATE can behave unpredictably on tables with multiple unique indexes. + If your table has multiple unique constraints, consider disabling this option. +

+
+
+
+
diff --git a/docs/reference/sinks/mysql.mdx b/docs/reference/sinks/mysql.mdx index 6a64e9288..d7dc50a81 100644 --- a/docs/reference/sinks/mysql.mdx +++ b/docs/reference/sinks/mysql.mdx @@ -122,6 +122,16 @@ ON DUPLICATE KEY UPDATE When disabled, Sequin performs direct inserts, which may fail if records with duplicate primary keys exist. + +**Important limitation**: MySQL's `ON DUPLICATE KEY UPDATE` can behave unpredictably on tables with multiple unique indexes. If your target table has multiple unique constraints (e.g., both a primary key and a unique email column), consider: + +- Disabling upsert and handling duplicates in your application logic +- Restructuring your table to have only one unique constraint +- Using a routing function to target different tables based on your data + +This is a known MySQL limitation documented in the [official MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html). + + ## Connection management Sequin maintains a connection pool to your MySQL server for optimal performance. Connections are cached and reused across multiple operations to minimize connection overhead. From 60ab7202a577d9d2409f7993e66e91a29cff8daa Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:41:17 -0400 Subject: [PATCH 23/30] fix: improve readability of warning message in MysqlSinkForm --- assets/svelte/sinks/mysql/MysqlSinkForm.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte index b747b6b79..4f2265495 100644 --- a/assets/svelte/sinks/mysql/MysqlSinkForm.svelte +++ b/assets/svelte/sinks/mysql/MysqlSinkForm.svelte @@ -179,8 +179,9 @@

- MySQL's ON DUPLICATE KEY UPDATE can behave unpredictably on tables with multiple unique indexes. - If your table has multiple unique constraints, consider disabling this option. + MySQL's ON DUPLICATE KEY UPDATE can behave unpredictably on + tables with multiple unique indexes. If your table has multiple + unique constraints, consider disabling this option.

From f68620fb4298b95f0f7edaf91fc89cb0dbf7492e Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:43:54 -0400 Subject: [PATCH 24/30] feat: enhance MySQL sink documentation with type casting details and limitations --- docs/reference/sinks/mysql.mdx | 27 ++++++++++++++++++++++++++- lib/sequin/sinks/mysql/client.ex | 6 ++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/reference/sinks/mysql.mdx b/docs/reference/sinks/mysql.mdx index d7dc50a81..3826bcac8 100644 --- a/docs/reference/sinks/mysql.mdx +++ b/docs/reference/sinks/mysql.mdx @@ -62,7 +62,32 @@ The **MySQL sink** writes change data into MySQL tables using direct SQL operati Your [transform](/reference/transforms) must return a map where keys correspond to MySQL table column names and values are properly typed for the target columns. -For complex data types like JSON objects or arrays, Sequin will automatically encode them as JSON strings when inserting into MySQL. +### Type casting + +Sequin automatically handles type conversion for the following Elixir types: + +| Elixir Type | MySQL Handling | Notes | +|-------------|----------------|--------| +| `nil` | `NULL` | Preserved as NULL | +| `String.t()` | Direct insertion | Strings, text, varchar columns | +| `number()` | Direct insertion | Integer, float, decimal columns | +| `boolean()` | Direct insertion | Boolean/tinyint columns | +| `DateTime.t()` | Direct insertion | DATETIME, TIMESTAMP columns | +| `Date.t()` | Direct insertion | DATE columns | +| `Time.t()` | Direct insertion | TIME columns | +| `NaiveDateTime.t()` | Direct insertion | DATETIME columns | +| `Decimal.t()` | Direct insertion | DECIMAL, NUMERIC columns | +| Other types | JSON encoding | Encoded as JSON strings | + +For complex data types like maps, lists, or custom structs, Sequin will automatically encode them as JSON strings when inserting into MySQL. + +### Type casting limitations + +- **Binary data (bytea)**: Currently encoded as JSON strings. Consider base64 encoding in your transform for proper handling. +- **Custom types**: All non-standard types are JSON-encoded, which may not be suitable for all MySQL column types. +- **Timezone handling**: `DateTime` values are passed directly to MySQL. Ensure your MySQL timezone settings match your expectations. + +For more explicit type control, handle type conversion in your [transform function](/reference/transforms) before data reaches the sink. ### Example transforms diff --git a/lib/sequin/sinks/mysql/client.ex b/lib/sequin/sinks/mysql/client.ex index 6584f075a..a81504e11 100644 --- a/lib/sequin/sinks/mysql/client.ex +++ b/lib/sequin/sinks/mysql/client.ex @@ -3,6 +3,7 @@ defmodule Sequin.Sinks.Mysql.Client do Client for interacting with MySQL databases using MyXQL. """ + alias Decimal alias Sequin.Consumers.MysqlSink alias Sequin.Error alias Sequin.Sinks.Mysql.ConnectionCache @@ -176,6 +177,11 @@ defmodule Sequin.Sinks.Mysql.Client do value when is_binary(value) -> value value when is_number(value) -> value value when is_boolean(value) -> value + %DateTime{} = dt -> dt + %Date{} = date -> date + %Time{} = time -> time + %NaiveDateTime{} = naive_dt -> naive_dt + %Decimal{} = decimal -> decimal value -> Jason.encode!(value) end end) From 954cdba3ad2d4d014cbf6938f7e32959aaaa1bf9 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:52:18 -0400 Subject: [PATCH 25/30] refactor: improve code readability and formatting in MySQL sink and client modules --- lib/sequin/consumers/mysql_sink.ex | 7 +- lib/sequin/runtime/routing/consumers/mysql.ex | 6 +- lib/sequin/sinks/mysql/client.ex | 130 ++++++++++++++++-- test/sequin/mysql_client_test.exs | 2 +- test/sequin/mysql_pipeline_test.exs | 15 +- test/sequin/mysql_sink_test.exs | 12 +- 6 files changed, 137 insertions(+), 35 deletions(-) diff --git a/lib/sequin/consumers/mysql_sink.ex b/lib/sequin/consumers/mysql_sink.ex index 6cebbc2bb..233c30d05 100644 --- a/lib/sequin/consumers/mysql_sink.ex +++ b/lib/sequin/consumers/mysql_sink.ex @@ -43,7 +43,7 @@ defmodule Sequin.Consumers.MysqlSink do :routing_mode ]) |> validate_required([:host, :database, :table_name, :username, :password]) - |> validate_number(:port, greater_than: 0, less_than_or_equal_to: 65535) + |> validate_number(:port, greater_than: 0, less_than_or_equal_to: 65_535) |> validate_number(:batch_size, greater_than: 0, less_than_or_equal_to: 10_000) |> validate_number(:timeout_seconds, greater_than: 0, less_than_or_equal_to: 300) |> validate_length(:host, max: 255) @@ -54,8 +54,7 @@ defmodule Sequin.Consumers.MysqlSink do end defp validate_table_name(changeset) do - changeset - |> validate_format(:table_name, ~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, + validate_format(changeset, :table_name, ~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, message: "must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)" ) end @@ -74,7 +73,7 @@ defmodule Sequin.Consumers.MysqlSink do database: sink.database, username: sink.username, password: sink.password, - timeout: :timer.seconds(sink.timeout_seconds), + timeout: to_timeout(second: sink.timeout_seconds), pool_size: 10 ] diff --git a/lib/sequin/runtime/routing/consumers/mysql.ex b/lib/sequin/runtime/routing/consumers/mysql.ex index 25697a45a..0ceec48cc 100644 --- a/lib/sequin/runtime/routing/consumers/mysql.ex +++ b/lib/sequin/runtime/routing/consumers/mysql.ex @@ -43,7 +43,9 @@ defmodule Sequin.Runtime.Routing.Consumers.Mysql do defp sanitize_table_name(name) do name |> String.replace(~r/[^a-zA-Z0-9_]/, "_") - |> String.replace(~r/^[0-9]/, "_\\0") # Ensure it doesn't start with a number - |> String.slice(0, 64) # MySQL table name limit + # Ensure it doesn't start with a number + |> String.replace(~r/^[0-9]/, "_\\0") + # MySQL table name limit + |> String.slice(0, 64) end end diff --git a/lib/sequin/sinks/mysql/client.ex b/lib/sequin/sinks/mysql/client.ex index a81504e11..a34235cf7 100644 --- a/lib/sequin/sinks/mysql/client.ex +++ b/lib/sequin/sinks/mysql/client.ex @@ -3,7 +3,6 @@ defmodule Sequin.Sinks.Mysql.Client do Client for interacting with MySQL databases using MyXQL. """ - alias Decimal alias Sequin.Consumers.MysqlSink alias Sequin.Error alias Sequin.Sinks.Mysql.ConnectionCache @@ -54,10 +53,10 @@ defmodule Sequin.Sinks.Mysql.Client do {:ok} else case ConnectionCache.get_connection(sink) do - {:ok, pid} -> - try do - # Assume primary key is 'id' for simplicity - placeholders = Enum.map(record_pks, fn _ -> "?" end) |> Enum.join(", ") + {:ok, pid} -> + try do + # Assume primary key is 'id' for simplicity + placeholders = Enum.map_join(record_pks, ", ", fn _ -> "?" end) delete_sql = "DELETE FROM `#{sink.table_name}` WHERE `id` IN (#{placeholders})" case MyXQL.query(pid, delete_sql, record_pks) do @@ -116,17 +115,14 @@ defmodule Sequin.Sinks.Mysql.Client do defp build_upsert_query(%MysqlSink{} = sink, records) do case extract_columns_and_values(records) do {:ok, {columns, values}} -> - column_list = Enum.map(columns, &"`#{&1}`") |> Enum.join(", ") - placeholders = Enum.map(columns, fn _ -> "?" end) |> Enum.join(", ") + column_list = Enum.map_join(columns, ", ", &"`#{&1}`") + placeholders = Enum.map_join(columns, ", ", fn _ -> "?" end) - update_clause = - columns - |> Enum.map(&"`#{&1}` = VALUES(`#{&1}`)") - |> Enum.join(", ") + update_clause = Enum.map_join(columns, ", ", &"`#{&1}` = VALUES(`#{&1}`)") sql = """ INSERT INTO `#{sink.table_name}` (#{column_list}) - VALUES #{Enum.map(values, fn _ -> "(#{placeholders})" end) |> Enum.join(", ")} + VALUES #{Enum.map_join(values, ", ", fn _ -> "(#{placeholders})" end)} ON DUPLICATE KEY UPDATE #{update_clause} """ @@ -142,12 +138,12 @@ defmodule Sequin.Sinks.Mysql.Client do defp build_insert_query(%MysqlSink{} = sink, records) do case extract_columns_and_values(records) do {:ok, {columns, values}} -> - column_list = Enum.map(columns, &"`#{&1}`") |> Enum.join(", ") - placeholders = Enum.map(columns, fn _ -> "?" end) |> Enum.join(", ") + column_list = Enum.map_join(columns, ", ", &"`#{&1}`") + placeholders = Enum.map_join(columns, ", ", fn _ -> "?" end) sql = """ INSERT INTO `#{sink.table_name}` (#{column_list}) - VALUES #{Enum.map(values, fn _ -> "(#{placeholders})" end) |> Enum.join(", ")} + VALUES #{Enum.map_join(values, ", ", fn _ -> "(#{placeholders})" end)} """ params = Enum.flat_map(values, & &1) @@ -191,6 +187,26 @@ defmodule Sequin.Sinks.Mysql.Client do end end + defp format_error(%DBConnection.ConnectionError{reason: reason, message: message} = error) do + message = format_connection_error_reason(reason, message) + + Error.service( + service: :mysql, + message: message, + details: %{original_error: error} + ) + end + + defp format_error(%MyXQL.Error{mysql: %{code: code, name: name}} = error) when not is_nil(code) do + message = format_mysql_error(code, name, Exception.message(error)) + + Error.service( + service: :mysql, + message: message, + details: %{mysql_error: error, code: code, name: name} + ) + end + defp format_error(%MyXQL.Error{} = error) do Error.service( service: :mysql, @@ -210,4 +226,88 @@ defmodule Sequin.Sinks.Mysql.Client do details: %{original_error: error} ) end + + defp format_connection_error_reason(reason, message) do + # First check the reason field + case reason do + :queue_timeout -> + """ + Connection test failed. Unable to connect to MySQL database. + + Please check: + • Database server is running and accessible + • Host and port are correct + • Network connectivity (firewall, security groups) + • Username and password are valid + • Database exists and user has access permissions + """ + + :error -> + # For :error reason, parse the message for more specific error types + parse_connection_message(message) + + _ -> + "Connection failed: #{inspect(reason)}" + end + end + + defp parse_connection_message(message) when is_binary(message) do + cond do + String.contains?(message, "econnrefused") -> + """ + Connection refused. Please check: + • The database server is running + • The host and port are correct + • Firewall settings allow connections + """ + + String.contains?(message, "timeout") -> + """ + Connection timed out. Please verify: + • The hostname and port are correct + • The database server is running + • Network connectivity is available + """ + + String.contains?(message, "nxdomain") -> + "Unable to resolve the hostname. Please check if the hostname is correct." + + String.contains?(message, "Access denied") -> + "Authentication failed. Please verify your username and password." + + true -> + """ + Connection failed. Please verify: + • Database server is running and accessible + • Host and port are correct + • Username and password are valid + • Database name is correct + + Technical details: #{message} + """ + end + end + + defp parse_connection_message(message) do + "Connection failed: #{inspect(message)}" + end + + defp format_mysql_error(code, name, original_message) do + case code do + 1045 -> "Authentication failed. Please verify your username and password." + 1049 -> "Database '#{extract_database_name(original_message)}' does not exist. Please verify the database name." + 1044 -> "Access denied to database. Please verify the user has access permissions." + 1146 -> "Table does not exist. Please verify the table name and ensure it exists in the database." + 1062 -> "Duplicate entry error. #{original_message}" + 1054 -> "Unknown column error. #{original_message}" + _ -> "MySQL error (#{code}/#{name}): #{original_message}" + end + end + + defp extract_database_name(message) do + case Regex.run(~r/database '([^']+)'/, message) do + [_, db_name] -> db_name + _ -> "unknown" + end + end end diff --git a/test/sequin/mysql_client_test.exs b/test/sequin/mysql_client_test.exs index dcfa45a75..d466c2925 100644 --- a/test/sequin/mysql_client_test.exs +++ b/test/sequin/mysql_client_test.exs @@ -16,7 +16,7 @@ defmodule Sequin.Sinks.Mysql.ClientTest do timeout_seconds: 30 } - # Integration tests would be needed to test the client functions + # Integration tests would be needed to test the client functions # against a real MySQL database, but that's beyond the scope of unit testing end diff --git a/test/sequin/mysql_pipeline_test.exs b/test/sequin/mysql_pipeline_test.exs index e51a7be3a..51ba1750d 100644 --- a/test/sequin/mysql_pipeline_test.exs +++ b/test/sequin/mysql_pipeline_test.exs @@ -49,8 +49,7 @@ defmodule Sequin.Runtime.MysqlPipelineTest do ) ) - MysqlMock - |> expect(:upsert_records, fn sink, records -> + expect(MysqlMock, :upsert_records, fn sink, records -> assert sink.table_name == consumer.sink.table_name assert length(records) == 1 assert List.first(records)["name"] == "test-name" @@ -80,8 +79,7 @@ defmodule Sequin.Runtime.MysqlPipelineTest do ) end - MysqlMock - |> expect(:upsert_records, fn sink, records -> + expect(MysqlMock, :upsert_records, fn sink, records -> assert sink.table_name == consumer.sink.table_name assert length(records) == 3 assert Enum.map(records, & &1["name"]) == ["test-name-1", "test-name-2", "test-name-3"] @@ -110,8 +108,7 @@ defmodule Sequin.Runtime.MysqlPipelineTest do ) ) - MysqlMock - |> expect(:delete_records, fn sink, record_pks -> + expect(MysqlMock, :delete_records, fn sink, record_pks -> assert sink.table_name == consumer.sink.table_name assert record_pks == [1] :ok @@ -140,8 +137,7 @@ defmodule Sequin.Runtime.MysqlPipelineTest do error = Sequin.Error.service(service: :mysql, message: "Connection failed") - MysqlMock - |> expect(:upsert_records, fn _sink, _records -> + expect(MysqlMock, :upsert_records, fn _sink, _records -> {:error, error} end) @@ -178,8 +174,7 @@ defmodule Sequin.Runtime.MysqlPipelineTest do ) ) - MysqlMock - |> expect(:upsert_records, fn sink, records -> + expect(MysqlMock, :upsert_records, fn sink, records -> assert sink.table_name == "dynamic_table" assert length(records) == 1 :ok diff --git a/test/sequin/mysql_sink_test.exs b/test/sequin/mysql_sink_test.exs index c6ce941a6..961e6bce1 100644 --- a/test/sequin/mysql_sink_test.exs +++ b/test/sequin/mysql_sink_test.exs @@ -37,7 +37,7 @@ defmodule Sequin.Consumers.MysqlSinkTest do changeset = MysqlSink.changeset(%MysqlSink{}, %{params | port: 0}) assert Sequin.Error.errors_on(changeset)[:port] == ["must be greater than 0"] - changeset = MysqlSink.changeset(%MysqlSink{}, %{params | port: 70000}) + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | port: 70_000}) assert Sequin.Error.errors_on(changeset)[:port] == ["must be less than or equal to 65535"] end @@ -45,7 +45,7 @@ defmodule Sequin.Consumers.MysqlSinkTest do changeset = MysqlSink.changeset(%MysqlSink{}, %{params | batch_size: 0}) assert Sequin.Error.errors_on(changeset)[:batch_size] == ["must be greater than 0"] - changeset = MysqlSink.changeset(%MysqlSink{}, %{params | batch_size: 20000}) + changeset = MysqlSink.changeset(%MysqlSink{}, %{params | batch_size: 20_000}) assert Sequin.Error.errors_on(changeset)[:batch_size] == ["must be less than or equal to 10000"] end @@ -59,15 +59,21 @@ defmodule Sequin.Consumers.MysqlSinkTest do test "validates table_name format", %{valid_params: params} do valid_names = ["table", "user_data", "Table123", "_private"] + for name <- valid_names do changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: name}) assert Sequin.Error.errors_on(changeset)[:table_name] == nil, "#{name} should be valid" end invalid_names = ["123table", "table-name", "table space", ""] + for name <- invalid_names do changeset = MysqlSink.changeset(%MysqlSink{}, %{params | table_name: name}) - assert Sequin.Error.errors_on(changeset)[:table_name] == ["must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)"], "#{name} should be invalid" + + assert Sequin.Error.errors_on(changeset)[:table_name] == [ + "must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)" + ], + "#{name} should be invalid" end end From 1289906eef12d521c1c52da56d3bb11998b9a99d Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:53:01 -0400 Subject: [PATCH 26/30] fix: correct function call for environment configuration in MySQL sink module --- lib/sequin/sinks/mysql.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequin/sinks/mysql.ex b/lib/sequin/sinks/mysql.ex index b7487771e..a79fa57a7 100644 --- a/lib/sequin/sinks/mysql.ex +++ b/lib/sequin/sinks/mysql.ex @@ -7,7 +7,7 @@ defmodule Sequin.Sinks.Mysql do @callback upsert_records(MysqlSink.t(), [map()]) :: :ok | {:error, Error.t()} @callback delete_records(MysqlSink.t(), [any()]) :: :ok | {:error, Error.t()} - @module Application.compiled_env(:sequin, :mysql_module, Sequin.Sinks.Mysql.Client) + @module Application.compile_env(:sequin, :mysql_module, Sequin.Sinks.Mysql.Client) @spec test_connection(MysqlSink.t()) :: :ok | {:error, Error.t()} def test_connection(%MysqlSink{} = sink) do From 2f9303a0e23ba7885cdb913d23f9b31c15b58761 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 17:59:07 -0400 Subject: [PATCH 27/30] feat: include password field in consumer form for MySQL sink configuration --- lib/sequin_web/live/components/consumer_form.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sequin_web/live/components/consumer_form.ex b/lib/sequin_web/live/components/consumer_form.ex index fa0478cae..944d9347b 100644 --- a/lib/sequin_web/live/components/consumer_form.ex +++ b/lib/sequin_web/live/components/consumer_form.ex @@ -1276,6 +1276,7 @@ defmodule SequinWeb.Components.ConsumerForm do "database" => sink.database, "table_name" => sink.table_name, "username" => sink.username, + "password" => sink.password, "ssl" => sink.ssl, "batch_size" => sink.batch_size, "timeout_seconds" => sink.timeout_seconds, From 17653954c380cced292dcc4d39c84d2dee967768 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 18:01:50 -0400 Subject: [PATCH 28/30] feat: add MySQL routing function to edit module and factory --- lib/sequin_web/live/functions/edit.ex | 16 +++++++++++++++- test/support/factory/functions_factory.ex | 7 +++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/sequin_web/live/functions/edit.ex b/lib/sequin_web/live/functions/edit.ex index d319296aa..cf8161bc2 100644 --- a/lib/sequin_web/live/functions/edit.ex +++ b/lib/sequin_web/live/functions/edit.ex @@ -190,6 +190,19 @@ defmodule SequinWeb.FunctionsLive.Edit do end """ + @initial_route_mysql """ + def route(action, record, changes, metadata) do + # Route to table based on source table name + table_name = if metadata.table_schema do + "\#{metadata.table_schema}_\#{metadata.table_name}" + else + metadata.table_name + end + + %{table_name: table_name} + end + """ + @initial_filter """ def filter(action, record, changes, metadata) do # Must return true or false! @@ -231,7 +244,8 @@ defmodule SequinWeb.FunctionsLive.Edit do "routing_rabbitmq" => @initial_route_rabbitmq, "routing_azure_event_hub" => @initial_route_azure_event_hub, "routing_kinesis" => @initial_route_kinesis, - "routing_s2" => @initial_route_s2 + "routing_s2" => @initial_route_s2, + "routing_mysql" => @initial_route_mysql } # We generate the function completions at compile time because diff --git a/test/support/factory/functions_factory.ex b/test/support/factory/functions_factory.ex index d3bee5f48..e03bedb73 100644 --- a/test/support/factory/functions_factory.ex +++ b/test/support/factory/functions_factory.ex @@ -272,6 +272,13 @@ defmodule Sequin.Factory.FunctionsFactory do exchange_name: metadata.table_name } """ + + :mysql -> + """ + %{ + table_name: metadata.table_name + } + """ end end) From f2738f57be8194c0a609f020565f830b0a511020 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 7 Aug 2025 18:05:37 -0400 Subject: [PATCH 29/30] feat: add routing validation to MySQL sink configuration --- lib/sequin/consumers/mysql_sink.ex | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/sequin/consumers/mysql_sink.ex b/lib/sequin/consumers/mysql_sink.ex index 233c30d05..e743b19fc 100644 --- a/lib/sequin/consumers/mysql_sink.ex +++ b/lib/sequin/consumers/mysql_sink.ex @@ -42,7 +42,8 @@ defmodule Sequin.Consumers.MysqlSink do :upsert_on_duplicate, :routing_mode ]) - |> validate_required([:host, :database, :table_name, :username, :password]) + |> validate_required([:host, :database, :username, :password]) + |> validate_routing() |> validate_number(:port, greater_than: 0, less_than_or_equal_to: 65_535) |> validate_number(:batch_size, greater_than: 0, less_than_or_equal_to: 10_000) |> validate_number(:timeout_seconds, greater_than: 0, less_than_or_equal_to: 300) @@ -53,6 +54,21 @@ defmodule Sequin.Consumers.MysqlSink do |> validate_table_name() end + defp validate_routing(changeset) do + routing_mode = get_field(changeset, :routing_mode) + + cond do + routing_mode == :dynamic -> + put_change(changeset, :table_name, nil) + + routing_mode == :static -> + validate_required(changeset, [:table_name]) + + true -> + add_error(changeset, :routing_mode, "is required") + end + end + defp validate_table_name(changeset) do validate_format(changeset, :table_name, ~r/^[a-zA-Z_][a-zA-Z0-9_]*$/, message: "must be a valid MySQL table name (alphanumeric and underscores, starting with letter or underscore)" From f868d56f4318107ba937b8df043d809f161820f2 Mon Sep 17 00:00:00 2001 From: Yordis Prieto Date: Thu, 21 Aug 2025 15:20:58 -0400 Subject: [PATCH 30/30] feat: add MySQL sink feature toggle and update consumer handling --- assets/svelte/consumers/SinkIndex.svelte | 12 +++++++++++- config/config.exs | 3 ++- config/runtime.exs | 16 ++++++++++++++-- lib/sequin_web/live/sink_consumers/index.ex | 2 ++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/assets/svelte/consumers/SinkIndex.svelte b/assets/svelte/consumers/SinkIndex.svelte index 4d3318fb7..3bf288261 100644 --- a/assets/svelte/consumers/SinkIndex.svelte +++ b/assets/svelte/consumers/SinkIndex.svelte @@ -73,6 +73,7 @@ export let live: any; export let hasDatabases: boolean; export let selfHosted: boolean; + export let isMysqlSinkEnabled: boolean; export let page: number; export let pageSize: number; export let totalCount: number; @@ -85,6 +86,15 @@ $: startIndex = page * pageSize + 1; $: endIndex = Math.min((page + 1) * pageSize, totalCount); + function isSinkEnabled(sink: { id: string }): boolean { + switch (sink.id) { + case "mysql": + return isMysqlSinkEnabled; + default: + return true; + } + } + const sinks = [ { id: "http_push", @@ -171,7 +181,7 @@ name: "MySQL", icon: MysqlIcon, }, - ]; + ].filter(isSinkEnabled); function handleConsumerClick(id: string, type: string) { // Store current page before navigation diff --git a/config/config.exs b/config/config.exs index 591db31a5..e4286e67e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -123,7 +123,8 @@ config :sequin, datadog_req_opts: [], datadog: [configured: false], api_base_url: "http://localhost:4000", - message_handler_module: Sequin.Runtime.MessageHandler + message_handler_module: Sequin.Runtime.MessageHandler, + features: [mysql_sink: :disabled] # Configure tailwind (the version is required) config :tailwind, diff --git a/config/runtime.exs b/config/runtime.exs index 682f94a93..e841da867 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -108,6 +108,11 @@ if config_env() == :prod and self_hosted do do: :enabled, else: :disabled + mysql_sink = + if System.get_env("FEATURE_MYSQL_SINK", "disabled") in enabled_feature_values, + do: :enabled, + else: :disabled + backfill_max_pending_messages = ConfigParser.backfill_max_pending_messages(env_vars) database_url = @@ -203,7 +208,8 @@ if config_env() == :prod and self_hosted do config :sequin, :features, account_self_signup: account_self_signup, provision_default_user: provision_default_user, - function_transforms: :enabled + function_transforms: :enabled, + mysql_sink: mysql_sink config :sequin, :koala, public_key: "pk_ec2e6140b3d56f5eb1735350eb20e92b8002", @@ -223,6 +229,11 @@ if config_env() == :prod and not self_hosted do function_transforms = if System.get_env("FEATURE_FUNCTION_TRANSFORMS", "disabled") in enabled_feature_values, do: :enabled, else: :disabled + mysql_sink = + if System.get_env("FEATURE_MYSQL_SINK", "disabled") in enabled_feature_values, + do: :enabled, + else: :disabled + config :logger, default_handler: [ formatter: {Datadog, metadata: :all, redactors: [{Redactor, []}]} @@ -265,7 +276,8 @@ if config_env() == :prod and not self_hosted do config :sequin, :features, account_self_signup: :enabled, - function_transforms: function_transforms + function_transforms: function_transforms, + mysql_sink: mysql_sink config :sequin, :koala, public_key: "pk_ec2e6140b3d56f5eb1735350eb20e92b8002" diff --git a/lib/sequin_web/live/sink_consumers/index.ex b/lib/sequin_web/live/sink_consumers/index.ex index 90b8b2983..1928f615a 100644 --- a/lib/sequin_web/live/sink_consumers/index.ex +++ b/lib/sequin_web/live/sink_consumers/index.ex @@ -56,6 +56,7 @@ defmodule SequinWeb.SinkConsumersLive.Index do socket |> assign(:has_databases?, has_databases?) |> assign(:self_hosted, Application.get_env(:sequin, :self_hosted)) + |> assign(:is_mysql_sink_enabled, Sequin.feature_enabled?(account.id, :mysql_sink)) {:ok, socket} end @@ -77,6 +78,7 @@ defmodule SequinWeb.SinkConsumersLive.Index do consumers: @encoded_consumers, hasDatabases: @has_databases?, selfHosted: @self_hosted, + isMysqlSinkEnabled: @is_mysql_sink_enabled, page: @page, pageSize: @page_size, totalCount: @total_count