diff --git a/internal-packages/clickhouse/src/taskRuns.test.ts b/internal-packages/clickhouse/src/taskRuns.test.ts index 08adb1f2ba..6aa9ae8ab9 100644 --- a/internal-packages/clickhouse/src/taskRuns.test.ts +++ b/internal-packages/clickhouse/src/taskRuns.test.ts @@ -344,4 +344,54 @@ describe("Task Runs V2", () => { expect(result2).toEqual([]); } ); + + clickhouseTest( + "should be able to insert payloads with a duplicate path", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insertPayloads = insertRawTaskRunPayloads(client, { + async_insert: 0, // turn off async insert for this test + }); + + const [insertPayloadsError, insertPayloadsResult] = await insertPayloads([ + { + run_id: "run_1234", + created_at: Date.now(), + payload: { + data: { + title: { + id: "123", + }, + "title.id": 123, + }, + }, + }, + ]); + + expect(insertPayloadsError).toBeNull(); + expect(insertPayloadsResult).toEqual(expect.objectContaining({ executed: true })); + expect(insertPayloadsResult?.summary?.written_rows).toEqual("1"); + + const queryPayloads = client.query({ + name: "query-raw-task-run-payloads", + query: "SELECT * FROM trigger_dev.raw_task_runs_payload_v1", + schema: z.object({ + run_id: z.string(), + created_at: z.coerce.date(), + payload: z.unknown(), + }), + }); + + const [queryPayloadsError, resultPayloads] = await queryPayloads({ run_id: "run_1234" }); + + expect(queryPayloadsError).toBeNull(); + expect(resultPayloads).toEqual( + expect.arrayContaining([expect.objectContaining({ run_id: "run_1234" })]) + ); + } + ); }); diff --git a/internal-packages/clickhouse/src/taskRuns.ts b/internal-packages/clickhouse/src/taskRuns.ts index 294e47fde0..aff8af829c 100644 --- a/internal-packages/clickhouse/src/taskRuns.ts +++ b/internal-packages/clickhouse/src/taskRuns.ts @@ -59,6 +59,7 @@ export function insertTaskRuns(ch: ClickhouseWriter, settings?: ClickHouseSettin async_insert_max_data_size: "1000000", async_insert_busy_timeout_ms: 1000, enable_json_type: 1, + type_json_skip_duplicated_paths: 1, ...settings, }, }); @@ -83,6 +84,7 @@ export function insertRawTaskRunPayloads(ch: ClickhouseWriter, settings?: ClickH async_insert_max_data_size: "1000000", async_insert_busy_timeout_ms: 1000, enable_json_type: 1, + type_json_skip_duplicated_paths: 1, ...settings, }, });