Add minimal Variant/Dynamic/JSON examples and tests using JSONEachRow format (#305)

Author: slvrtrn

.docker/clickhouse/single_node_tls/Dockerfile

@@ -1,4 +1,4 @@
-FROM clickhouse/clickhouse-server:24.3-alpine
+FROM clickhouse/clickhouse-server:24.8-alpine
 COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs
 RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \
     && chmod 600 /etc/clickhouse-server/certs/* \

docker-compose.cluster.yml

@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   clickhouse1:
-    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.3-alpine}'
+    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}'
     ulimits:
       nofile:
         soft: 262144
@@ -19,7 +19,7 @@ services:
       - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml'
 
   clickhouse2:
-    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.3-alpine}'
+    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}'
     ulimits:
       nofile:
         soft: 262144

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   clickhouse:
-    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.3-alpine}'
+    image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}'
     container_name: 'clickhouse-js-clickhouse-server'
     ports:
       - '8123:8123'

examples/README.md

@@ -58,6 +58,10 @@ If something is missing, or you found a mistake in one of these examples, please
 - [select_streaming_json_each_row_for_await.ts](node/select_streaming_json_each_row_for_await.ts) - (Node.js only) similar to [select_streaming_json_each_row.ts](node/select_streaming_json_each_row.ts), but using the `for await` loop syntax.
 - [select_streaming_text_line_by_line.ts](node/select_streaming_text_line_by_line.ts) - (Node.js only) streaming text formats from ClickHouse and processing it line by line. In this example, CSV format is used.
 
+#### Data types
+
+- [dynamic_variant_json.ts](./dynamic_variant_json.ts) - using experimental [Dynamic](https://clickhouse.com/docs/en/sql-reference/data-types/dynamic)/[Variant](https://clickhouse.com/docs/en/sql-reference/data-types/variant)/[JSON](https://clickhouse.com/docs/en/sql-reference/data-types/newjson) data types with [JSONEachRow](https://clickhouse.com/docs/en/interfaces/formats#jsoneachrow) format.
+
 #### Special cases
 
 - [default_format_setting.ts](default_format_setting.ts) - sending queries using `exec` method without a `FORMAT` clause; the default format will be set from the client settings.

examples/dynamic_variant_json.ts

@@ -0,0 +1,68 @@
+import { createClient } from '@clickhouse/client' // or '@clickhouse/client-web'
+
+void (async () => {
+  const tableName = `chjs_dynamic_variant_json`
+  const client = createClient({
+    clickhouse_settings: {
+      // Since ClickHouse 24.1
+      allow_experimental_variant_type: 1,
+      // Since ClickHouse 24.5
+      allow_experimental_dynamic_type: 1,
+      // Since ClickHouse 24.8
+      allow_experimental_json_type: 1,
+    },
+  })
+  await client.command({
+    query: `
+      CREATE OR REPLACE TABLE ${tableName}
+      (
+        id              UInt64,
+        var             Variant(Int64, String),
+        dynamic         Dynamic,
+        json            JSON
+      )
+      ENGINE MergeTree
+      ORDER BY id
+    `,
+  })
+  // Sample representation in JSONEachRow format
+  const values = [
+    {
+      id: 1,
+      var: 42,
+      dynamic: 'foo',
+      json: {
+        foo: 'x',
+      },
+    },
+    {
+      id: 2,
+      var: 'str',
+      // defaults to Int64; will be represented as a string in JSON* family formats
+      // this behavior can be changed with `output_format_json_quote_64bit_integers` setting (default is 1).
+      // see https://clickhouse.com/docs/en/operations/settings/formats#output_format_json_quote_64bit_integers
+      dynamic: 144,
+      json: {
+        bar: 10,
+      },
+    },
+  ]
+  await client.insert({
+    table: tableName,
+    format: 'JSONEachRow',
+    values,
+  })
+  const rs = await client.query({
+    query: `
+      SELECT *,
+             variantType(var),
+             dynamicType(dynamic),
+             dynamicType(json.foo),
+             dynamicType(json.bar)
+      FROM ${tableName}
+    `,
+    format: 'JSONEachRow',
+  })
+  console.log(await rs.json())
+  await client.close()
+})()

packages/client-common/__tests__/integration/data_types.test.ts

@@ -129,7 +129,9 @@ describe('data types', () => {
     })
     const result = await client
       .query({
-        query: `SELECT * FROM ${table} ORDER BY id ASC`,
+        query: `SELECT *
+                FROM ${table}
+                ORDER BY id ASC`,
         format: 'TabSeparated',
       })
       .then((r) => r.text())
@@ -238,7 +240,9 @@ describe('data types', () => {
     await insertData(table, values)
     const result = await client
       .query({
-        query: `SELECT CAST(e1, 'Int8') AS e1, CAST(e2, 'Int8') AS e2 FROM ${table} ORDER BY id ASC`,
+        query: `SELECT CAST(e1, 'Int8') AS e1, CAST(e2, 'Int8') AS e2
+                FROM ${table}
+                ORDER BY id ASC`,
         format: 'JSONEachRow',
       })
       .then((r) => r.json())
@@ -426,23 +430,26 @@ describe('data types', () => {
     expect(
       await client
         .query({
-          query: `SELECT sum(distance) FROM ${table}`,
+          query: `SELECT sum(distance)
+                  FROM ${table}`,
           format: 'TabSeparated',
         })
         .then((r) => r.text()),
     ).toEqual('125.53\n')
     expect(
       await client
         .query({
-          query: `SELECT max(distance) FROM ${table}`,
+          query: `SELECT max(distance)
+                  FROM ${table}`,
           format: 'TabSeparated',
         })
         .then((r) => r.text()),
     ).toEqual('100.52\n')
     expect(
       await client
         .query({
-          query: `SELECT uniqExact(distance) FROM ${table}`,
+          query: `SELECT uniqExact(distance)
+                  FROM ${table}`,
           format: 'TabSeparated',
         })
         .then((r) => r.text()),
@@ -504,9 +511,10 @@ describe('data types', () => {
     await insertAndAssert(table, values)
   })
 
-  // JSON cannot be used on a "modern" Cloud instance
-  whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster, TestEnv.Cloud).it(
-    'should work with JSON',
+  // New experimental JSON type
+  // https://clickhouse.com/docs/en/sql-reference/data-types/newjson
+  whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster).it(
+    'should work with (new) JSON',
     async () => {
       const values = [
         {
@@ -516,8 +524,40 @@ describe('data types', () => {
           o: { a: 2, b: { c: 3, d: [4, 5, 6] } },
         },
       ]
-      const table = await createTableWithFields(client, `o Object('json')`, {
-        allow_experimental_object_type: 1,
+      const table = await createTableWithFields(client, `o JSON`, {
+        allow_experimental_json_type: 1,
+      })
+      await insertAndAssert(table, values, {
+        output_format_json_quote_64bit_integers: 0,
+      })
+    },
+  )
+
+  // New experimental Variant type
+  // https://clickhouse.com/docs/en/sql-reference/data-types/variant
+  whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster).it(
+    'should work with Variant',
+    async () => {
+      const values = [{ var: 'foo' }, { var: 42 }]
+      const table = await createTableWithFields(
+        client,
+        `var Variant(String, Int32)`,
+        {
+          allow_experimental_variant_type: 1,
+        },
+      )
+      await insertAndAssert(table, values)
+    },
+  )
+
+  // New experimental Dynamic type
+  // https://clickhouse.com/docs/en/sql-reference/data-types/dynamic
+  whenOnEnv(TestEnv.LocalSingleNode, TestEnv.LocalCluster).it(
+    'should work with Dynamic',
+    async () => {
+      const values = [{ dyn: 'foo' }, { dyn: { bar: 'qux' } }]
+      const table = await createTableWithFields(client, `dyn Dynamic`, {
+        allow_experimental_dynamic_type: 1,
       })
       await insertAndAssert(table, values)
     },
@@ -655,7 +695,9 @@ describe('data types', () => {
       })
       const result = await client
         .query({
-          query: `SELECT n.id, n.name, n.createdAt, n.roles FROM ${table} ORDER BY id ASC`,
+          query: `SELECT n.id, n.name, n.createdAt, n.roles
+                  FROM ${table}
+                  ORDER BY id ASC`,
           format: 'JSONEachRow',
         })
         .then((r) => r.json())
@@ -683,25 +725,36 @@ describe('data types', () => {
   ) {
     const values = data.map((v, i) => ({ ...v, id: i + 1 }))
     await client.insert({
+      format: 'JSONEachRow',
       table,
       values,
-      format: 'JSONEachRow',
       clickhouse_settings,
     })
   }
 
-  async function assertData<T>(table: string, data: T[]) {
+  async function assertData<T>(
+    table: string,
+    data: T[],
+    clickhouse_settings: ClickHouseSettings = {},
+  ) {
     const result = await client
       .query({
-        query: `SELECT * EXCEPT (id) FROM ${table} ORDER BY id ASC`,
+        query: `SELECT * EXCEPT (id)
+                FROM ${table}
+                ORDER BY id ASC`,
         format: 'JSONEachRow',
+        clickhouse_settings,
       })
       .then((r) => r.json())
     expect(result).toEqual(data)
   }
 
-  async function insertAndAssert<T>(table: string, data: T[]) {
-    await insertData(table, data)
-    await assertData(table, data)
+  async function insertAndAssert<T>(
+    table: string,
+    data: T[],
+    clickhouse_settings: ClickHouseSettings = {},
+  ) {
+    await insertData(table, data, clickhouse_settings)
+    await assertData(table, data, clickhouse_settings)
   }
 })