Skip to content

Commit a9532ed

Browse files
theorytheory
committed
Add JSON support for the HTTP engine
Map it to the `JSON` data type and test table importing, insertion, and selection. Add a test for both the binary and HTTP engines, and tweak the ClickHouse configuration to enable the JSON type in 24 and earlier. The Binary engine will successfully import a table with a `JSON` column but not be able to insert or select that column until `JSON` support is added to `clickhouse-cpp` (issue ClickHouse/clickhouse-cpp#422; PR ClickHouse/clickhouse-cpp#450).
1 parent 6ec5cff commit a9532ed

File tree

6 files changed

+234
-0
lines changed

6 files changed

+234
-0
lines changed

‎.github/ubuntu/clickhouse.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,11 @@ for pkg in clickhouse-common-static clickhouse-server; do
3838
rm "${pkg}.deb"
3939
done
4040

41+
if [ "${CH_VERSION%%.*}" -lt 25 ]; then
42+
# Enable the JSON type in release prior to 2025.
43+
printf "~~~~ Configuring ClickHouse %s ~~~~\n" "$CH_VERSION"
44+
perl -i -pe 's{^((\s+)<default>)}{$1\n$2 <allow_experimental_json_type>1</allow_experimental_json_type>}' /etc/clickhouse-server/users.xml
45+
fi
46+
4147
printf "~~~~ Starting ClickHouse %s ~~~~\n" "$CH_VERSION"
4248
/etc/init.d/clickhouse-server start

‎src/pglink.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,7 @@ static char *str_types_map[][2] = {
760760
{"UUID", "UUID"},
761761
{"IPv4", "inet"},
762762
{"IPv6", "inet"},
763+
{"JSON", "JSON"},
763764
{NULL, NULL},
764765
};
765766

‎test/expected/json.out‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA bin;
30+
CREATE SCHEMA http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
32+
\d bin.things
33+
Foreign table "bin.things"
34+
Column | Type | Collation | Nullable | Default | FDW options
35+
--------+---------+-----------+----------+---------+-------------
36+
id | integer | | not null | |
37+
data | json | | not null | |
38+
Server: binary_json_loopback
39+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
40+
41+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
42+
\d http.things
43+
Foreign table "http.things"
44+
Column | Type | Collation | Nullable | Default | FDW options
45+
--------+---------+-----------+----------+---------+-------------
46+
id | integer | | not null | |
47+
data | json | | not null | |
48+
Server: http_json_loopback
49+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
50+
51+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
52+
INSERT INTO bin.things VALUES
53+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
54+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
55+
;
56+
ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON
57+
INSERT INTO http.things VALUES
58+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
59+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
60+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
61+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
62+
;
63+
SELECT * FROM bin.things ORDER BY id;
64+
ERROR: pg_clickhouse: unsupported column type: JSON
65+
DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC
66+
SELECT * FROM http.things ORDER BY id;
67+
id | data
68+
----+----------------------------------------------------------
69+
1 | {"id":1,"name":"widget","size":"large","stocked":true}
70+
2 | {"id":2,"name":"sprocket","size":"small","stocked":true}
71+
3 | {"id":3,"name":"gizmo","size":"medium","stocked":true}
72+
4 | {"id":4,"name":"doodad","size":"large","stocked":false}
73+
(4 rows)
74+
75+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
76+
clickhouse_raw_query
77+
----------------------
78+
79+
(1 row)
80+
81+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
82+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
83+
DROP SERVER binary_json_loopback CASCADE;
84+
NOTICE: drop cascades to foreign table bin.things
85+
DROP SERVER http_json_loopback CASCADE;
86+
NOTICE: drop cascades to foreign table http.things

‎test/expected/json_1.out‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE json_test.things (
20+
id Int32 NOT NULL,
21+
data JSON NOT NULL
22+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
23+
$$);
24+
clickhouse_raw_query
25+
----------------------
26+
27+
(1 row)
28+
29+
CREATE SCHEMA bin;
30+
CREATE SCHEMA http;
31+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
32+
\d bin.things
33+
Foreign table "bin.things"
34+
Column | Type | Collation | Nullable | Default | FDW options
35+
--------+---------+-----------+----------+---------+-------------
36+
id | integer | | not null | |
37+
data | json | | not null | |
38+
Server: binary_json_loopback
39+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
40+
41+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
42+
\d http.things
43+
Foreign table "http.things"
44+
Column | Type | Collation | Nullable | Default | FDW options
45+
--------+---------+-----------+----------+---------+-------------
46+
id | integer | | not null | |
47+
data | json | | not null | |
48+
Server: http_json_loopback
49+
FDW options: (database 'json_test', table_name 'things', engine 'MergeTree')
50+
51+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
52+
INSERT INTO bin.things VALUES
53+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
54+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
55+
;
56+
ERROR: pg_clickhouse: could not prepare insert - unsupported column type: JSON
57+
INSERT INTO http.things VALUES
58+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
59+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
60+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
61+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
62+
;
63+
SELECT * FROM bin.things ORDER BY id;
64+
ERROR: pg_clickhouse: unsupported column type: JSON
65+
DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC
66+
SELECT * FROM http.things ORDER BY id;
67+
id | data
68+
----+------------------------------------------------------------
69+
1 | {"id":"1","name":"widget","size":"large","stocked":true}
70+
2 | {"id":"2","name":"sprocket","size":"small","stocked":true}
71+
3 | {"id":"3","name":"gizmo","size":"medium","stocked":true}
72+
4 | {"id":"4","name":"doodad","size":"large","stocked":false}
73+
(4 rows)
74+
75+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
76+
clickhouse_raw_query
77+
----------------------
78+
79+
(1 row)
80+
81+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
82+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
83+
DROP SERVER binary_json_loopback CASCADE;
84+
NOTICE: drop cascades to foreign table bin.things
85+
DROP SERVER http_json_loopback CASCADE;
86+
NOTICE: drop cascades to foreign table http.things

‎test/expected/result_map.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,15 @@ import_schema.sql
100100
ClickHouse | File
101101
------------|-------------------
102102
22-25 | import_schema.out
103+
104+
json.sql
105+
--------
106+
107+
Postgres | File
108+
----------|----------
109+
13-18 | json.out
110+
111+
ClickHouse | File
112+
------------|------------
113+
25.8+ | json.out
114+
22-25.3 | json_1.out

‎test/sql/json.sql‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'binary');
3+
CREATE SERVER http_json_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'json_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
6+
7+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS json_test');
8+
SELECT clickhouse_raw_query('CREATE DATABASE json_test');
9+
SELECT clickhouse_raw_query($$
10+
CREATE TABLE json_test.things (
11+
id Int32 NOT NULL,
12+
data JSON NOT NULL
13+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
14+
$$);
15+
16+
CREATE SCHEMA bin;
17+
CREATE SCHEMA http;
18+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER binary_json_loopback INTO bin;
19+
\d bin.things
20+
IMPORT FOREIGN SCHEMA "json_test" FROM SERVER http_json_loopback INTO http;
21+
\d http.things
22+
23+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
24+
INSERT INTO bin.things VALUES
25+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
26+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}')
27+
;
28+
29+
INSERT INTO http.things VALUES
30+
(1, '{"id": 1, "name": "widget", "size": "large", "stocked": true}'),
31+
(2, '{"id": 2, "name": "sprocket", "size": "small", "stocked": true}'),
32+
(3, '{"id": 3, "name": "gizmo", "size": "medium", "stocked": true}'),
33+
(4, '{"id": 4, "name": "doodad", "size": "large", "stocked": false}')
34+
;
35+
36+
SELECT * FROM bin.things ORDER BY id;
37+
SELECT * FROM http.things ORDER BY id;
38+
39+
SELECT clickhouse_raw_query('DROP DATABASE json_test');
40+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_json_loopback;
41+
DROP USER MAPPING FOR CURRENT_USER SERVER http_json_loopback;
42+
DROP SERVER binary_json_loopback CASCADE;
43+
DROP SERVER http_json_loopback CASCADE;

0 commit comments

Comments
 (0)