Skip to content

Commit a6d0527

Browse files
Only set json_type_escape_dots_in_keys if CH version > 25.8 (#3611)
### Why Setting introduced in #3606 only exists in ClickHouse versions 25.8 and above according to https://clickhouse.com/docs/sql-reference/data-types/newjson#handling-json-keys-with-dots. Lower versions of ClickHouse fail in initial load with this error: ``` Setting json_type_escape_dots_in_keys is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings ``` ### What This PR only sets this setting if CH version is > 25.8
1 parent 4149009 commit a6d0527

File tree

3 files changed

+13
-2
lines changed

3 files changed

+13
-2
lines changed

flow/connectors/clickhouse/avro_sync.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ func (s *ClickHouseAvroSyncMethod) pushS3DataToClickHouse(
286286
"throw_on_max_partitions_per_insert_block": "0",
287287
"type_json_skip_duplicated_paths": "1",
288288
}
289-
if config.Version >= shared.InternalVersion_JsonEscapeDotsInKeys {
289+
if config.Version >= shared.InternalVersion_JsonEscapeDotsInKeys && ShouldUseJSONEscapeDotsInKeys(ctx, s.chVersion) {
290290
settings["json_type_escape_dots_in_keys"] = "1"
291291
}
292292

flow/connectors/clickhouse/normalize_query.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,7 @@ func (t *NormalizeQueryGenerator) BuildQuery(ctx context.Context) (string, error
316316
if t.cluster {
317317
settings["parallel_distributed_insert_select"] = "0"
318318
}
319-
if t.version >= shared.InternalVersion_JsonEscapeDotsInKeys {
319+
if t.version >= shared.InternalVersion_JsonEscapeDotsInKeys && ShouldUseJSONEscapeDotsInKeys(ctx, t.chVersion) {
320320
settings["json_type_escape_dots_in_keys"] = "1"
321321
}
322322
settingsStr := buildSettingsStr(settings)

flow/connectors/clickhouse/validate.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import (
77
"slices"
88
"strings"
99

10+
chproto "github.com/ClickHouse/clickhouse-go/v2/lib/proto"
11+
1012
"github.com/PeerDB-io/peerdb/flow/generated/protos"
1113
"github.com/PeerDB-io/peerdb/flow/internal"
1214
chvalidate "github.com/PeerDB-io/peerdb/flow/pkg/clickhouse"
@@ -85,3 +87,12 @@ func (c *ClickHouseConnector) ValidateMirrorDestination(
8587
}
8688
return nil
8789
}
90+
91+
func ShouldUseJSONEscapeDotsInKeys(ctx context.Context, chVersion *chproto.Version) bool {
92+
if chVersion == nil {
93+
return false
94+
}
95+
// https://clickhouse.com/docs/operations/settings/formats#json_type_escape_dots_in_keys
96+
// Available in ClickHouse 25.8 and later
97+
return chproto.CheckMinVersion(chproto.Version{Major: 25, Minor: 8, Patch: 0}, *chVersion)
98+
}

0 commit comments

Comments
 (0)