Skip to content

Commit ec24633

Browse files
BenoitRanqueBenoitRanque
authored
Merge pull request #25 from hasura:fix-typeparser
if introspection returns no columns, preserve any manually written columns
2 parents a5df7ec + ec41a87 commit ec24633

File tree

82 files changed

+5477
-341
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+5477
-341
lines changed

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
## [1.0.2]
11+
12+
- Allow `DateTime64` shorthand for `DateTime64(3)`
13+
- Allow `Decimal` shorthand for `Decimal(10, 0)`
14+
- Make datatypes case insensitive
15+
- When introspection returns no columns ([parameterized view issue](https://github.com/ClickHouse/ClickHouse/issues/65402)), preserve any manually written columns
16+
- Correct support for casting from JSON Objects paramters to named Tuples, and JSON Arrays to anonymous tuples
17+
- Fix printing tuples in bound parameters
18+
1019
## [1.0.1]
1120

1221
- Bug fix: remove erroneous group by and order by clauses in `foreach` queries. Remote relationships should now function as expected. The previous fix was incorrect.

‎Cargo.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ members = [
66
]
77
resolver = "2"
88

9-
package.version = "1.0.1"
9+
package.version = "1.0.2"
1010
package.edition = "2021"

‎ci/templates/connector-metadata.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ packagingDefinition:
44
supportedEnvironmentVariables:
55
- name: CLICKHOUSE_URL
66
description: The ClickHouse connection URL
7+
defaultValue: ""
78
- name: CLICKHOUSE_USERNAME
89
description: The ClickHouse connection username
10+
defaultValue: ""
911
- name: CLICKHOUSE_PASSWORD
1012
description: The ClickHouse connection password
13+
defaultValue: ""
1114
commands:
1215
update: hasura-clickhouse update
1316
cliPlugin:

‎crates/common/src/clickhouse_parser.rs‎

Lines changed: 96 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -58,45 +58,48 @@ peg::parser! {
5858
/ aggregate_function()
5959
/ simple_aggregate_function()
6060
/ nothing()
61-
rule nullable() -> DT = "Nullable(" t:data_type() ")" { DT::Nullable(Box::new(t)) }
62-
rule uint8() -> DT = "UInt8" { DT::UInt8 }
63-
rule uint16() -> DT = "UInt16" { DT::UInt16 }
64-
rule uint32() -> DT = "UInt32" { DT::UInt32 }
65-
rule uint64() -> DT = "UInt64" { DT::UInt64 }
66-
rule uint128() -> DT = "UInt128" { DT::UInt128 }
67-
rule uint256() -> DT = "UInt256" { DT::UInt256 }
68-
rule int8() -> DT = "Int8" { DT::Int8 }
69-
rule int16() -> DT = "Int16" { DT::Int16 }
70-
rule int32() -> DT = "Int32" { DT::Int32 }
71-
rule int64() -> DT = "Int64" { DT::Int64 }
72-
rule int128() -> DT = "Int128" { DT::Int128 }
73-
rule int256() -> DT = "Int256" { DT::Int256 }
74-
rule float32() -> DT = "Float32" { DT::Float32 }
75-
rule float64() -> DT = "Float64" { DT::Float64 }
76-
rule decimal() -> DT = "Decimal(" precision:integer_value() comma_separator() scale:integer_value() ")" { DT::Decimal { precision, scale } }
77-
rule decimal32() -> DT = "Decimal32(" scale:integer_value() ")" { DT::Decimal32 { scale } }
78-
rule decimal64() -> DT = "Decimal64(" scale:integer_value() ")" { DT::Decimal64 { scale } }
79-
rule decimal128() -> DT = "Decimal128(" scale:integer_value() ")" { DT::Decimal128 { scale } }
80-
rule decimal256() -> DT = "Decimal256(" scale:integer_value() ")" { DT::Decimal256 { scale } }
81-
rule bool() -> DT = "Bool" { DT::Bool }
82-
rule string() -> DT = "String" { DT::String }
83-
rule fixed_string() -> DT = "FixedString(" n:integer_value() ")" { DT::FixedString(n) }
84-
rule date() -> DT = "Date" { DT::Date }
85-
rule date32() -> DT = "Date32" { DT::Date32 }
86-
rule date_time() -> DT = "DateTime" tz:("(" tz:single_quoted_string_value()? ")" { tz })? { DT::DateTime { timezone: tz.flatten().map(|s| s.to_owned()) } }
87-
rule date_time64() -> DT = "DateTime64(" precision:integer_value() tz:(comma_separator() tz:single_quoted_string_value()? { tz })? ")" { DT::DateTime64{ precision, timezone: tz.flatten().map(|s| s.to_owned())} }
88-
rule uuid() -> DT = "UUID" { DT::Uuid }
89-
rule ipv4() -> DT = "IPv4" { DT::IPv4 }
90-
rule ipv6() -> DT = "IPv6" { DT::IPv6 }
91-
rule low_cardinality() -> DT = "LowCardinality(" t:data_type() ")" { DT::LowCardinality(Box::new(t)) }
92-
rule nested() -> DT = "Nested(" e:((n:identifier() __ t:data_type() { (n, t)}) ** comma_separator()) ")" { DT::Nested(e) }
93-
rule array() -> DT = "Array(" t:data_type() ")" { DT::Array(Box::new(t)) }
94-
rule map() -> DT = "Map(" k:data_type() comma_separator() v:data_type() ")" { DT::Map { key: Box::new(k), value: Box::new(v) } }
95-
rule tuple() -> DT = "Tuple(" e:((n:(n:identifier() __ { n })? t:data_type() { (n, t) }) ** comma_separator()) ")" { DT::Tuple(e) }
96-
rule r#enum() -> DT = "Enum" ("8" / "16")? "(" e:((n:single_quoted_string_value() i:(_ "=" _ i:integer_value() { i })? { (n, i) }) ** comma_separator()) ")" { DT::Enum(e)}
97-
rule aggregate_function() -> DT = "AggregateFunction(" f:aggregate_function_definition() comma_separator() a:(data_type() ** comma_separator()) ")" { DT::AggregateFunction { function: f, arguments: a }}
98-
rule simple_aggregate_function() -> DT = "SimpleAggregateFunction(" f:aggregate_function_definition() comma_separator() a:(data_type() ** comma_separator()) ")" { DT::SimpleAggregateFunction { function: f, arguments: a }}
99-
rule nothing() -> DT = "Nothing" { DT::Nothing }
61+
rule nullable() -> DT = i("Nullable(") t:data_type() ")" { DT::Nullable(Box::new(t)) }
62+
rule uint8() -> DT = i("UInt8") { DT::UInt8 }
63+
rule uint16() -> DT = i("UInt16") { DT::UInt16 }
64+
rule uint32() -> DT = i("UInt32") { DT::UInt32 }
65+
rule uint64() -> DT = i("UInt64") { DT::UInt64 }
66+
rule uint128() -> DT = i("UInt128") { DT::UInt128 }
67+
rule uint256() -> DT = i("UInt256") { DT::UInt256 }
68+
rule int8() -> DT = i("Int8") { DT::Int8 }
69+
rule int16() -> DT = i("Int16") { DT::Int16 }
70+
rule int32() -> DT = i("Int32") { DT::Int32 }
71+
rule int64() -> DT = i("Int64") { DT::Int64 }
72+
rule int128() -> DT = i("Int128") { DT::Int128 }
73+
rule int256() -> DT = i("Int256") { DT::Int256 }
74+
rule float32() -> DT = i("Float32") { DT::Float32 }
75+
rule float64() -> DT = i("Float64") { DT::Float64 }
76+
rule decimal() -> DT = i("Decimal(") precision:integer_value() comma_separator() scale:integer_value() ")" { DT::Decimal { precision, scale } }
77+
/ i("Decimal(") precision:integer_value() ")" { DT::Decimal { precision, scale: 0 } }
78+
/ i("Decimal") { DT::Decimal { precision: 10, scale: 0 }}
79+
rule decimal32() -> DT = i("Decimal32(") scale:integer_value() ")" { DT::Decimal32 { scale } }
80+
rule decimal64() -> DT = i("Decimal64(") scale:integer_value() ")" { DT::Decimal64 { scale } }
81+
rule decimal128() -> DT = i("Decimal128(") scale:integer_value() ")" { DT::Decimal128 { scale } }
82+
rule decimal256() -> DT = i("Decimal256(") scale:integer_value() ")" { DT::Decimal256 { scale } }
83+
rule bool() -> DT = i("Bool") { DT::Bool }
84+
rule string() -> DT = i("String") { DT::String }
85+
rule fixed_string() -> DT = i("FixedString(") n:integer_value() ")" { DT::FixedString(n) }
86+
rule date() -> DT = i("Date") { DT::Date }
87+
rule date32() -> DT = i("Date32") { DT::Date32 }
88+
rule date_time() -> DT = i("DateTime") tz:("(" tz:single_quoted_string_value()? ")" { tz })? { DT::DateTime { timezone: tz.flatten().map(|s| s.to_owned()) } }
89+
rule date_time64() -> DT = i("DateTime64(") precision:integer_value() tz:(comma_separator() tz:single_quoted_string_value()? { tz })? ")" { DT::DateTime64{ precision, timezone: tz.flatten().map(|s| s.to_owned())} }
90+
/ i("DateTime64") { DT::DateTime64 { precision: 3, timezone: None }}
91+
rule uuid() -> DT = i("UUID") { DT::Uuid }
92+
rule ipv4() -> DT = i("IPv4") { DT::IPv4 }
93+
rule ipv6() -> DT = i("IPv6") { DT::IPv6 }
94+
rule low_cardinality() -> DT = i("LowCardinality(") t:data_type() ")" { DT::LowCardinality(Box::new(t)) }
95+
rule nested() -> DT = i("Nested(") e:((n:identifier() __ t:data_type() { (n, t)}) ** comma_separator()) ")" { DT::Nested(e) }
96+
rule array() -> DT = i("Array(") t:data_type() ")" { DT::Array(Box::new(t)) }
97+
rule map() -> DT = i("Map(") k:data_type() comma_separator() v:data_type() ")" { DT::Map { key: Box::new(k), value: Box::new(v) } }
98+
rule tuple() -> DT = i("Tuple(") e:((n:(n:identifier() __ { n })? t:data_type() { (n, t) }) ** comma_separator()) ")" { DT::Tuple(e) }
99+
rule r#enum() -> DT = i("Enum") ("8" / "16")? "(" e:((n:single_quoted_string_value() i:(_ "=" _ i:integer_value() { i })? { (n, i) }) ** comma_separator()) ")" { DT::Enum(e)}
100+
rule aggregate_function() -> DT = i("AggregateFunction(") f:aggregate_function_definition() comma_separator() a:(data_type() ** comma_separator()) ")" { DT::AggregateFunction { function: f, arguments: a }}
101+
rule simple_aggregate_function() -> DT = i("SimpleAggregateFunction(") f:aggregate_function_definition() comma_separator() a:(data_type() ** comma_separator()) ")" { DT::SimpleAggregateFunction { function: f, arguments: a }}
102+
rule nothing() -> DT = i("Nothing") { DT::Nothing }
100103

101104
rule aggregate_function_definition() -> AggregateFunctionDefinition = n:identifier() p:("(" p:(aggregate_function_parameter() ** comma_separator()) ")" { p })? { AggregateFunctionDefinition { name: n, parameters: p }}
102105
rule aggregate_function_parameter() -> AggregateFunctionParameter = s:single_quoted_string_value() { AggregateFunctionParameter::SingleQuotedString(s)}
@@ -120,6 +123,8 @@ peg::parser! {
120123
rule _ = [' ' | '\t' | '\r' | '\n']*
121124
/// A comma surrounded by optional whitespace
122125
rule comma_separator() = _ "," _
126+
/// A case insensitive string
127+
rule i(literal: &'static str) = input:$([_]*<{literal.len()}>) {? if input.eq_ignore_ascii_case(literal) { Ok(()) } else { Err(literal) } }
123128
}
124129
}
125130

@@ -195,7 +200,58 @@ fn can_parse_clickhouse_data_type() {
195200

196201
for (s, t) in data_types {
197202
let parsed = clickhouse_parser::data_type(s);
198-
assert_eq!(parsed, Ok(t), "Able to parse correctly");
203+
assert_eq!(parsed, Ok(t), "Able to parse {s} correctly");
204+
}
205+
}
206+
207+
#[test]
208+
fn support_shorthands() {
209+
let test_cases = vec![
210+
(
211+
"Decimal",
212+
DT::Decimal {
213+
precision: 10,
214+
scale: 0,
215+
},
216+
),
217+
(
218+
"Decimal(3)",
219+
DT::Decimal {
220+
precision: 3,
221+
scale: 0,
222+
},
223+
),
224+
(
225+
"DateTime64",
226+
DT::DateTime64 {
227+
precision: 3,
228+
timezone: None,
229+
},
230+
),
231+
];
232+
233+
for (s, t) in test_cases {
234+
let parsed = clickhouse_parser::data_type(s);
235+
assert_eq!(parsed, Ok(t), "Able to parse {s} correctly");
236+
}
237+
}
238+
239+
#[test]
240+
fn is_case_insensitive() {
241+
let test_cases = vec![
242+
("dateTime", "DateTime"),
243+
("daTetIme64", "DateTime64(3)"),
244+
("bool", "Bool"),
245+
("STRING", "String"),
246+
];
247+
248+
for (input, cased) in test_cases {
249+
let parsed = clickhouse_parser::data_type(input).expect("Able to parse type");
250+
assert_eq!(
251+
&parsed.to_string(),
252+
cased,
253+
"Should parse incorrectly cased type {input} into {cased}"
254+
);
199255
}
200256
}
201257

‎crates/ndc-clickhouse-cli/src/main.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,18 @@ fn get_table_return_type(
446446
let old_return_type =
447447
old_table.and_then(
448448
|(_table_alias, table_config)| match &table_config.return_type {
449-
ReturnType::Definition { .. } => None,
449+
ReturnType::Definition { columns } => {
450+
// introspection of parameterized views may return no columns
451+
// ref: https://github.com/ClickHouse/ClickHouse/issues/65402
452+
// if introspection returned no columns, and existing config does have (user written) columns, preserve those
453+
if new_columns.is_empty() && !columns.is_empty() {
454+
Some(ReturnType::Definition {
455+
columns: columns.clone(),
456+
})
457+
} else {
458+
None
459+
}
460+
}
450461
ReturnType::TableReference { table_name } => {
451462
// get the old table config for the referenced table
452463
let referenced_table_config = old_config

‎crates/ndc-clickhouse/src/connector/handler/schema.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ use std::collections::BTreeMap;
1313
pub async fn schema(
1414
configuration: &ServerConfig,
1515
) -> Result<JsonResponse<models::SchemaResponse>, SchemaError> {
16+
Ok(JsonResponse::Value(schema_response(configuration)))
17+
}
18+
19+
pub fn schema_response(configuration: &ServerConfig) -> models::SchemaResponse {
1620
let mut scalar_type_definitions = BTreeMap::new();
1721
let mut object_type_definitions = vec![];
1822

@@ -198,13 +202,13 @@ pub async fn schema(
198202

199203
let collections = table_collections.chain(query_collections).collect();
200204

201-
Ok(JsonResponse::Value(models::SchemaResponse {
205+
models::SchemaResponse {
202206
scalar_types: scalar_type_definitions,
203207
// converting vector to map drops any duplicate definitions
204208
// this could be an issue if there are name collisions
205209
object_types: object_type_definitions.into_iter().collect(),
206210
collections,
207211
functions: vec![],
208212
procedures: vec![],
209-
}))
213+
}
210214
}

0 commit comments

Comments
 (0)