Merge pull request ClickHouse#76974 from Avogar/variant-bool-parsing

Don't parse special Bool values in text formats inside Variant type by default

Author: Avogar

docs/en/sql-reference/data-types/variant.md

@@ -226,6 +226,19 @@ SELECT CAST(map('key1', '42', 'key2', 'true', 'key3', '2020-01-01'), 'Map(String
 └─────────────────────────────────────────────┴───────────────────────────────────────────────┘
 ```
 
+To disable parsing during conversion from `String` to `Variant` you can disable setting `cast_string_to_dynamic_use_inference`:
+
+```sql
+SET cast_string_to_variant_use_inference = 0;
+SELECT '[1, 2, 3]'::Variant(String, Array(UInt64)) as variant, variantType(variant) as variant_type
+```
+
+```text
+┌─variant───┬─variant_type─┐
+│ [1, 2, 3] │ String       │
+└───────────┴──────────────┘
+```
+
 ### Converting an ordinary column to a Variant column {#converting-an-ordinary-column-to-a-variant-column}
 
 It is possible to convert an ordinary column with type `T` to a `Variant` column containing this type:

src/Core/FormatFactorySettings.h

@@ -654,6 +654,10 @@ Text to represent true bool value in TSV/CSV/Vertical/Pretty formats.
 )", 0) \
     DECLARE(String, bool_false_representation, "false", R"(
 Text to represent false bool value in TSV/CSV/Vertical/Pretty formats.
+)", 0) \
+    \
+    DECLARE(Bool, allow_special_bool_values_inside_variant, false, R"(
+Allows to parse Bool values inside Variant type from special text bool values like "on", "off", "enable", "disable", etc.
 )", 0) \
     \
     DECLARE(Bool, input_format_values_interpret_expressions, true, R"(

src/Core/Settings.cpp

@@ -6074,6 +6074,9 @@ Allow to use the function `getClientHTTPHeader` which lets to obtain a value of
 )", 0) \
     DECLARE(Bool, cast_string_to_dynamic_use_inference, false, R"(
 Use types inference during String to Dynamic conversion
+)", 0) \
+    DECLARE(Bool, cast_string_to_variant_use_inference, true, R"(
+Use types inference during String to Variant conversion.
 )", 0) \
     DECLARE(Bool, enable_blob_storage_log, true, R"(
 Write information about blob storage operations to system.blob_storage_log table

src/Core/SettingsChangesHistory.cpp

@@ -75,6 +75,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"enable_hdfs_pread", true, true, "New setting."},
             {"low_priority_query_wait_time_ms", 1000, 1000, "New setting."},
             {"allow_experimental_shared_set_join", 0, 1, "A setting for ClickHouse Cloud to enable SharedSet and SharedJoin"},
+            {"allow_special_bool_values_inside_variant", true, false, "Don't allow special bool values during Variant type parsing"},
+            {"cast_string_to_variant_use_inference", true, true, "New setting to enable/disable types inference during CAST from String to Variant"},
             {"distributed_cache_read_request_max_tries", 20, 20, "New setting"},
             {"min_os_cpu_wait_time_ratio_to_throw", 0, 2, "New setting"},
             {"max_os_cpu_wait_time_ratio_to_throw", 0, 6, "New setting"},

src/DataTypes/Serializations/SerializationBool.cpp

@@ -189,7 +189,7 @@ ReturnType deserializeImpl(
     }
 
     buf.rollbackToCheckpoint();
-    if (tryDeserializeAllVariants(col, buf) && check_end_of_value(buf))
+    if (settings.allow_special_bool_values && tryDeserializeAllVariants(col, buf) && check_end_of_value(buf))
     {
         buf.dropCheckpoint();
         if (buf.hasUnreadData())

src/DataTypes/Serializations/SerializationVariant.cpp

@@ -920,7 +920,8 @@ bool SerializationVariant::tryDeserializeImpl(
     IColumn & column,
     const String & field,
     std::function<bool(ReadBuffer &)> check_for_null,
-    std::function<bool(IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer &)> try_deserialize_nested) const
+    std::function<bool(IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer &, const FormatSettings &)> try_deserialize_nested,
+    const FormatSettings & settings) const
 {
     auto & column_variant = assert_cast<ColumnVariant &>(column);
     ReadBufferFromString null_buf(field);
@@ -930,12 +931,14 @@ bool SerializationVariant::tryDeserializeImpl(
         return true;
     }
 
+    FormatSettings modified_settings = settings;
+    modified_settings.allow_special_bool_values = settings.allow_special_bool_values_inside_variant;
     for (size_t global_discr : deserialize_text_order)
     {
         ReadBufferFromString variant_buf(field);
         auto & variant_column = column_variant.getVariantByGlobalDiscriminator(global_discr);
         size_t prev_size = variant_column.size();
-        if (try_deserialize_nested(variant_column, variants[global_discr], variant_buf) && variant_buf.eof())
+        if (try_deserialize_nested(variant_column, variants[global_discr], variant_buf, modified_settings) && variant_buf.eof())
         {
             column_variant.getLocalDiscriminators().push_back(column_variant.localDiscriminatorByGlobal(global_discr));
             column_variant.getOffsets().push_back(prev_size);
@@ -981,12 +984,12 @@ bool SerializationVariant::tryDeserializeTextEscapedImpl(DB::IColumn & column, c
     {
         return SerializationNullable::tryDeserializeNullEscaped(buf, settings);
     };
-    auto try_deserialize_variant =[&](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf)
+    auto try_deserialize_variant = [](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf, const FormatSettings & settings_)
     {
-        return variant_serialization->tryDeserializeTextEscaped(variant_column, buf, settings);
+        return variant_serialization->tryDeserializeTextEscaped(variant_column, buf, settings_);
     };
 
-    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant);
+    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant, settings);
 }
 
 void SerializationVariant::serializeTextRaw(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@@ -1020,12 +1023,12 @@ bool SerializationVariant::tryDeserializeTextRawImpl(DB::IColumn & column, const
     {
         return SerializationNullable::tryDeserializeNullRaw(buf, settings);
     };
-    auto try_deserialize_variant =[&](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf)
+    auto try_deserialize_variant = [](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf, const FormatSettings & settings_)
     {
-        return variant_serialization->tryDeserializeTextRaw(variant_column, buf, settings);
+        return variant_serialization->tryDeserializeTextRaw(variant_column, buf, settings_);
     };
 
-    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant);
+    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant, settings);
 }
 
 void SerializationVariant::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@@ -1060,12 +1063,12 @@ bool SerializationVariant::tryDeserializeTextQuotedImpl(DB::IColumn & column, co
     {
         return SerializationNullable::tryDeserializeNullQuoted(buf);
     };
-    auto try_deserialize_variant =[&](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf)
+    auto try_deserialize_variant = [](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf, const FormatSettings & settings_)
     {
-        return variant_serialization->tryDeserializeTextQuoted(variant_column, buf, settings);
+        return variant_serialization->tryDeserializeTextQuoted(variant_column, buf, settings_);
     };
 
-    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant);
+    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant, settings);
 }
 
 void SerializationVariant::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@@ -1099,12 +1102,12 @@ bool SerializationVariant::tryDeserializeTextCSVImpl(DB::IColumn & column, const
     {
         return SerializationNullable::tryDeserializeNullCSV(buf, settings);
     };
-    auto try_deserialize_variant =[&](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf)
+    auto try_deserialize_variant = [](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf, const FormatSettings & settings_)
     {
-        return variant_serialization->tryDeserializeTextCSV(variant_column, buf, settings);
+        return variant_serialization->tryDeserializeTextCSV(variant_column, buf, settings_);
     };
 
-    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant);
+    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant, settings);
 }
 
 void SerializationVariant::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@@ -1138,12 +1141,12 @@ bool SerializationVariant::tryDeserializeWholeTextImpl(DB::IColumn & column, con
     {
         return SerializationNullable::tryDeserializeNullText(buf);
     };
-    auto try_deserialize_variant =[&](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf)
+    auto try_deserialize_variant = [](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf, const FormatSettings & settings_)
     {
-        return variant_serialization->tryDeserializeWholeText(variant_column, buf, settings);
+        return variant_serialization->tryDeserializeWholeText(variant_column, buf, settings_);
     };
 
-    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant);
+    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant, settings);
 }
 
 void SerializationVariant::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@@ -1188,12 +1191,12 @@ bool SerializationVariant::tryDeserializeTextJSONImpl(DB::IColumn & column, cons
     {
         return SerializationNullable::tryDeserializeNullJSON(buf);
     };
-    auto try_deserialize_variant =[&](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf)
+    auto try_deserialize_variant = [](IColumn & variant_column, const SerializationPtr & variant_serialization, ReadBuffer & buf, const FormatSettings & settings_)
     {
-        return variant_serialization->tryDeserializeTextJSON(variant_column, buf, settings);
+        return variant_serialization->tryDeserializeTextJSON(variant_column, buf, settings_);
     };
 
-    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant);
+    return tryDeserializeImpl(column, field, check_for_null, try_deserialize_variant, settings);
 }
 
 void SerializationVariant::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

src/DataTypes/Serializations/SerializationVariant.h

@@ -219,7 +219,8 @@ class SerializationVariant : public ISerialization
         IColumn & column,
         const String & field,
         std::function<bool(ReadBuffer &)> check_for_null,
-        std::function<bool(IColumn & variant_columm, const SerializationPtr & nested, ReadBuffer &)> try_deserialize_nested) const;
+        std::function<bool(IColumn & variant_columm, const SerializationPtr & nested, ReadBuffer &, const FormatSettings &)> try_deserialize_nested,
+        const FormatSettings & settings) const;
 
     VariantSerializations variants;
     std::vector<String> variant_names;

src/Formats/FormatFactory.cpp

@@ -335,6 +335,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.date_time_overflow_behavior = settings[Setting::date_time_overflow_behavior];
     format_settings.try_infer_variant = settings[Setting::input_format_try_infer_variants];
     format_settings.client_protocol_version = context->getClientProtocolVersion();
+    format_settings.allow_special_bool_values_inside_variant = settings[Setting::allow_special_bool_values_inside_variant];
 
     /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context
     if (format_settings.schema.is_server)

src/Formats/FormatSettings.h

@@ -156,6 +156,8 @@ struct FormatSettings
 
     String bool_true_representation = "true";
     String bool_false_representation = "false";
+    bool allow_special_bool_values = true;
+    bool allow_special_bool_values_inside_variant = false;
 
     struct CSV
     {

src/Functions/FunctionsConversion.h

@@ -83,6 +83,7 @@ namespace Setting
 {
     extern const SettingsBool cast_ipv4_ipv6_default_on_conversion_error;
     extern const SettingsBool cast_string_to_dynamic_use_inference;
+    extern const SettingsBool cast_string_to_variant_use_inference;
     extern const SettingsDateTimeOverflowBehavior date_time_overflow_behavior;
     extern const SettingsBool input_format_ipv4_default_on_conversion_error;
     extern const SettingsBool input_format_ipv6_default_on_conversion_error;
@@ -5170,7 +5171,7 @@ class FunctionCast final : public IFunctionBase
 
         auto variant_discr_opt = to_variant.tryGetVariantDiscriminator(removeNullableOrLowCardinalityNullable(from_type)->getName());
         /// Cast String to Variant through parsing if it's not Variant(String).
-        if (isStringOrFixedString(removeNullable(removeLowCardinality(from_type))) && (!variant_discr_opt || to_variant.getVariants().size() > 1))
+        if (context && context->getSettingsRef()[Setting::cast_string_to_variant_use_inference] && isStringOrFixedString(removeNullable(removeLowCardinality(from_type))) && (!variant_discr_opt || to_variant.getVariants().size() > 1))
             return createStringToVariantWrapper();
 
         if (!variant_discr_opt)