chore: added the enable_binary_to_utf8_lossy option for inserting lossy utf8 data (#18532)

* chore: added the `enable_binary_to_utf8_lossy` option for inserting lossy utf8 data

* chore: codefmt

Author: KKould

src/query/expression/src/function.rs

@@ -169,6 +169,7 @@ pub struct FunctionContext {
     pub random_function_seed: bool,
     pub week_start: u8,
     pub date_format_style: String,
+    pub enable_binary_to_utf8_lossy: bool,
 }
 
 impl Default for FunctionContext {
@@ -186,6 +187,7 @@ impl Default for FunctionContext {
             random_function_seed: false,
             week_start: 0,
             date_format_style: "oracle".to_string(),
+            enable_binary_to_utf8_lossy: false,
         }
     }
 }

src/query/functions/src/scalars/binary.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::Cow;
 use std::io::Write;
 
 use databend_common_expression::error_to_null;
@@ -209,11 +210,16 @@ fn eval_binary_to_string(val: Value<BinaryType>, ctx: &mut EvalContext) -> Value
     vectorize_binary_to_string(
         |col| col.total_bytes_len(),
         |val, output, ctx| {
-            if let Ok(val) = simdutf8::basic::from_utf8(val) {
-                output.put_str(val);
+            let val = if ctx.func_ctx.enable_binary_to_utf8_lossy {
+                String::from_utf8_lossy(val)
+            } else if let Ok(val) = simdutf8::basic::from_utf8(val) {
+                Cow::Borrowed(val)
             } else {
                 ctx.set_error(output.len(), "invalid utf8 sequence");
-            }
+                output.commit_row();
+                return;
+            };
+            output.put_str(&val);
             output.commit_row();
         },
     )(val, ctx)

src/query/service/src/sessions/query_ctx.rs

@@ -1009,6 +1009,7 @@ impl TableContext for QueryContext {
         let week_start = settings.get_week_start()? as u8;
         let date_format_style = settings.get_date_format_style()?;
         let random_function_seed = settings.get_random_function_seed()?;
+        let enable_binary_to_utf8_lossy = settings.get_enable_binary_to_utf8_lossy()?;
 
         Ok(FunctionContext {
             now,
@@ -1023,6 +1024,7 @@ impl TableContext for QueryContext {
             random_function_seed,
             week_start,
             date_format_style,
+            enable_binary_to_utf8_lossy,
         })
     }
 

src/query/settings/src/settings_default.rs

@@ -1424,6 +1424,13 @@ impl DefaultSettings {
                     scope: SettingScope::Both,
                     range: Some(SettingRange::Numeric(0..=1)),
                 }),
+                ("enable_binary_to_utf8_lossy", DefaultSettingValue {
+                    value: UserSettingValue::UInt64(0),
+                    desc: "Enable binary-to-UTF8 lossy conversion, default is 0, 1 for enable",
+                    mode: SettingMode::Both,
+                    scope: SettingScope::Both,
+                    range: Some(SettingRange::Numeric(0..=1)),
+                }),
             ]);
 
             Ok(Arc::new(DefaultSettings {

src/query/settings/src/settings_getter_setter.rs

@@ -1055,4 +1055,8 @@ impl Settings {
     pub fn get_enforce_local(&self) -> Result<bool> {
         Ok(self.try_get_u64("enforce_local")? == 1)
     }
+
+    pub fn get_enable_binary_to_utf8_lossy(&self) -> Result<bool> {
+        Ok(self.try_get_u64("enable_binary_to_utf8_lossy")? == 1)
+    }
 }

tests/sqllogictests/suites/query/functions/cast.test

@@ -28,6 +28,17 @@ select to_string('a')
 ----
 a
 
+statement error
+select to_string(UNHEX('C328'));
+
+statement ok
+set enable_binary_to_utf8_lossy = 1;
+
+query T
+select to_string(UNHEX('C328'));
+----
+�(
+
 query I
 select try_cast(3 as int);
 ----