use utfview8

Author: xudong963

datafusion/core/tests/parquet/page_pruning.rs

@@ -160,7 +160,7 @@ async fn page_index_filter_one_col() {
 
     // 5.create filter date_string_col == "01/01/09"`;
     // Note this test doesn't apply type coercion so the literal must match the actual view type
-    let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8("01/01/09")));
+    let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8view("01/01/09")));
     let parquet_exec = get_parquet_exec(&state, filter).await;
     let mut results = parquet_exec.execute(0, task_ctx.clone()).unwrap();
     let batch = results.next().await.unwrap().unwrap();

datafusion/sqllogictest/test_files/describe.slt

@@ -81,8 +81,8 @@ int_col Int32 YES
 bigint_col Int64 YES
 float_col Float32 YES
 double_col Float64 YES
-date_string_col Utf8 YES
-string_col Utf8 YES
+date_string_col Utf8View YES
+string_col Utf8View YES
 timestamp_col Timestamp(Nanosecond, None) YES
 year Int32 YES
 month Int32 YES

datafusion/sqllogictest/test_files/information_schema.slt

@@ -249,7 +249,7 @@ datafusion.execution.parquet.metadata_size_hint NULL
 datafusion.execution.parquet.pruning true
 datafusion.execution.parquet.pushdown_filters false
 datafusion.execution.parquet.reorder_filters false
-datafusion.execution.parquet.schema_force_view_types false
+datafusion.execution.parquet.schema_force_view_types true
 datafusion.execution.parquet.skip_arrow_metadata false
 datafusion.execution.parquet.skip_metadata true
 datafusion.execution.parquet.statistics_enabled page
@@ -359,7 +359,7 @@ datafusion.execution.parquet.metadata_size_hint NULL (reading) If specified, the
 datafusion.execution.parquet.pruning true (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file
 datafusion.execution.parquet.pushdown_filters false (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization".
 datafusion.execution.parquet.reorder_filters false (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query
-datafusion.execution.parquet.schema_force_view_types false (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`.
+datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`.
 datafusion.execution.parquet.skip_arrow_metadata false (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to <https://docs.rs/parquet/53.3.0/parquet/arrow/arrow_writer/struct.ArrowWriterOptions.html#method.with_skip_arrow_metadata>
 datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata
 datafusion.execution.parquet.statistics_enabled page (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting

datafusion/sqllogictest/test_files/map.slt

@@ -45,7 +45,7 @@ describe data;
 ----
 ints Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO
 strings Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO
-timestamp Utf8 NO
+timestamp Utf8View NO
 
 query ??T
 SELECT * FROM data ORDER by ints['bytes'] DESC LIMIT 10;

datafusion/sqllogictest/test_files/parquet.slt

@@ -384,15 +384,15 @@ select
   arrow_typeof(binaryview_col),  binaryview_col
  FROM binary_as_string_default;
 ----
-Binary 616161 Binary 616161 Binary 616161
-Binary 626262 Binary 626262 Binary 626262
-Binary 636363 Binary 636363 Binary 636363
-Binary 646464 Binary 646464 Binary 646464
-Binary 656565 Binary 656565 Binary 656565
-Binary 666666 Binary 666666 Binary 666666
-Binary 676767 Binary 676767 Binary 676767
-Binary 686868 Binary 686868 Binary 686868
-Binary 696969 Binary 696969 Binary 696969
+BinaryView 616161 BinaryView 616161 BinaryView 616161
+BinaryView 626262 BinaryView 626262 BinaryView 626262
+BinaryView 636363 BinaryView 636363 BinaryView 636363
+BinaryView 646464 BinaryView 646464 BinaryView 646464
+BinaryView 656565 BinaryView 656565 BinaryView 656565
+BinaryView 666666 BinaryView 666666 BinaryView 666666
+BinaryView 676767 BinaryView 676767 BinaryView 676767
+BinaryView 686868 BinaryView 686868 BinaryView 686868
+BinaryView 696969 BinaryView 696969 BinaryView 696969
 
 # Run an explain plan to show the cast happens in the plan (a CAST is needed for the predicates)
 query TT
@@ -405,11 +405,11 @@ EXPLAIN
    binaryview_col LIKE '%a%';
 ----
 logical_plan
-01)Filter: CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8) LIKE Utf8("%a%")
-02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8) LIKE Utf8("%a%")]
+01)Filter: CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%")
+02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%")]
 physical_plan
 01)CoalesceBatchesExec: target_batch_size=8192
-02)--FilterExec: CAST(binary_col@0 AS Utf8) LIKE %a% AND CAST(largebinary_col@1 AS Utf8) LIKE %a% AND CAST(binaryview_col@2 AS Utf8) LIKE %a%
+02)--FilterExec: CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a%
 03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/binary_as_string.parquet]]}, projection=[binary_col, largebinary_col, binaryview_col], file_type=parquet, predicate=CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a%
 
@@ -432,15 +432,15 @@ select
   arrow_typeof(binaryview_col),  binaryview_col
  FROM binary_as_string_option;
 ----
-Utf8 aaa Utf8 aaa Utf8 aaa
-Utf8 bbb Utf8 bbb Utf8 bbb
-Utf8 ccc Utf8 ccc Utf8 ccc
-Utf8 ddd Utf8 ddd Utf8 ddd
-Utf8 eee Utf8 eee Utf8 eee
-Utf8 fff Utf8 fff Utf8 fff
-Utf8 ggg Utf8 ggg Utf8 ggg
-Utf8 hhh Utf8 hhh Utf8 hhh
-Utf8 iii Utf8 iii Utf8 iii
+Utf8View aaa Utf8View aaa Utf8View aaa
+Utf8View bbb Utf8View bbb Utf8View bbb
+Utf8View ccc Utf8View ccc Utf8View ccc
+Utf8View ddd Utf8View ddd Utf8View ddd
+Utf8View eee Utf8View eee Utf8View eee
+Utf8View fff Utf8View fff Utf8View fff
+Utf8View ggg Utf8View ggg Utf8View ggg
+Utf8View hhh Utf8View hhh Utf8View hhh
+Utf8View iii Utf8View iii Utf8View iii
 
 # Run an explain plan to show the cast happens in the plan (there should be no casts)
 query TT
@@ -453,8 +453,8 @@ EXPLAIN
    binaryview_col LIKE '%a%';
 ----
 logical_plan
-01)Filter: binary_as_string_option.binary_col LIKE Utf8("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8("%a%")
-02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8("%a%"), binary_as_string_option.largebinary_col LIKE Utf8("%a%"), binary_as_string_option.binaryview_col LIKE Utf8("%a%")]
+01)Filter: binary_as_string_option.binary_col LIKE Utf8View("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8View("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8View("%a%")
+02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8View("%a%"), binary_as_string_option.largebinary_col LIKE Utf8View("%a%"), binary_as_string_option.binaryview_col LIKE Utf8View("%a%")]
 physical_plan
 01)CoalesceBatchesExec: target_batch_size=8192
 02)--FilterExec: binary_col@0 LIKE %a% AND largebinary_col@1 LIKE %a% AND binaryview_col@2 LIKE %a%

datafusion/sqllogictest/test_files/simplify_predicates.slt

@@ -84,7 +84,7 @@ query TT
 EXPLAIN SELECT * FROM test_data WHERE str_col > 'apple' AND str_col > 'banana';
 ----
 logical_plan
-01)Filter: test_data.str_col > Utf8("banana")
+01)Filter: test_data.str_col > Utf8View("banana")
 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col]
 
 # date_col > '2023-01-01' AND date_col > '2023-02-01' should simplify to date_col > '2023-02-01'
@@ -120,7 +120,7 @@ WHERE int_col > 5
   AND float_col BETWEEN 1 AND 100;
 ----
 logical_plan
-01)Filter: test_data.str_col LIKE Utf8("A%") AND test_data.float_col >= Float32(1) AND test_data.float_col <= Float32(100) AND test_data.int_col > Int32(10)
+01)Filter: test_data.str_col LIKE Utf8View("A%") AND test_data.float_col >= Float32(1) AND test_data.float_col <= Float32(100) AND test_data.int_col > Int32(10)
 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col]
 
 statement ok