refactor(query): refactor test style (#17630)

* refactor(query): refactor test style

* refactor(query): refactor test style

Author: sundy-li

src/query/config/src/config.rs

@@ -3304,3 +3304,28 @@ mod cache_config_converters {
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+    use std::ffi::OsString;
+
+    use clap::Parser;
+    use pretty_assertions::assert_eq;
+
+    use crate::Config;
+    use crate::InnerConfig;
+
+    /// It's required to make sure setting's default value is the same with clap.
+    #[test]
+    fn test_config_default() {
+        let setting_default = InnerConfig::default();
+        let config_default: InnerConfig = Config::parse_from(Vec::<OsString>::new())
+            .try_into()
+            .expect("parse from args must succeed");
+
+        assert_eq!(
+            setting_default, config_default,
+            "default setting is different from default config, please check again"
+        )
+    }
+}

src/query/config/tests/it/config.rs

@@ -262,7 +262,7 @@ impl<T: ValueType> ArrayColumn<T> {
     }
 
     pub fn memory_size(&self) -> usize {
-        T::column_memory_size(&self.values) + self.offsets.len() * 8
+        T::column_memory_size(&self.underlying_column()) + self.offsets.len() * 8
     }
 
     // Note: if the array column has been sliced, the number of values may not match the offsets.

src/query/config/tests/it/main.rs

@@ -1,10 +1,15 @@
+use databend_common_column::buffer::Buffer;
 use databend_common_expression::block_debug::box_render;
 use databend_common_expression::types::number::NumberScalar;
 use databend_common_expression::types::string::StringColumnBuilder;
 use databend_common_expression::types::AnyType;
+use databend_common_expression::types::ArrayColumn;
 use databend_common_expression::types::DataType;
 use databend_common_expression::types::Int32Type;
+use databend_common_expression::types::Int64Type;
 use databend_common_expression::types::NumberDataType;
+use databend_common_expression::types::StringType;
+use databend_common_expression::types::ValueType;
 use databend_common_expression::BlockEntry;
 use databend_common_expression::Column;
 use databend_common_expression::DataField;
@@ -75,4 +80,26 @@ fn test_block_entry_memory_size() {
     let scalar_str = Scalar::String("abc".to_string());
     let entry = BlockEntry::new(DataType::String, Value::<AnyType>::Scalar(scalar_str));
     assert_eq!(3, entry.memory_size());
+
+    let col = StringType::from_data((0..10).map(|x| x.to_string()).collect::<Vec<_>>());
+    assert_eq!(col.memory_size(), 10 + 10 * 12);
+
+    let array = ArrayColumn::<Int64Type>::new(
+        Buffer::from_iter(0..10i64),
+        Buffer::from(vec![0u64, 1, 3, 6, 10]),
+    );
+    let total_memory_size = array.memory_size(); // 10 * 8 + 5 * 8 = 80 + 40 = 120
+    let expected = array
+        .iter()
+        .map(|x| Int64Type::column_memory_size(&x))
+        .sum::<usize>()
+        + (array.len() + 1) * 8;
+    assert_eq!(total_memory_size, expected);
+
+    let array2 = array.slice(0..2);
+    let array3 = array.slice(2..4);
+    assert_eq!(
+        total_memory_size,
+        array2.memory_size() + array3.memory_size() - 8
+    );
 }

src/query/expression/src/types/array.rs

@@ -49,10 +49,17 @@ pub fn run_filter(file: &mut impl Write, predicate: Vec<bool>, block: &DataBlock
 }
 
 pub fn run_concat(file: &mut impl Write, blocks: &[DataBlock]) {
+    let total_block_size: usize = blocks.iter().map(|b| b.memory_size()).sum();
     let result = DataBlock::concat(blocks);
 
     match result {
         Ok(result_block) => {
+            let result_block_size = result_block.memory_size();
+            // bits could be compressed into one byte in nullable column
+            assert!(
+                total_block_size >= result_block_size
+                    && total_block_size as f64 <= result_block_size as f64 * 1.5f64
+            );
             for (i, c) in blocks.iter().enumerate() {
                 writeln!(file, "Concat-Column {i}:").unwrap();
                 writeln!(file, "{c:?}").unwrap();

src/query/expression/tests/it/block.rs

@@ -709,3 +709,134 @@ pub fn skip_to_next_row<R: AsRef<[u8]>>(reader: &mut Cursor<R>, mut balance: i32
     }
     Ok(())
 }
+
+#[cfg(test)]
+mod test {
+
+    use std::io::Write;
+
+    use databend_common_exception::ErrorCode;
+    use databend_common_exception::Result;
+    use databend_common_expression::types::DataType;
+    use databend_common_expression::types::NumberDataType;
+    use databend_common_expression::ColumnBuilder;
+    use databend_common_expression::DataBlock;
+    use databend_common_expression::Scalar;
+    use databend_common_io::prelude::FormatSettings;
+    use goldenfile::Mint;
+
+    use super::FastFieldDecoderValues;
+    use super::FastValuesDecodeFallback;
+    use super::FastValuesDecoder;
+
+    struct DummyFastValuesDecodeFallback {}
+
+    #[async_trait::async_trait]
+    impl FastValuesDecodeFallback for DummyFastValuesDecodeFallback {
+        async fn parse_fallback(&self, _data: &str) -> Result<Vec<Scalar>> {
+            Err(ErrorCode::Unimplemented("fallback".to_string()))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_fast_values_decoder_multi() -> Result<()> {
+        let mut mint = Mint::new("testdata");
+        let file = &mut mint.new_goldenfile("fast_values.txt").unwrap();
+
+        struct Test {
+            data: &'static str,
+            column_types: Vec<DataType>,
+        }
+
+        let cases = vec![
+            Test {
+                data: "(0, 1, 2), (3,4,5)",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                ],
+            },
+            Test {
+                data: "(0, 1, 2), (3,4,5), ",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                ],
+            },
+            Test {
+                data: "('', '', '')",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                ],
+            },
+            Test {
+                data: "( 1, '', '2022-10-01')",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::String,
+                    DataType::Date,
+                ],
+            },
+            Test {
+                data: "(1, 2, 3), (1, 1, 1), (1, 1, 1);",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                ],
+            },
+            Test {
+                data: "(1, 2, 3), (1, 1, 1), (1, 1, 1);  ",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                ],
+            },
+            Test {
+                data: "(1.2, -2.9, 3.55), (3.12e2, 3.45e+3, -1.9e-3);",
+                column_types: vec![
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                    DataType::Number(NumberDataType::Int16),
+                ],
+            },
+        ];
+
+        for case in cases {
+            writeln!(file, "---------- Input Data ----------")?;
+            writeln!(file, "{:?}", case.data)?;
+
+            writeln!(file, "---------- Input Column Types ----------")?;
+            writeln!(file, "{:?}", case.column_types)?;
+
+            let field_decoder =
+                FastFieldDecoderValues::create_for_insert(FormatSettings::default(), true);
+            let mut values_decoder = FastValuesDecoder::new(case.data, &field_decoder);
+            let fallback = DummyFastValuesDecodeFallback {};
+            let mut columns = case
+                .column_types
+                .into_iter()
+                .map(|dt| ColumnBuilder::with_capacity(&dt, values_decoder.estimated_rows()))
+                .collect::<Vec<_>>();
+            let result = values_decoder.parse(&mut columns, &fallback).await;
+
+            writeln!(file, "---------- Output ---------")?;
+
+            if let Err(err) = result {
+                writeln!(file, "{}", err)?;
+            } else {
+                let columns = columns.into_iter().map(|cb| cb.build()).collect::<Vec<_>>();
+                let got = DataBlock::new_from_columns(columns);
+                writeln!(file, "{}", got)?;
+            }
+            writeln!(file, "\n")?;
+        }
+
+        Ok(())
+    }
+}

src/query/expression/tests/it/common.rs

@@ -91,3 +91,106 @@ pub fn write_tsv_escaped_string(bytes: &[u8], buf: &mut Vec<u8>, field_delimiter
         buf.extend_from_slice(&bytes[start..]);
     }
 }
+
+#[cfg(test)]
+mod test {
+    use crate::field_encoder::helpers::write_json_string;
+    use crate::field_encoder::helpers::write_tsv_escaped_string;
+    use crate::field_encoder::write_csv_string;
+
+    #[test]
+    fn test_escape() {
+        {
+            let mut buf = vec![];
+            write_tsv_escaped_string(b"\0\n\r\t\\\'", &mut buf, b'\t');
+            assert_eq!(&buf, b"\\0\\n\\r\\t\\\\\'")
+        }
+
+        {
+            let mut buf = vec![];
+            write_tsv_escaped_string(b"\n123\n456\n", &mut buf, b'\t');
+            assert_eq!(&buf, b"\\n123\\n456\\n")
+        }
+
+        {
+            let mut buf = vec![];
+            write_tsv_escaped_string(b"123\n", &mut buf, b'\t');
+            assert_eq!(&buf, b"123\\n")
+        }
+
+        {
+            let mut buf = vec![];
+            write_tsv_escaped_string(b"\n123", &mut buf, b'\t');
+            assert_eq!(&buf, b"\\n123")
+        }
+
+        {
+            let mut buf = vec![];
+            write_tsv_escaped_string(b"\n,23", &mut buf, b',');
+            assert_eq!(&buf, b"\\n\\,23")
+        }
+    }
+
+    #[test]
+    fn test_json_escape() {
+        let basic = b"\0\n\r\t\'/\"\\";
+        {
+            let mut buf = vec![];
+            write_json_string(basic, &mut buf, false, true);
+            assert_eq!(&buf, b"\\u0000\\n\\r\\t\'\\/\\\"\\\\")
+        }
+
+        {
+            let mut buf = vec![];
+            write_json_string(basic, &mut buf, false, false);
+            assert_eq!(&buf, b"\\u0000\\n\\r\\t\'/\\\"\\\\")
+        }
+
+        {
+            let mut buf = vec![];
+            write_json_string(basic, &mut buf, true, true);
+            assert_eq!(&buf, b"\\u0000\\n\\r\\t\'\\/\"\"\\\\")
+        }
+
+        {
+            let mut buf = vec![];
+            write_json_string(b"\n123\n456\n", &mut buf, true, true);
+            assert_eq!(&buf, b"\\n123\\n456\\n")
+        }
+
+        {
+            let mut buf = vec![];
+            write_json_string(b"123\n", &mut buf, true, true);
+            assert_eq!(&buf, b"123\\n")
+        }
+
+        {
+            let mut buf = vec![];
+            write_json_string(b"\n123", &mut buf, true, true);
+            assert_eq!(&buf, b"\\n123")
+        }
+
+        {
+            let mut buf = vec![];
+            write_json_string(b"\n123", &mut buf, true, true);
+            assert_eq!(&buf, b"\\n123")
+        }
+
+        {
+            let s = "123\u{2028}\u{2029}abc";
+            let mut buf = vec![];
+            write_json_string(s.as_bytes(), &mut buf, true, true);
+            assert_eq!(&buf, b"123\\u2028\\u2029abc")
+        }
+    }
+
+    #[test]
+    fn test_csv_string() {
+        {
+            let s = "a\"\nb";
+            let mut buf = vec![];
+            write_csv_string(s.as_bytes(), &mut buf, b'"');
+            assert_eq!(&buf, b"\"a\"\"\nb\"")
+        }
+    }
+}