Merge branch 'master' into support-date32

Author: Enmk

BUCK

@@ -653,7 +653,12 @@ void Client::Impl::WriteBlock(const Block& block, OutputStream& output) {
         WireFormat::WriteString(output, bi.Name());
         WireFormat::WriteString(output, bi.Type()->GetName());
 
-        bi.Column()->Save(&output);
+        // Empty columns are not serialized and occupy exactly 0 bytes.
+        // ref https://github.com/ClickHouse/ClickHouse/blob/39b37a3240f74f4871c8c1679910e065af6bea19/src/Formats/NativeWriter.cpp#L163
+        const bool containsData = block.GetRowCount() > 0;
+        if (containsData) {
+            bi.Column()->Save(&output);
+        }
     }
     output.Flush();
 }

clickhouse/client.cpp

@@ -86,6 +86,7 @@ struct ClientOptions {
     // TCP options
     DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true);
 
+    // TODO deprecate setting
     /** It helps to ease migration of the old codebases, which can't afford to switch
     * to using ColumnLowCardinalityT or ColumnLowCardinality directly,
     * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint.

clickhouse/client.h

@@ -1,5 +1,6 @@
 #include "array.h"
 #include "numeric.h"
+
 #include <stdexcept>
 
 namespace clickhouse {
@@ -45,10 +46,9 @@ ColumnRef ColumnArray::Slice(size_t begin, size_t size) const {
     if (size && begin + size > Size())
         throw ValidationError("Slice indexes are out of bounds");
 
-    auto result = std::make_shared<ColumnArray>(data_->CloneEmpty());
-    for (size_t i = 0; i < size; i++) {
-        result->AppendAsColumn(GetAsColumn(begin + i));
-    }
+    auto result = std::make_shared<ColumnArray>(data_->Slice(GetOffset(begin), GetOffset(begin + size) - GetOffset(begin)));
+    for (size_t i = 0; i < size; i++)
+        result->AddOffset(GetSize(begin + i));
 
     return result;
 }

clickhouse/columns/array.cpp

@@ -58,7 +58,7 @@ void ColumnDate::Swap(Column& other) {
 }
 
 ItemView ColumnDate::GetItem(size_t index) const {
-    return data_->GetItem(index);
+    return ItemView(Type::Date, data_->GetItem(index));
 }
 
 
@@ -179,7 +179,7 @@ ColumnRef ColumnDateTime::Slice(size_t begin, size_t len) const {
 }
 
 ColumnRef ColumnDateTime::CloneEmpty() const {
-    return std::make_shared<ColumnDate>();
+    return std::make_shared<ColumnDateTime>();
 }
 
 void ColumnDateTime::Swap(Column& other) {
@@ -188,7 +188,7 @@ void ColumnDateTime::Swap(Column& other) {
 }
 
 ItemView ColumnDateTime::GetItem(size_t index) const {
-    return data_->GetItem(index);
+    return ItemView(Type::DateTime, data_->GetItem(index));
 }
 
 ColumnDateTime64::ColumnDateTime64(size_t precision)
@@ -246,7 +246,7 @@ size_t ColumnDateTime64::Size() const {
 }
 
 ItemView ColumnDateTime64::GetItem(size_t index) const {
-    return data_->GetItem(index);
+    return ItemView(Type::DateTime64, data_->GetItem(index));
 }
 
 void ColumnDateTime64::Swap(Column& other) {

clickhouse/columns/date.cpp

@@ -229,7 +229,7 @@ void ColumnDecimal::Swap(Column& other) {
 }
 
 ItemView ColumnDecimal::GetItem(size_t index) const {
-    return data_->GetItem(index);
+    return ItemView{GetType().GetCode(), data_->GetItem(index)};
 }
 
 size_t ColumnDecimal::GetScale() const

clickhouse/columns/decimal.cpp

@@ -165,6 +165,8 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
                         return std::make_shared<LowCardinalitySerializationAdaptor<ColumnString>>();
                     case Type::FixedString:
                         return std::make_shared<LowCardinalitySerializationAdaptor<ColumnFixedString>>(nested.elements.front().value);
+                    case Type::Nullable:
+                        throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported with LowCardinalityAsWrappedColumn on");
                     default:
                         throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported");
                 }
@@ -176,6 +178,13 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
                         return std::make_shared<ColumnLowCardinalityT<ColumnString>>();
                     case Type::FixedString:
                         return std::make_shared<ColumnLowCardinalityT<ColumnFixedString>>(nested.elements.front().value);
+                    case Type::Nullable:
+                        return std::make_shared<ColumnLowCardinality>(
+                            std::make_shared<ColumnNullable>(
+                                CreateColumnFromAst(nested.elements.front(), settings),
+                                std::make_shared<ColumnUInt8>()
+                            )
+                        );
                     default:
                         throw UnimplementedError("LowCardinality(" + nested.name + ") is not supported");
                 }

clickhouse/columns/factory.cpp

@@ -97,7 +97,7 @@ void ColumnIPv4::Swap(Column& other) {
 }
 
 ItemView ColumnIPv4::GetItem(size_t index) const {
-    return data_->GetItem(index);
+    return ItemView(Type::IPv4, data_->GetItem(index));
 }
 
 }

clickhouse/columns/ip4.cpp

@@ -97,7 +97,7 @@ void ColumnIPv6::Swap(Column& other) {
 }
 
 ItemView ColumnIPv6::GetItem(size_t index) const {
-    return data_->GetItem(index);
+    return ItemView{Type::IPv6, data_->GetItem(index)};
 }
 
 }

clickhouse/columns/ip6.cpp

@@ -1,44 +1,71 @@
 #include "../columns/itemview.h"
 
+#include <algorithm>
+#include <sstream>
+
+namespace {
+
+template <typename Container>
+std::string ContainerToString(Container container, const char * separator = ", ") {
+    std::stringstream sstr;
+    const auto end = std::end(container);
+    for (auto i = std::begin(container); i != end; /*intentionally no ++i*/) {
+        const auto & elem = *i;
+        sstr << elem;
+
+        if (++i != end) {
+            sstr << separator;
+        }
+    }
+
+    return sstr.str();
+}
+
+}
+
 namespace clickhouse {
 
 void ItemView::ValidateData(Type::Code type, DataType data) {
-    int expected_size = 0;
+
+    auto AssertSize = [type, &data](std::initializer_list<int> allowed_sizes) -> void {
+        const auto end = std::end(allowed_sizes);
+        if (std::find(std::begin(allowed_sizes), end, static_cast<int>(data.size())) == end) {
+            throw AssertionError(std::string("ItemView value size mismatch for ")
+                    + Type::TypeName(type)
+                    + " expected: " + ContainerToString(allowed_sizes, " or ")
+                    + ", got: " + std::to_string(data.size()));
+        }
+    };
+
     switch (type) {
         case Type::Code::Void:
-            expected_size = 0;
-            break;
+            return AssertSize({0});
 
         case Type::Code::Int8:
         case Type::Code::UInt8:
         case Type::Code::Enum8:
-            expected_size = 1;
-            break;
+            return AssertSize({1});
 
         case Type::Code::Int16:
         case Type::Code::UInt16:
         case Type::Code::Date:
         case Type::Code::Enum16:
-            expected_size = 2;
-            break;
+            return AssertSize({2});
 
         case Type::Code::Int32:
         case Type::Code::UInt32:
         case Type::Code::Float32:
         case Type::Code::DateTime:
         case Type::Code::IPv4:
         case Type::Code::Decimal32:
-            expected_size = 4;
-            break;
+            return AssertSize({4});
 
         case Type::Code::Int64:
         case Type::Code::UInt64:
         case Type::Code::Float64:
         case Type::Code::DateTime64:
-        case Type::Code::IPv6:
         case Type::Code::Decimal64:
-            expected_size = 8;
-            break;
+            return AssertSize({8});
 
         case Type::Code::String:
         case Type::Code::FixedString:
@@ -49,24 +76,21 @@ void ItemView::ValidateData(Type::Code type, DataType data) {
         case Type::Code::Nullable:
         case Type::Code::Tuple:
         case Type::Code::LowCardinality:
-            throw UnimplementedError("Unsupported type in ItemView: " + std::to_string(static_cast<int>(type)));
+            throw AssertionError("Unsupported type in ItemView: " + std::string(Type::TypeName(type)));
 
+        case Type::Code::IPv6:
         case Type::Code::UUID:
         case Type::Code::Int128:
-        case Type::Code::Decimal:
         case Type::Code::Decimal128:
-            expected_size = 16;
-            break;
+            return AssertSize({16});
+
+        case Type::Code::Decimal:
+            // Could be either Decimal32, Decimal64 or Decimal128
+            return AssertSize({4, 8, 16});
 
         default:
             throw UnimplementedError("Unknon type code:" + std::to_string(static_cast<int>(type)));
     }
-
-    if (expected_size != static_cast<int>(data.size())) {
-        throw AssertionError("Value size mismatch for type "
-                + std::to_string(static_cast<int>(type)) + " expected: "
-                + std::to_string(expected_size) + ", got: " + std::to_string(data.size()));
-    }
 }
 
 }