feat: implement literal expressions with binary serialization support

Author: HeartLinked

src/iceberg/CMakeLists.txt

@@ -49,12 +49,11 @@ set(ICEBERG_SOURCES
     manifest_reader_internal.cc
     manifest_writer.cc
     arrow_c_data_guard_internal.cc
+    util/conversions.cc
     util/decimal.cc
-    util/murmurhash3_internal.cc
-    util/timepoint.cc
     util/gzip_internal.cc
-    util/conversions.cc
-    util/literal_format.cc)
+    util/murmurhash3_internal.cc
+    util/timepoint.cc)
 
 set(ICEBERG_STATIC_BUILD_INTERFACE_LIBS)
 set(ICEBERG_SHARED_BUILD_INTERFACE_LIBS)

src/iceberg/expression/literal.cc

@@ -24,7 +24,6 @@
 
 #include "iceberg/exception.h"
 #include "iceberg/util/conversions.h"
-#include "iceberg/util/literal_format.h"
 
 namespace iceberg {
 

src/iceberg/expression/literal.h

@@ -143,8 +143,8 @@ class ICEBERG_EXPORT Literal {
  private:
   Literal(Value value, std::shared_ptr<PrimitiveType> type);
 
-  friend class LiteralCaster;
   friend class Conversions;
+  friend class LiteralCaster;
 
   Value value_;
   std::shared_ptr<PrimitiveType> type_;

src/iceberg/util/conversions.cc

@@ -19,23 +19,24 @@
 
 #include "iceberg/util/conversions.h"
 
-#include <cctype>
+#include <array>
 #include <cstring>
-#include <ranges>
+#include <span>
+#include <string>
 
-#include "iceberg/exception.h"
-#include "iceberg/type.h"
 #include "iceberg/util/endian.h"
 #include "iceberg/util/macros.h"
 
 namespace iceberg {
 
-/// \brief Write a value in little-endian format to the buffer.
+/// \brief Write a value in little-endian format and return as vector.
 template <EndianConvertible T>
-void WriteLittleEndian(std::vector<uint8_t>& buffer, T value) {
+std::vector<uint8_t> WriteLittleEndian(T value) {
   value = ToLittleEndian(value);
   const auto* bytes = reinterpret_cast<const uint8_t*>(&value);
-  buffer.insert(buffer.end(), bytes, bytes + sizeof(T));
+  std::vector<uint8_t> result;
+  result.insert(result.end(), bytes, bytes + sizeof(T));
+  return result;
 }
 
 /// \brief Read a value in little-endian format from the data.
@@ -58,78 +59,63 @@ Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
 
   switch (type_id) {
     case TypeId::kBoolean: {
-      // 0x00 for false, 0x01 for true
       result.push_back(std::get<bool>(value) ? 0x01 : 0x00);
       return result;
     }
 
     case TypeId::kInt: {
-      // Stored as 4-byte little-endian
-      WriteLittleEndian(result, std::get<int32_t>(value));
+      result = WriteLittleEndian(std::get<int32_t>(value));
       return result;
     }
 
     case TypeId::kDate: {
-      // Stores days from 1970-01-01 in a 4-byte little-endian int
-      WriteLittleEndian(result, std::get<int32_t>(value));
+      result = WriteLittleEndian(std::get<int32_t>(value));
       return result;
     }
 
     case TypeId::kLong: {
-      // Stored as 8-byte little-endian
-      WriteLittleEndian(result, std::get<int64_t>(value));
+      result = WriteLittleEndian(std::get<int64_t>(value));
       return result;
     }
 
     case TypeId::kTime: {
-      // Stores microseconds from midnight in an 8-byte little-endian long
-      WriteLittleEndian(result, std::get<int64_t>(value));
+      result = WriteLittleEndian(std::get<int64_t>(value));
       return result;
     }
 
     case TypeId::kTimestamp: {
-      // Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian
-      // long
-      WriteLittleEndian(result, std::get<int64_t>(value));
+      result = WriteLittleEndian(std::get<int64_t>(value));
       return result;
     }
 
     case TypeId::kTimestampTz: {
-      // Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte
-      // little-endian long
-      WriteLittleEndian(result, std::get<int64_t>(value));
+      result = WriteLittleEndian(std::get<int64_t>(value));
       return result;
     }
 
     case TypeId::kFloat: {
-      // Stored as 4-byte little-endian
-      WriteLittleEndian(result, std::get<float>(value));
+      result = WriteLittleEndian(std::get<float>(value));
       return result;
     }
 
     case TypeId::kDouble: {
-      // Stored as 8-byte little-endian
-      WriteLittleEndian(result, std::get<double>(value));
+      result = WriteLittleEndian(std::get<double>(value));
       return result;
     }
 
     case TypeId::kString: {
-      // UTF-8 bytes (without length)
       const auto& str = std::get<std::string>(value);
       result.insert(result.end(), str.begin(), str.end());
       return result;
     }
 
     case TypeId::kBinary: {
-      // Binary value (without length)
       const auto& binary_data = std::get<std::vector<uint8_t>>(value);
       result.insert(result.end(), binary_data.begin(), binary_data.end());
       return result;
     }
 
     case TypeId::kFixed: {
-      // Fixed(L) - Binary value, could be stored in std::array<uint8_t, 16> or
-      // std::vector<uint8_t>
       if (std::holds_alternative<std::array<uint8_t, 16>>(value)) {
         const auto& fixed_bytes = std::get<std::array<uint8_t, 16>>(value);
         result.insert(result.end(), fixed_bytes.begin(), fixed_bytes.end());
@@ -144,13 +130,7 @@ Result<std::vector<uint8_t>> Conversions::ToBytes(const PrimitiveType& type,
       }
       return result;
     }
-
-    case TypeId::kUuid: {
-      // 16-byte big-endian value
-      const auto& uuid_bytes = std::get<std::array<uint8_t, 16>>(value);
-      WriteBigEndian16(result, uuid_bytes);
-      return result;
-    }
+      // TODO(Li Feiyang): Add support for UUID and Decimal
 
     default:
       return NotSupported("Serialization for type {} is not supported", type.ToString());
@@ -174,9 +154,8 @@ Result<std::vector<uint8_t>> Conversions::ToBytes(const Literal& literal) {
 
 Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
                                               std::span<const uint8_t> data) {
-  // Empty data represents null value
   if (data.empty()) {
-    return Literal::Value{std::monostate{}};
+    return InvalidArgument("Data cannot be empty");
   }
 
   const auto type_id = type.type_id();
@@ -187,7 +166,6 @@ Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
         return InvalidArgument("Boolean requires 1 byte, got {}", data.size());
       }
       ICEBERG_ASSIGN_OR_RAISE(auto value, ReadLittleEndian<uint8_t>(data));
-      // 0x00 for false, non-zero byte for true
       return Literal::Value{static_cast<bool>(value != 0x00)};
     }
 
@@ -215,20 +193,14 @@ Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
     case TypeId::kTimestampTz: {
       int64_t value;
       if (data.size() == 8) {
-        // Standard 8-byte long
         ICEBERG_ASSIGN_OR_RAISE(auto long_value, ReadLittleEndian<int64_t>(data));
         value = long_value;
       } else if (data.size() == 4) {
         // Type was promoted from int to long
         ICEBERG_ASSIGN_OR_RAISE(auto int_value, ReadLittleEndian<int32_t>(data));
         value = static_cast<int64_t>(int_value);
       } else {
-        auto type_name_view = ToString(type_id);
-        std::string type_name{type_name_view};
-        if (!type_name.empty()) {
-          type_name[0] = static_cast<char>(std::toupper(type_name[0]));
-        }
-        return InvalidArgument("{} requires 4 or 8 bytes, got {}", type_name,
+        return InvalidArgument("{} requires 4 or 8 bytes, got {}", ToString(type_id),
                                data.size());
       }
 
@@ -246,7 +218,6 @@ Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
 
     case TypeId::kDouble: {
       if (data.size() == 8) {
-        // Standard 8-byte double
         ICEBERG_ASSIGN_OR_RAISE(auto double_value, ReadLittleEndian<double>(data));
         return Literal::Value{double_value};
       } else if (data.size() == 4) {
@@ -276,27 +247,7 @@ Result<Literal::Value> Conversions::FromBytes(const PrimitiveType& type,
         return Literal::Value{std::vector<uint8_t>(data.begin(), data.end())};
       }
     }
-
-    case TypeId::kUuid: {
-      if (data.size() != 16) {
-        return InvalidArgument("UUID requires 16 bytes, got {}", data.size());
-      }
-      ICEBERG_ASSIGN_OR_RAISE(auto uuid_value, ReadBigEndian16(data));
-      return Literal::Value{uuid_value};
-    }
-
-    case TypeId::kDecimal: {
-      if (data.size() > 16) {
-        return InvalidArgument(
-            "Decimal data too large, maximum 16 bytes supported, got {}", data.size());
-      }
-
-      std::array<uint8_t, 16> decimal_bytes{};
-      // Copy data to the end of the array (big-endian format for decimals)
-      // This handles variable-length decimals by right-aligning them
-      std::ranges::copy(data, decimal_bytes.end() - data.size());
-      return Literal::Value{decimal_bytes};
-    }
+      // TODO(Li Feiyang): Add support for UUID and Decimal
 
     default:
       return NotSupported("Deserialization for type {} is not supported",
@@ -314,10 +265,10 @@ Result<Literal> Conversions::FromBytes(std::shared_ptr<PrimitiveType> type,
 
   // If we got a null value (monostate), create a null Literal
   if (std::holds_alternative<std::monostate>(value)) {
-    return Literal::Null(type);
+    return Literal::Null(std::move(type));
   }
 
-  return Literal(value, type);
+  return Literal(std::move(value), std::move(type));
 }
 
 }  // namespace iceberg

src/iceberg/util/conversions.h

@@ -19,25 +19,33 @@
 
 #pragma once
 
+#include <span>
 #include <vector>
 
 #include "iceberg/expression/literal.h"
-#include "iceberg/iceberg_export.h"
 #include "iceberg/result.h"
-#include "iceberg/type.h"
+#include "iceberg/type_fwd.h"
+
+/// \file iceberg/util/conversions.h
+/// \brief Conversion utilities for primitive types
 
 namespace iceberg {
+
+/// \brief Conversion utilities for primitive types
 class ICEBERG_EXPORT Conversions {
  public:
+  /// \brief Convert a literal value to bytes
   static Result<std::vector<uint8_t>> ToBytes(const PrimitiveType& type,
                                               const Literal::Value& value);
 
   static Result<std::vector<uint8_t>> ToBytes(const Literal& literal);
 
+  /// \brief Convert bytes to a literal value
   static Result<Literal::Value> FromBytes(const PrimitiveType& type,
                                           std::span<const uint8_t> data);
 
   static Result<Literal> FromBytes(std::shared_ptr<PrimitiveType> type,
                                    std::span<const uint8_t> data);
 };
+
 }  // namespace iceberg

src/iceberg/util/endian.h

@@ -19,16 +19,11 @@
 
 #pragma once
 
-// #include <bit>
-// #include <concepts>
-// #include <cstdint>
 #include <algorithm>
+#include <array>
 #include <bit>
-#include <cstring>
-#include <span>
-#include <vector>
-
-#include "iceberg/result.h"
+#include <concepts>
+#include <cstdint>
 
 /// \file iceberg/util/endian.h
 /// \brief Endianness conversion utilities