feat: Literal support decimal & respect Appendix B: 32-bit Hash Requirements

Author: zhjwpku

src/iceberg/CMakeLists.txt

@@ -55,11 +55,14 @@ set(ICEBERG_SOURCES
     manifest_reader_internal.cc
     manifest_writer.cc
     arrow_c_data_guard_internal.cc
+    util/bucket_util.cc
     util/conversions.cc
     util/decimal.cc
     util/gzip_internal.cc
     util/murmurhash3_internal.cc
+    util/temporal_util.cc
     util/timepoint.cc
+    util/truncate_util.cc
     util/uuid.cc)
 
 set(ICEBERG_STATIC_BUILD_INTERFACE_LIBS)

src/iceberg/expression/literal.cc

@@ -24,9 +24,11 @@
 #include <cstdint>
 #include <string>
 
-#include "iceberg/type_fwd.h"
+#include "iceberg/exception.h"
 #include "iceberg/util/checked_cast.h"
 #include "iceberg/util/conversions.h"
+#include "iceberg/util/decimal.h"
+#include "iceberg/util/macros.h"
 
 namespace iceberg {
 
@@ -296,6 +298,10 @@ Literal Literal::Fixed(std::vector<uint8_t> value) {
   return {Value{std::move(value)}, fixed(size)};
 }
 
+Literal Literal::Decimal(int128_t value, int32_t precision, int32_t scale) {
+  return {Value{::iceberg::Decimal(value)}, decimal(precision, scale)};
+}
+
 Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
                                      std::shared_ptr<PrimitiveType> type) {
   return Conversions::FromBytes(std::move(type), data);
@@ -331,12 +337,42 @@ std::strong_ordering CompareFloat(T lhs, T rhs) {
   return lhs_is_negative <=> rhs_is_negative;
 }
 
+std::strong_ordering CompareDecimal(Literal const& lhs, Literal const& rhs) {
+  ICEBERG_DCHECK(std::holds_alternative<Decimal>(lhs.value()),
+                 "LHS of decimal comparison must hold Decimal");
+  ICEBERG_DCHECK(std::holds_alternative<Decimal>(rhs.value()),
+                 "RHS of decimal comparison must hold decimal");
+  const auto& lhs_type = std::dynamic_pointer_cast<DecimalType>(lhs.type());
+  const auto& rhs_type = std::dynamic_pointer_cast<DecimalType>(rhs.type());
+  auto lhs_decimal = std::get<Decimal>(lhs.value());
+  auto rhs_decimal = std::get<Decimal>(rhs.value());
+  if (lhs_type->scale() == rhs_type->scale()) {
+    return lhs_decimal <=> rhs_decimal;
+  } else if (lhs_type->scale() > rhs_type->scale()) {
+    // Rescale to larger scale
+    auto rhs_res = rhs_decimal.Rescale(rhs_type->scale(), lhs_type->scale());
+    if (!rhs_res) {
+      // Rescale would cause data loss, so lhs is definitely less than rhs
+      return std::strong_ordering::less;
+    }
+    return lhs_decimal <=> rhs_res.value();
+  } else {
+    // Rescale to larger scale
+    auto lhs_res = lhs_decimal.Rescale(lhs_type->scale(), rhs_type->scale());
+    if (!lhs_res) {
+      // Rescale would cause data loss, so lhs is definitely greater than rhs
+      return std::strong_ordering::greater;
+    }
+    return lhs_res.value() <=> rhs_decimal;
+  }
+}
+
 bool Literal::operator==(const Literal& other) const { return (*this <=> other) == 0; }
 
 // Three-way comparison operator
 std::partial_ordering Literal::operator<=>(const Literal& other) const {
   // If types are different, comparison is unordered
-  if (*type_ != *other.type_) {
+  if (type_->type_id() != other.type_->type_id()) {
     return std::partial_ordering::unordered;
   }
 
@@ -385,6 +421,10 @@ std::partial_ordering Literal::operator<=>(const Literal& other) const {
       return CompareFloat(this_val, other_val);
     }
 
+    case TypeId::kDecimal: {
+      return CompareDecimal(*this, other);
+    }
+
     case TypeId::kString: {
       auto& this_val = std::get<std::string>(value_);
       auto& other_val = std::get<std::string>(other.value_);
@@ -440,6 +480,13 @@ std::string Literal::ToString() const {
     case TypeId::kDouble: {
       return std::to_string(std::get<double>(value_));
     }
+    case TypeId::kDecimal: {
+      auto decimal_type = internal::checked_pointer_cast<DecimalType>(type_);
+      auto decimal = std::get<::iceberg::Decimal>(value_);
+      auto result = decimal.ToString(decimal_type->scale());
+      ICEBERG_CHECK(result, "Decimal ToString failed");
+      return *result;
+    }
     case TypeId::kString: {
       return "\"" + std::get<std::string>(value_) + "\"";
     }

src/iceberg/expression/literal.h

@@ -27,7 +27,9 @@
 
 #include "iceberg/result.h"
 #include "iceberg/type.h"
+#include "iceberg/util/decimal.h"
 #include "iceberg/util/formattable.h"
+#include "iceberg/util/int128.h"
 #include "iceberg/util/uuid.h"
 
 namespace iceberg {
@@ -56,10 +58,10 @@ class ICEBERG_EXPORT Literal : public util::Formattable {
                              int64_t,         // for long, timestamp, timestamp_tz, time
                              float,           // for float
                              double,          // for double
-                             std::string,     // for string
-                             Uuid,            // for uuid
-                             std::vector<uint8_t>,     // for binary, fixed
-                             std::array<uint8_t, 16>,  // for decimal
+                             ::iceberg::Decimal,    // for decimal
+                             std::string,           // for string
+                             Uuid,                  // for uuid
+                             std::vector<uint8_t>,  // for binary, fixed
                              BelowMin, AboveMax>;
 
   /// \brief Factory methods for primitive types
@@ -77,6 +79,10 @@ class ICEBERG_EXPORT Literal : public util::Formattable {
   static Literal Binary(std::vector<uint8_t> value);
   static Literal Fixed(std::vector<uint8_t> value);
 
+  /// \brief Create a decimal literal.
+  /// \param value The unscaled 128-bit integer value.
+  static Literal Decimal(int128_t value, int32_t precision, int32_t scale);
+
   /// \brief Create a literal representing a null value.
   static Literal Null(std::shared_ptr<PrimitiveType> type) {
     return {Value{std::monostate{}}, std::move(type)};
@@ -205,6 +211,11 @@ struct LiteralTraits<TypeId::kDouble> {
   using ValueType = double;
 };
 
+template <>
+struct LiteralTraits<TypeId::kDecimal> {
+  using ValueType = Decimal;
+};
+
 template <>
 struct LiteralTraits<TypeId::kString> {
   using ValueType = std::string;

src/iceberg/meson.build

@@ -77,11 +77,14 @@ iceberg_sources = files(
     'transform.cc',
     'transform_function.cc',
     'type.cc',
+    'util/bucket_util.cc',
     'util/conversions.cc',
     'util/decimal.cc',
     'util/gzip_internal.cc',
     'util/murmurhash3_internal.cc',
+    'util/temporal_util.cc',
     'util/timepoint.cc',
+    'util/truncate_util.cc',
     'util/uuid.cc',
 )
 

src/iceberg/test/CMakeLists.txt

@@ -99,11 +99,13 @@ add_iceberg_test(json_serde_test
 
 add_iceberg_test(util_test
                  SOURCES
+                 bucket_util_test.cc
                  config_test.cc
                  decimal_test.cc
                  endian_test.cc
                  formatter_test.cc
                  string_util_test.cc
+                 truncate_util_test.cc
                  uuid_test.cc
                  visit_type_test.cc)
 

src/iceberg/test/bucket_util_test.cc

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/bucket_util.h"
+
+#include <chrono>
+
+#include <gtest/gtest.h>
+
+#include "iceberg/util/decimal.h"
+#include "iceberg/util/uuid.h"
+
+namespace iceberg {
+
+// The following tests are from
+// https://iceberg.apache.org/spec/#appendix-b-32-bit-hash-requirements
+TEST(BucketUtilsTest, HashHelper) {
+  // int and long
+  EXPECT_EQ(BucketUtils::HashInt(34), 2017239379);
+  EXPECT_EQ(BucketUtils::HashLong(34L), 2017239379);
+
+  // decimal hash
+  auto decimal = Decimal::FromString("14.20");
+  ASSERT_TRUE(decimal.has_value());
+  EXPECT_EQ(BucketUtils::HashBytes(decimal->ToBigEndian()), -500754589);
+
+  // date hash
+  std::chrono::sys_days sd = std::chrono::year{2017} / 11 / 16;
+  std::chrono::sys_days epoch{std::chrono::year{1970} / 1 / 1};
+  int32_t days = (sd - epoch).count();
+  EXPECT_EQ(BucketUtils::HashInt(days), -653330422);
+
+  // time
+  // 22:31:08 in microseconds
+  int64_t time_micros = (22 * 3600 + 31 * 60 + 8) * 1000000LL;
+  EXPECT_EQ(BucketUtils::HashLong(time_micros), -662762989);
+
+  // timestamp
+  // 2017-11-16T22:31:08 in microseconds
+  std::chrono::system_clock::time_point tp =
+      std::chrono::sys_days{std::chrono::year{2017} / 11 / 16} + std::chrono::hours{22} +
+      std::chrono::minutes{31} + std::chrono::seconds{8};
+  int64_t timestamp_micros =
+      std::chrono::duration_cast<std::chrono::microseconds>(tp.time_since_epoch())
+          .count();
+  EXPECT_EQ(BucketUtils::HashLong(timestamp_micros), -2047944441);
+  // 2017-11-16T22:31:08.000001 in microseconds
+  EXPECT_EQ(BucketUtils::HashLong(timestamp_micros + 1), -1207196810);
+
+  // string
+  std::string str = "iceberg";
+  EXPECT_EQ(BucketUtils::HashBytes(std::span<const uint8_t>(
+                reinterpret_cast<const uint8_t*>(str.data()), str.size())),
+            1210000089);
+
+  // uuid
+  auto uuid = Uuid::FromString("f79c3e09-677c-4bbd-a479-3f349cb785e7");
+  EXPECT_EQ(BucketUtils::HashBytes(uuid->bytes()), 1488055340);
+
+  // fixed & binary
+  std::vector<uint8_t> fixed = {0, 1, 2, 3};
+  EXPECT_EQ(BucketUtils::HashBytes(fixed), -188683207);
+}
+
+}  // namespace iceberg

src/iceberg/test/decimal_test.cc

@@ -490,6 +490,50 @@ TEST(DecimalTest, FromBigEndianInvalid) {
               IsError(ErrorKind::kInvalidArgument));
 }
 
+TEST(DecimalTest, ToBigEndian) {
+  std::vector<int64_t> high_values = {0,
+                                      1,
+                                      -1,
+                                      INT32_MAX,
+                                      INT32_MIN,
+                                      static_cast<int64_t>(INT32_MAX) + 1,
+                                      static_cast<int64_t>(INT32_MIN) - 1,
+                                      INT64_MAX,
+                                      INT64_MIN};
+  std::vector<uint64_t> low_values = {0,
+                                      1,
+                                      255,
+                                      UINT32_MAX,
+                                      static_cast<uint64_t>(UINT32_MAX) + 1,
+                                      static_cast<uint64_t>(UINT32_MAX) + 2,
+                                      static_cast<uint64_t>(UINT32_MAX) + 3,
+                                      static_cast<uint64_t>(UINT32_MAX) + 4,
+                                      static_cast<uint64_t>(UINT32_MAX) + 5,
+                                      static_cast<uint64_t>(UINT32_MAX) + 6,
+                                      static_cast<uint64_t>(UINT32_MAX) + 7,
+                                      static_cast<uint64_t>(UINT32_MAX) + 8,
+                                      UINT64_MAX};
+
+  for (int64_t high : high_values) {
+    for (uint64_t low : low_values) {
+      Decimal decimal(high, low);
+      auto bytes = decimal.ToBigEndian();
+      auto result = Decimal::FromBigEndian(bytes.data(), bytes.size());
+      ASSERT_THAT(result, IsOk());
+      EXPECT_EQ(result.value(), decimal);
+    }
+  }
+
+  for (int128_t value : std::vector<int128_t>{-INT64_MAX, -INT32_MAX, -255, -1, 0, 1, 255,
+                                              256, INT32_MAX, INT64_MAX}) {
+    Decimal decimal(value);
+    auto bytes = decimal.ToBigEndian();
+    auto result = Decimal::FromBigEndian(bytes.data(), bytes.size());
+    ASSERT_THAT(result, IsOk());
+    EXPECT_EQ(result.value(), decimal);
+  }
+}
+
 TEST(DecimalTestFunctionality, Multiply) {
   ASSERT_EQ(Decimal(60501), Decimal(301) * Decimal(201));
   ASSERT_EQ(Decimal(-60501), Decimal(-301) * Decimal(201));

src/iceberg/test/literal_test.cc

@@ -256,6 +256,20 @@ TEST(LiteralTest, DoubleZeroComparison) {
   EXPECT_EQ(neg_zero <=> pos_zero, std::partial_ordering::less);
 }
 
+TEST(LiteralTest, UuidComparison) {
+  auto uuid1 = Uuid::FromString("123e4567-e89b-12d3-a456-426614174000").value();
+  auto uuid2 = Uuid::FromString("123e4567-e89b-12d3-a456-426614174001").value();
+  auto uuid3 = Uuid::FromString("123e4567-e89b-12d3-a456-426614174000").value();
+
+  auto literal1 = Literal::UUID(uuid1);
+  auto literal2 = Literal::UUID(uuid2);
+  auto literal3 = Literal::UUID(uuid3);
+
+  EXPECT_EQ(literal1 <=> literal3, std::partial_ordering::equivalent);
+  EXPECT_EQ(literal1 <=> literal2, std::partial_ordering::unordered);
+  EXPECT_EQ(literal2 <=> literal1, std::partial_ordering::unordered);
+}
+
 // Parameter struct for literal serialization and deserialization tests
 struct LiteralParam {
   std::string test_name;
@@ -346,6 +360,17 @@ INSTANTIATE_TEST_SUITE_P(
                      Literal::Double(std::numeric_limits<double>::lowest()),
                      float64()},
 
+        // Decimal type
+        LiteralParam{"DecimalPositive",
+                     {1, 226, 64},
+                     Literal::Decimal(123456, 6, 2),
+                     decimal(6, 2)},
+        LiteralParam{"DecimalNegative",
+                     {254, 29, 192},
+                     Literal::Decimal(-123456, 6, 2),
+                     decimal(6, 2)},
+        LiteralParam{"DecimalZero", {0}, Literal::Decimal(0, 3, 0), decimal(3, 0)},
+
         LiteralParam{"String",
                      {105, 99, 101, 98, 101, 114, 103},
                      Literal::String("iceberg"),
@@ -506,10 +531,28 @@ INSTANTIATE_TEST_SUITE_P(
                               .literal = Literal::Double(std::numbers::pi),
                               .expected_type_id = TypeId::kDouble,
                               .expected_string = "3.141593"},
+        BasicLiteralTestParam{.test_name = "DecimalPositive",
+                              .literal = Literal::Decimal(123456, 6, 2),
+                              .expected_type_id = TypeId::kDecimal,
+                              .expected_string = "1234.56"},
+        BasicLiteralTestParam{.test_name = "DecimalNegative",
+                              .literal = Literal::Decimal(-123456, 6, 2),
+                              .expected_type_id = TypeId::kDecimal,
+                              .expected_string = "-1234.56"},
+        BasicLiteralTestParam{.test_name = "DecimalZero",
+                              .literal = Literal::Decimal(0, 3, 0),
+                              .expected_type_id = TypeId::kDecimal,
+                              .expected_string = "0"},
         BasicLiteralTestParam{.test_name = "String",
                               .literal = Literal::String("hello world"),
                               .expected_type_id = TypeId::kString,
                               .expected_string = "\"hello world\""},
+        BasicLiteralTestParam{
+            .test_name = "Uuid",
+            .literal = Literal::UUID(
+                Uuid::FromString("123e4567-e89b-12d3-a456-426614174000").value()),
+            .expected_type_id = TypeId::kUuid,
+            .expected_string = "123e4567-e89b-12d3-a456-426614174000"},
         BasicLiteralTestParam{
             .test_name = "Binary",
             .literal = Literal::Binary(std::vector<uint8_t>{0x01, 0x02, 0x03, 0xFF}),
@@ -563,6 +606,10 @@ INSTANTIATE_TEST_SUITE_P(
                                    .small_literal = Literal::Double(1.5),
                                    .large_literal = Literal::Double(2.5),
                                    .equal_literal = Literal::Double(1.5)},
+        ComparisonLiteralTestParam{.test_name = "Decimal",
+                                   .small_literal = Literal::Decimal(123456, 6, 2),
+                                   .large_literal = Literal::Decimal(234567, 6, 2),
+                                   .equal_literal = Literal::Decimal(123456, 6, 2)},
         ComparisonLiteralTestParam{.test_name = "String",
                                    .small_literal = Literal::String("apple"),
                                    .large_literal = Literal::String("banana"),