feat: Literal support decimal & respect Appendix B: 32-bit Hash Requirements

Author: zhjwpku

src/iceberg/CMakeLists.txt

@@ -55,11 +55,14 @@ set(ICEBERG_SOURCES
     manifest_reader_internal.cc
     manifest_writer.cc
     arrow_c_data_guard_internal.cc
+    util/bucket_util.cc
     util/conversions.cc
     util/decimal.cc
     util/gzip_internal.cc
     util/murmurhash3_internal.cc
+    util/temporal_util.cc
     util/timepoint.cc
+    util/truncate_util.cc
     util/uuid.cc)
 
 set(ICEBERG_STATIC_BUILD_INTERFACE_LIBS)

src/iceberg/expression/literal.cc

@@ -158,6 +158,18 @@ Literal Literal::Fixed(std::vector<uint8_t> value) {
   return {Value{std::move(value)}, fixed(length)};
 }
 
+Literal Literal::MakeDecimal(Decimal value, int32_t precision, int32_t scale) {
+  return {Value{value}, decimal(precision, scale)};
+}
+
+Result<Literal> Literal::MakeDecimal(std::string_view value) {
+  int32_t precision = 0;
+  int32_t scale = 0;
+  ICEBERG_ASSIGN_OR_RAISE(auto decimal_value,
+                          Decimal::FromString(value, &precision, &scale));
+  return Literal{Value{decimal_value}, decimal(precision, scale)};
+}
+
 Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
                                      std::shared_ptr<PrimitiveType> type) {
   return Conversions::FromBytes(std::move(type), data);
@@ -193,12 +205,44 @@ std::strong_ordering CompareFloat(T lhs, T rhs) {
   return lhs_is_negative <=> rhs_is_negative;
 }
 
+std::strong_ordering CompareDecimal(Literal const& lhs, Literal const& rhs) {
+  ICEBERG_DCHECK(std::holds_alternative<Decimal>(lhs.value()),
+                 "LHS of decimal comparison must hold Decimal");
+  ICEBERG_DCHECK(std::holds_alternative<Decimal>(rhs.value()),
+                 "RHS of decimal comparison must hold decimal");
+  const auto& lhs_type = std::dynamic_pointer_cast<DecimalType>(lhs.type());
+  const auto& rhs_type = std::dynamic_pointer_cast<DecimalType>(rhs.type());
+  ICEBERG_DCHECK(lhs_type != nullptr, "LHS type must be DecimalType");
+  ICEBERG_DCHECK(rhs_type != nullptr, "RHS type must be DecimalType");
+  auto lhs_decimal = std::get<Decimal>(lhs.value());
+  auto rhs_decimal = std::get<Decimal>(rhs.value());
+  if (lhs_type->scale() == rhs_type->scale()) {
+    return lhs_decimal <=> rhs_decimal;
+  } else if (lhs_type->scale() > rhs_type->scale()) {
+    // Rescale to larger scale
+    auto rhs_res = rhs_decimal.Rescale(rhs_type->scale(), lhs_type->scale());
+    if (!rhs_res) {
+      // Rescale would cause data loss, so lhs is definitely less than rhs
+      return std::strong_ordering::less;
+    }
+    return lhs_decimal <=> rhs_res.value();
+  } else {
+    // Rescale to larger scale
+    auto lhs_res = lhs_decimal.Rescale(lhs_type->scale(), rhs_type->scale());
+    if (!lhs_res) {
+      // Rescale would cause data loss, so lhs is definitely greater than rhs
+      return std::strong_ordering::greater;
+    }
+    return lhs_res.value() <=> rhs_decimal;
+  }
+}
+
 bool Literal::operator==(const Literal& other) const { return (*this <=> other) == 0; }
 
 // Three-way comparison operator
 std::partial_ordering Literal::operator<=>(const Literal& other) const {
   // If types are different, comparison is unordered
-  if (*type_ != *other.type_) {
+  if (type_->type_id() != other.type_->type_id()) {
     return std::partial_ordering::unordered;
   }
 
@@ -247,6 +291,10 @@ std::partial_ordering Literal::operator<=>(const Literal& other) const {
       return CompareFloat(this_val, other_val);
     }
 
+    case TypeId::kDecimal: {
+      return CompareDecimal(*this, other);
+    }
+
     case TypeId::kString: {
       auto& this_val = std::get<std::string>(value_);
       auto& other_val = std::get<std::string>(other.value_);
@@ -307,6 +355,14 @@ std::string Literal::ToString() const {
     case TypeId::kDouble: {
       return std::to_string(std::get<double>(value_));
     }
+    case TypeId::kDecimal: {
+      auto decimal = std::get<Decimal>(value_);
+      auto decimal_type = std::dynamic_pointer_cast<DecimalType>(type_);
+      ICEBERG_DCHECK(decimal_type != nullptr, "Type must be DecimalType");
+      auto result = decimal.ToString(decimal_type->scale());
+      ICEBERG_CHECK(result, "Decimal ToString failed");
+      return *result;
+    }
     case TypeId::kString: {
       return std::get<std::string>(value_);
     }
@@ -331,7 +387,6 @@ std::string Literal::ToString() const {
       }
       return result;
     }
-    case TypeId::kDecimal:
     case TypeId::kDate:
     case TypeId::kTime:
     case TypeId::kTimestamp:

src/iceberg/expression/literal.h

@@ -22,11 +22,13 @@
 #include <compare>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <variant>
 #include <vector>
 
 #include "iceberg/result.h"
 #include "iceberg/type.h"
+#include "iceberg/util/decimal.h"
 #include "iceberg/util/formattable.h"
 #include "iceberg/util/uuid.h"
 
@@ -56,10 +58,10 @@ class ICEBERG_EXPORT Literal : public util::Formattable {
                              int64_t,         // for long, timestamp, timestamp_tz, time
                              float,           // for float
                              double,          // for double
+                             Decimal,         // for decimal
                              std::string,     // for string
                              Uuid,            // for uuid
-                             std::vector<uint8_t>,     // for binary, fixed
-                             std::array<uint8_t, 16>,  // for decimal
+                             std::vector<uint8_t>,  // for binary, fixed
                              BelowMin, AboveMax>;
 
   /// \brief Factory methods for primitive types
@@ -76,6 +78,8 @@ class ICEBERG_EXPORT Literal : public util::Formattable {
   static Literal UUID(Uuid value);
   static Literal Binary(std::vector<uint8_t> value);
   static Literal Fixed(std::vector<uint8_t> value);
+  static Literal MakeDecimal(Decimal value, int32_t precision, int32_t scale);
+  static Result<Literal> MakeDecimal(std::string_view value);
 
   /// \brief Create a literal representing a null value.
   static Literal Null(std::shared_ptr<PrimitiveType> type) {
@@ -205,6 +209,11 @@ struct LiteralTraits<TypeId::kDouble> {
   using ValueType = double;
 };
 
+template <>
+struct LiteralTraits<TypeId::kDecimal> {
+  using ValueType = Decimal;
+};
+
 template <>
 struct LiteralTraits<TypeId::kString> {
   using ValueType = std::string;

src/iceberg/test/CMakeLists.txt

@@ -99,11 +99,13 @@ add_iceberg_test(json_serde_test
 
 add_iceberg_test(util_test
                  SOURCES
+                 bucket_util_test.cc
                  config_test.cc
                  decimal_test.cc
                  endian_test.cc
                  formatter_test.cc
                  string_util_test.cc
+                 truncate_util_test.cc
                  uuid_test.cc
                  visit_type_test.cc)
 

src/iceberg/test/bucket_util_test.cc

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/util/bucket_util.h"
+
+#include <chrono>
+
+#include <gtest/gtest.h>
+
+namespace iceberg {
+
+// The following tests are from
+// https://iceberg.apache.org/spec/#appendix-b-32-bit-hash-requirements
+TEST(BucketUtilsTest, HashHelper) {
+  // int and long
+  EXPECT_EQ(BucketUtils::HashInt(34), 2017239379);
+  EXPECT_EQ(BucketUtils::HashLong(34L), 2017239379);
+
+  // decimal hash
+  auto decimal = Decimal::FromString("14.20");
+  ASSERT_TRUE(decimal.has_value());
+  EXPECT_EQ(BucketUtils::HashBytes(Decimal::ToBigEndian(decimal->value())), -500754589);
+
+  // date hash
+  // 2017-11-16
+  std::chrono::sys_days sd = std::chrono::year{2017} / 11 / 16;
+  std::chrono::sys_days epoch{std::chrono::year{1970} / 1 / 1};
+  int32_t days = (sd - epoch).count();
+  std::cout << "days: " << days << std::endl;
+  EXPECT_EQ(BucketUtils::HashInt(days), -653330422);
+
+  // time
+  // 22:31:08 in microseconds
+  int64_t time_micros = (22 * 3600 + 31 * 60 + 8) * 1000000LL;
+  std::cout << "time micros: " << time_micros << std::endl;
+  EXPECT_EQ(BucketUtils::HashLong(time_micros), -662762989);
+
+  // timestamp
+  // 2017-11-16T22:31:08 in microseconds
+  std::chrono::system_clock::time_point tp =
+      std::chrono::sys_days{std::chrono::year{2017} / 11 / 16} + std::chrono::hours{22} +
+      std::chrono::minutes{31} + std::chrono::seconds{8};
+  int64_t timestamp_micros =
+      std::chrono::duration_cast<std::chrono::microseconds>(tp.time_since_epoch())
+          .count();
+  std::cout << "timestamp micros: " << timestamp_micros << std::endl;
+  EXPECT_EQ(BucketUtils::HashLong(timestamp_micros), -2047944441);
+  // 2017-11-16T22:31:08.000001 in microseconds
+  EXPECT_EQ(BucketUtils::HashLong(timestamp_micros + 1), -1207196810);
+
+  // string
+  std::string str = "iceberg";
+  EXPECT_EQ(BucketUtils::HashBytes(std::span<const uint8_t>(
+                reinterpret_cast<const uint8_t*>(str.data()), str.size())),
+            1210000089);
+
+  // uuid
+  // f79c3e09-677c-4bbd-a479-3f349cb785e7
+  std::array<uint8_t, 16> uuid = {0xf7, 0x9c, 0x3e, 0x09, 0x67, 0x7c, 0x4b, 0xbd,
+                                  0xa4, 0x79, 0x3f, 0x34, 0x9c, 0xb7, 0x85, 0xe7};
+  EXPECT_EQ(BucketUtils::HashBytes(uuid), 1488055340);
+
+  // fixed & binary
+  std::vector<uint8_t> fixed = {0, 1, 2, 3};
+  EXPECT_EQ(BucketUtils::HashBytes(fixed), -188683207);
+}
+
+}  // namespace iceberg

src/iceberg/test/decimal_test.cc

@@ -490,6 +490,50 @@ TEST(DecimalTest, FromBigEndianInvalid) {
               IsError(ErrorKind::kInvalidArgument));
 }
 
+TEST(DecimalTest, ToBigEndian) {
+  std::vector<int64_t> high_values = {0,
+                                      1,
+                                      -1,
+                                      INT32_MAX,
+                                      INT32_MIN,
+                                      static_cast<int64_t>(INT32_MAX) + 1,
+                                      static_cast<int64_t>(INT32_MIN) - 1,
+                                      INT64_MAX,
+                                      INT64_MIN};
+  std::vector<uint64_t> low_values = {0,
+                                      1,
+                                      255,
+                                      UINT32_MAX,
+                                      static_cast<uint64_t>(UINT32_MAX) + 1,
+                                      static_cast<uint64_t>(UINT32_MAX) + 2,
+                                      static_cast<uint64_t>(UINT32_MAX) + 3,
+                                      static_cast<uint64_t>(UINT32_MAX) + 4,
+                                      static_cast<uint64_t>(UINT32_MAX) + 5,
+                                      static_cast<uint64_t>(UINT32_MAX) + 6,
+                                      static_cast<uint64_t>(UINT32_MAX) + 7,
+                                      static_cast<uint64_t>(UINT32_MAX) + 8,
+                                      UINT64_MAX};
+
+  for (int64_t high : high_values) {
+    for (uint64_t low : low_values) {
+      Decimal value(high, low);
+      auto bytes = Decimal::ToBigEndian(value.value());
+      auto result = Decimal::FromBigEndian(bytes.data(), bytes.size());
+      ASSERT_THAT(result, IsOk());
+      EXPECT_EQ(result.value(), value);
+    }
+  }
+
+  for (int128_t value : std::vector<int128_t>{-INT64_MAX, -INT32_MAX, -255, -1, 0, 1, 255,
+                                              256, INT32_MAX, INT64_MAX}) {
+    Decimal decimal(value);
+    auto bytes = Decimal::ToBigEndian(decimal.value());
+    auto result = Decimal::FromBigEndian(bytes.data(), bytes.size());
+    ASSERT_THAT(result, IsOk());
+    EXPECT_EQ(result.value(), decimal);
+  }
+}
+
 TEST(DecimalTestFunctionality, Multiply) {
   ASSERT_EQ(Decimal(60501), Decimal(301) * Decimal(201));
   ASSERT_EQ(Decimal(-60501), Decimal(-301) * Decimal(201));