Merge branch 'apache:main' into main

Author: nullccxsy

.github/workflows/cpp-linter.yml

@@ -47,7 +47,7 @@ jobs:
         with:
           style: file
           tidy-checks: ''
-          version: 19
+          version: 22
           files-changed-only: true
           lines-changed-only: true
           thread-comments: true

.pre-commit-config.yaml

@@ -30,7 +30,7 @@ repos:
       - id: check-added-large-files
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v19.1.5
+    rev: v20.1.8
     hooks:
       - id: clang-format
         exclude: ^test/resources/.*\.json$

src/iceberg/catalog.h

@@ -176,15 +176,6 @@ class ICEBERG_EXPORT Catalog {
   virtual Result<std::shared_ptr<Table>> RegisterTable(
       const TableIdentifier& identifier, const std::string& metadata_file_location) = 0;
 
-  /// \brief Instantiate a builder to either create a table or start a create/replace
-  /// transaction
-  ///
-  /// \param identifier a table identifier
-  /// \param schema a schema
-  /// \return the builder to create a table or start a create/replace transaction
-  virtual std::unique_ptr<class TableBuilder> BuildTable(
-      const TableIdentifier& identifier, const Schema& schema) const = 0;
-
   /// \brief A builder used to create valid tables or start create/replace transactions
   class TableBuilder {
    public:
@@ -233,6 +224,15 @@ class ICEBERG_EXPORT Catalog {
     /// \return the Transaction to create the table
     virtual std::unique_ptr<Transaction> StageCreate() = 0;
   };
+
+  /// \brief Instantiate a builder to either create a table or start a create/replace
+  /// transaction
+  ///
+  /// \param identifier a table identifier
+  /// \param schema a schema
+  /// \return the builder to create a table or start a create/replace transaction
+  virtual std::unique_ptr<TableBuilder> BuildTable(const TableIdentifier& identifier,
+                                                   const Schema& schema) const = 0;
 };
 
 }  // namespace iceberg

src/iceberg/catalog/in_memory_catalog.cc

@@ -441,7 +441,7 @@ Result<std::shared_ptr<Table>> InMemoryCatalog::RegisterTable(
   return LoadTable(identifier);
 }
 
-std::unique_ptr<TableBuilder> InMemoryCatalog::BuildTable(
+std::unique_ptr<Catalog::TableBuilder> InMemoryCatalog::BuildTable(
     const TableIdentifier& identifier, const Schema& schema) const {
   throw IcebergError("not implemented");
 }

src/iceberg/catalog/in_memory_catalog.h

@@ -90,7 +90,7 @@ class ICEBERG_EXPORT InMemoryCatalog
       const TableIdentifier& identifier,
       const std::string& metadata_file_location) override;
 
-  std::unique_ptr<iceberg::TableBuilder> BuildTable(const TableIdentifier& identifier,
+  std::unique_ptr<Catalog::TableBuilder> BuildTable(const TableIdentifier& identifier,
                                                     const Schema& schema) const override;
 
  private:

src/iceberg/expression/literal.cc

@@ -21,7 +21,6 @@
 
 #include <cmath>
 #include <concepts>
-#include <sstream>
 
 #include "iceberg/exception.h"
 
@@ -126,22 +125,28 @@ Literal::Literal(Value value, std::shared_ptr<PrimitiveType> type)
     : value_(std::move(value)), type_(std::move(type)) {}
 
 // Factory methods
-Literal Literal::Boolean(bool value) { return {Value{value}, iceberg::boolean()}; }
+Literal Literal::Boolean(bool value) { return {Value{value}, boolean()}; }
 
-Literal Literal::Int(int32_t value) { return {Value{value}, iceberg::int32()}; }
+Literal Literal::Int(int32_t value) { return {Value{value}, int32()}; }
 
-Literal Literal::Long(int64_t value) { return {Value{value}, iceberg::int64()}; }
+Literal Literal::Date(int32_t value) { return {Value{value}, date()}; }
 
-Literal Literal::Float(float value) { return {Value{value}, iceberg::float32()}; }
+Literal Literal::Long(int64_t value) { return {Value{value}, int64()}; }
 
-Literal Literal::Double(double value) { return {Value{value}, iceberg::float64()}; }
+Literal Literal::Time(int64_t value) { return {Value{value}, time()}; }
 
-Literal Literal::String(std::string value) {
-  return {Value{std::move(value)}, iceberg::string()};
-}
+Literal Literal::Timestamp(int64_t value) { return {Value{value}, timestamp()}; }
+
+Literal Literal::TimestampTz(int64_t value) { return {Value{value}, timestamp_tz()}; }
+
+Literal Literal::Float(float value) { return {Value{value}, float32()}; }
+
+Literal Literal::Double(double value) { return {Value{value}, float64()}; }
+
+Literal Literal::String(std::string value) { return {Value{std::move(value)}, string()}; }
 
 Literal Literal::Binary(std::vector<uint8_t> value) {
-  return {Value{std::move(value)}, iceberg::binary()};
+  return {Value{std::move(value)}, binary()};
 }
 
 Result<Literal> Literal::Deserialize(std::span<const uint8_t> data,
@@ -188,8 +193,9 @@ std::partial_ordering Literal::operator<=>(const Literal& other) const {
     return std::partial_ordering::unordered;
   }
 
-  // If either value is AboveMax or BelowMin, comparison is unordered
-  if (IsAboveMax() || IsBelowMin() || other.IsAboveMax() || other.IsBelowMin()) {
+  // If either value is AboveMax, BelowMin or null, comparison is unordered
+  if (IsAboveMax() || IsBelowMin() || other.IsAboveMax() || other.IsBelowMin() ||
+      IsNull() || other.IsNull()) {
     return std::partial_ordering::unordered;
   }
 
@@ -202,13 +208,16 @@ std::partial_ordering Literal::operator<=>(const Literal& other) const {
       return this_val ? std::partial_ordering::greater : std::partial_ordering::less;
     }
 
-    case TypeId::kInt: {
+    case TypeId::kInt:
+    case TypeId::kDate: {
       auto this_val = std::get<int32_t>(value_);
       auto other_val = std::get<int32_t>(other.value_);
       return this_val <=> other_val;
     }
 
-    case TypeId::kLong: {
+    case TypeId::kLong:
+    case TypeId::kTimestamp:
+    case TypeId::kTimestampTz: {
       auto this_val = std::get<int64_t>(value_);
       auto other_val = std::get<int64_t>(other.value_);
       return this_val <=> other_val;
@@ -253,6 +262,9 @@ std::string Literal::ToString() const {
   if (std::holds_alternative<AboveMax>(value_)) {
     return "aboveMax";
   }
+  if (std::holds_alternative<std::monostate>(value_)) {
+    return "null";
+  }
 
   switch (type_->type_id()) {
     case TypeId::kBoolean: {
@@ -301,6 +313,8 @@ bool Literal::IsBelowMin() const { return std::holds_alternative<BelowMin>(value
 
 bool Literal::IsAboveMax() const { return std::holds_alternative<AboveMax>(value_); }
 
+bool Literal::IsNull() const { return std::holds_alternative<std::monostate>(value_); }
+
 // LiteralCaster implementation
 
 Result<Literal> LiteralCaster::CastTo(const Literal& literal,
@@ -312,7 +326,8 @@ Result<Literal> LiteralCaster::CastTo(const Literal& literal,
 
   // Handle special values
   if (std::holds_alternative<Literal::BelowMin>(literal.value_) ||
-      std::holds_alternative<Literal::AboveMax>(literal.value_)) {
+      std::holds_alternative<Literal::AboveMax>(literal.value_) ||
+      std::holds_alternative<std::monostate>(literal.value_)) {
     // Cannot cast type for special values
     return NotSupported("Cannot cast type for {}", literal.ToString());
   }

src/iceberg/expression/literal.h

@@ -32,7 +32,7 @@ namespace iceberg {
 
 /// \brief Literal is a literal value that is associated with a primitive type.
 class ICEBERG_EXPORT Literal {
- private:
+ public:
   /// \brief Sentinel value to indicate that the literal value is below the valid range
   /// of a specific primitive type. It can happen when casting a literal to a narrower
   /// primitive type.
@@ -48,27 +48,35 @@ class ICEBERG_EXPORT Literal {
     bool operator==(const AboveMax&) const = default;
     std::strong_ordering operator<=>(const AboveMax&) const = default;
   };
-
-  using Value = std::variant<bool,         // for boolean
-                             int32_t,      // for int, date
-                             int64_t,      // for long, timestamp, timestamp_tz, time
-                             float,        // for float
-                             double,       // for double
-                             std::string,  // for string
+  using Value = std::variant<std::monostate,  // for null
+                             bool,            // for boolean
+                             int32_t,         // for int, date
+                             int64_t,         // for long, timestamp, timestamp_tz, time
+                             float,           // for float
+                             double,          // for double
+                             std::string,     // for string
                              std::vector<uint8_t>,     // for binary, fixed
                              std::array<uint8_t, 16>,  // for uuid and decimal
                              BelowMin, AboveMax>;
 
- public:
   /// \brief Factory methods for primitive types
   static Literal Boolean(bool value);
   static Literal Int(int32_t value);
+  static Literal Date(int32_t value);
   static Literal Long(int64_t value);
+  static Literal Time(int64_t value);
+  static Literal Timestamp(int64_t value);
+  static Literal TimestampTz(int64_t value);
   static Literal Float(float value);
   static Literal Double(double value);
   static Literal String(std::string value);
   static Literal Binary(std::vector<uint8_t> value);
 
+  /// \brief Create a literal representing a null value.
+  static Literal Null(std::shared_ptr<PrimitiveType> type) {
+    return {Value{std::monostate{}}, std::move(type)};
+  }
+
   /// \brief Restore a literal from single-value serialization.
   ///
   /// See [this spec](https://iceberg.apache.org/spec/#binary-single-value-serialization)
@@ -85,6 +93,9 @@ class ICEBERG_EXPORT Literal {
   /// \brief Get the literal type.
   const std::shared_ptr<PrimitiveType>& type() const;
 
+  /// \brief Get the literal value.
+  const Value& value() const { return value_; }
+
   /// \brief Converts this literal to a literal of the given type.
   ///
   /// When a predicate is bound to a concrete data column, literals are converted to match
@@ -123,6 +134,10 @@ class ICEBERG_EXPORT Literal {
   /// \return true if this literal represents a BelowMin value, false otherwise
   bool IsBelowMin() const;
 
+  /// Check if this literal is null.
+  /// \return true if this literal is null, false otherwise
+  bool IsNull() const;
+
   std::string ToString() const;
 
  private:

src/iceberg/file_writer.h

@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/file_writer.h
+/// Writer interface for file formats like Parquet, Avro and ORC.
+
+#include <functional>
+#include <memory>
+#include <optional>
+
+#include "iceberg/arrow_c_data.h"
+#include "iceberg/file_format.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Options for creating a writer.
+struct ICEBERG_EXPORT WriterOptions {
+  /// \brief The path to the file to write.
+  std::string path;
+  /// \brief The schema of the data to write.
+  ArrowSchema schema;
+  /// \brief FileIO instance to open the file. Writer implementations should down cast it
+  /// to the specific FileIO implementation. By default, the `iceberg-bundle` library uses
+  /// `ArrowFileSystemFileIO` as the default implementation.
+  std::shared_ptr<class FileIO> io;
+  /// \brief Format-specific or implementation-specific properties.
+  std::unordered_map<std::string, std::string> properties;
+};
+
+/// \brief Base writer class to write data from different file formats.
+class ICEBERG_EXPORT Writer {
+ public:
+  virtual ~Writer() = default;
+  Writer() = default;
+  Writer(const Writer&) = delete;
+  Writer& operator=(const Writer&) = delete;
+
+  /// \brief Open the writer.
+  virtual Status Open(const struct WriterOptions& options) = 0;
+
+  /// \brief Close the writer.
+  virtual Status Close() = 0;
+
+  /// \brief Write arrow data to the file.
+  ///
+  /// \return Status of write results.
+  virtual Status Write(ArrowArray data) = 0;
+};
+
+/// \brief Factory function to create a writer of a specific file format.
+using WriterFactory = std::function<Result<std::unique_ptr<Writer>>()>;
+
+/// \brief Registry of writer factories for different file formats.
+struct ICEBERG_EXPORT WriterFactoryRegistry {
+  /// \brief Register a factory function for a specific file format.
+  WriterFactoryRegistry(FileFormatType format_type, WriterFactory factory);
+
+  /// \brief Get the factory function for a specific file format.
+  static WriterFactory& GetFactory(FileFormatType format_type);
+
+  /// \brief Open a writer for a specific file format.
+  static Result<std::unique_ptr<Writer>> Open(FileFormatType format_type,
+                                              const WriterOptions& options);
+};
+
+}  // namespace iceberg

src/iceberg/manifest_writer.h

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/manifest_writer.h
+/// Data writer interface for manifest files.
+
+#include <memory>
+#include <vector>
+
+#include "iceberg/file_writer.h"
+#include "iceberg/iceberg_export.h"
+#include "iceberg/type_fwd.h"
+
+namespace iceberg {
+
+/// \brief Write manifest entries to a manifest file.
+class ICEBERG_EXPORT ManifestWriter {
+ public:
+  virtual ~ManifestWriter() = default;
+  virtual Status WriteManifestEntries(
+      const std::vector<ManifestEntry>& entries) const = 0;
+
+  /// \brief Creates a writer for a manifest file.
+  /// \param manifest_location Path to the manifest file.
+  /// \param file_io File IO implementation to use.
+  /// \return A Result containing the writer or an error.
+  static Result<std::unique_ptr<ManifestWriter>> MakeWriter(
+      std::string_view manifest_location, std::shared_ptr<FileIO> file_io,
+      std::shared_ptr<Schema> partition_schema);
+};
+
+/// \brief Write manifest files to a manifest list file.
+class ICEBERG_EXPORT ManifestListWriter {
+ public:
+  virtual ~ManifestListWriter() = default;
+  virtual Status WriteManifestFiles(const std::vector<ManifestFile>& files) const = 0;
+
+  /// \brief Creates a writer for the manifest list.
+  /// \param manifest_list_location Path to the manifest list file.
+  /// \param file_io File IO implementation to use.
+  /// \return A Result containing the writer or an error.
+  static Result<std::unique_ptr<ManifestListWriter>> MakeWriter(
+      std::string_view manifest_list_location, std::shared_ptr<FileIO> file_io);
+};
+
+}  // namespace iceberg