Merge remote-tracking branch 'origin/main' into expr

Author: yingcai-cy

LICENSE

@@ -227,3 +227,30 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency nlohmann-json is statically linked in certain binary
+distributions. nlohmann-json has the following license:
+
+MIT License
+
+Copyright (c) 2013-2022 Niels Lohmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -226,3 +226,36 @@ function(resolve_nanoarrow_dependency)
 endfunction()
 
 resolve_nanoarrow_dependency()
+
+# ----------------------------------------------------------------------
+# nlohmann-json
+
+function(resolve_nlohmann_json_dependency)
+  prepare_fetchcontent()
+
+  set(JSON_BuildTests
+      OFF
+      CACHE BOOL "" FORCE)
+
+  fetchcontent_declare(nlohmann_json
+                       ${FC_DECLARE_COMMON_OPTIONS}
+                       URL "https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz"
+  )
+  fetchcontent_makeavailable(nlohmann_json)
+
+  set_target_properties(nlohmann_json
+                        PROPERTIES OUTPUT_NAME "iceberg_vendored_nlohmann_json"
+                                   POSITION_INDEPENDENT_CODE ON)
+  if(MSVC_TOOLCHAIN)
+    set(NLOHMANN_NATVIS_FILE ${nlohmann_json_SOURCE_DIR}/nlohmann_json.natvis)
+    install(FILES ${NLOHMANN_NATVIS_FILE} DESTINATION .)
+  endif()
+
+  install(TARGETS nlohmann_json
+          EXPORT iceberg_targets
+          RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
+          ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
+          LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
+endfunction()
+
+resolve_nlohmann_json_dependency()

src/iceberg/CMakeLists.txt

@@ -21,11 +21,16 @@ set(ICEBERG_SOURCES
     arrow_c_data_internal.cc
     demo.cc
     expression.cc
+    json_internal.cc
     schema.cc
     schema_field.cc
     schema_internal.cc
     partition_field.cc
     partition_spec.cc
+    sort_field.cc
+    sort_order.cc
+    statistics_file.cc
+    table_metadata.cc
     transform.cc
     type.cc)
 
@@ -34,10 +39,14 @@ set(ICEBERG_SHARED_BUILD_INTERFACE_LIBS)
 set(ICEBERG_STATIC_INSTALL_INTERFACE_LIBS)
 set(ICEBERG_SHARED_INSTALL_INTERFACE_LIBS)
 
-list(APPEND ICEBERG_STATIC_BUILD_INTERFACE_LIBS nanoarrow::nanoarrow)
-list(APPEND ICEBERG_SHARED_BUILD_INTERFACE_LIBS nanoarrow::nanoarrow)
-list(APPEND ICEBERG_STATIC_INSTALL_INTERFACE_LIBS "Iceberg::nanoarrow")
-list(APPEND ICEBERG_SHARED_INSTALL_INTERFACE_LIBS "Iceberg::nanoarrow")
+list(APPEND ICEBERG_STATIC_BUILD_INTERFACE_LIBS nanoarrow::nanoarrow
+     nlohmann_json::nlohmann_json)
+list(APPEND ICEBERG_SHARED_BUILD_INTERFACE_LIBS nanoarrow::nanoarrow
+     nlohmann_json::nlohmann_json)
+list(APPEND ICEBERG_STATIC_INSTALL_INTERFACE_LIBS "Iceberg::nanoarrow"
+     "Iceberg::nlohmann_json")
+list(APPEND ICEBERG_SHARED_INSTALL_INTERFACE_LIBS "Iceberg::nanoarrow"
+     "Iceberg::nlohmann_json")
 
 add_iceberg_lib(iceberg
                 SOURCES

src/iceberg/arrow/arrow_error_transform_internal.h

@@ -22,8 +22,8 @@
 #include <arrow/result.h>
 #include <arrow/status.h>
 
-#include "iceberg/error.h"
 #include "iceberg/expected.h"
+#include "iceberg/result.h"
 
 namespace iceberg::arrow {
 

src/iceberg/arrow/arrow_fs_file_io.cc

@@ -26,8 +26,8 @@
 namespace iceberg::arrow {
 
 /// \brief Read the content of the file at the given location.
-expected<std::string, Error> ArrowFileSystemFileIO::ReadFile(
-    const std::string& file_location, std::optional<size_t> length) {
+Result<std::string> ArrowFileSystemFileIO::ReadFile(const std::string& file_location,
+                                                    std::optional<size_t> length) {
   ::arrow::fs::FileInfo file_info(file_location);
   if (length.has_value()) {
     file_info.set_size(length.value());
@@ -52,8 +52,8 @@ expected<std::string, Error> ArrowFileSystemFileIO::ReadFile(
 }
 
 /// \brief Write the given content to the file at the given location.
-expected<void, Error> ArrowFileSystemFileIO::WriteFile(const std::string& file_location,
-                                                       std::string_view content) {
+Status ArrowFileSystemFileIO::WriteFile(const std::string& file_location,
+                                        std::string_view content) {
   ICEBERG_ARROW_ASSIGN_OR_RETURN(auto file, arrow_fs_->OpenOutputStream(file_location));
   ICEBERG_ARROW_RETURN_NOT_OK(file->Write(content.data(), content.size()));
   ICEBERG_ARROW_RETURN_NOT_OK(file->Flush());
@@ -62,8 +62,7 @@ expected<void, Error> ArrowFileSystemFileIO::WriteFile(const std::string& file_l
 }
 
 /// \brief Delete a file at the given location.
-expected<void, Error> ArrowFileSystemFileIO::DeleteFile(
-    const std::string& file_location) {
+Status ArrowFileSystemFileIO::DeleteFile(const std::string& file_location) {
   ICEBERG_ARROW_RETURN_NOT_OK(arrow_fs_->DeleteFile(file_location));
   return {};
 }

src/iceberg/arrow/arrow_fs_file_io.h

@@ -37,15 +37,14 @@ class ICEBERG_BUNDLE_EXPORT ArrowFileSystemFileIO : public FileIO {
   ~ArrowFileSystemFileIO() override = default;
 
   /// \brief Read the content of the file at the given location.
-  expected<std::string, Error> ReadFile(const std::string& file_location,
-                                        std::optional<size_t> length) override;
+  Result<std::string> ReadFile(const std::string& file_location,
+                               std::optional<size_t> length) override;
 
   /// \brief Write the given content to the file at the given location.
-  expected<void, Error> WriteFile(const std::string& file_location,
-                                  std::string_view content) override;
+  Status WriteFile(const std::string& file_location, std::string_view content) override;
 
   /// \brief Delete a file at the given location.
-  expected<void, Error> DeleteFile(const std::string& file_location) override;
+  Status DeleteFile(const std::string& file_location) override;
 
  private:
   std::shared_ptr<::arrow::fs::FileSystem> arrow_fs_;

src/iceberg/arrow_c_data_internal.cc

@@ -23,6 +23,8 @@
 #include <string>
 #include <utility>
 
+#include <nlohmann/json.hpp>
+
 namespace iceberg::internal {
 
 std::pair<ArrowSchema, ArrowArray> CreateExampleArrowSchemaAndArrayByNanoarrow() {
@@ -73,4 +75,11 @@ std::pair<ArrowSchema, ArrowArray> CreateExampleArrowSchemaAndArrayByNanoarrow()
   return {out_schema, out_array};
 }
 
+void TestNlohmannJsonCompile() {
+  nlohmann::json j;
+  j["name"] = "foo";
+  j["age"] = 30;
+  j["city"] = "New York";
+}
+
 }  // namespace iceberg::internal

src/iceberg/catalog.h

@@ -19,14 +19,13 @@
 
 #pragma once
 
-#include <map>
 #include <memory>
 #include <string>
 #include <string_view>
+#include <unordered_map>
 #include <vector>
 
-#include "iceberg/error.h"
-#include "iceberg/expected.h"
+#include "iceberg/result.h"
 #include "iceberg/table_identifier.h"
 #include "iceberg/type_fwd.h"
 
@@ -48,8 +47,7 @@ class ICEBERG_EXPORT Catalog {
   /// \param ns a namespace
   /// \return a list of identifiers for tables or ErrorKind::kNoSuchNamespace
   /// if the namespace does not exist
-  virtual expected<std::vector<TableIdentifier>, Error> ListTables(
-      const Namespace& ns) const = 0;
+  virtual Result<std::vector<TableIdentifier>> ListTables(const Namespace& ns) const = 0;
 
   /// \brief Create a table
   ///
@@ -59,18 +57,18 @@ class ICEBERG_EXPORT Catalog {
   /// \param location a location for the table; leave empty if unspecified
   /// \param properties a string map of table properties
   /// \return a Table instance or ErrorKind::kAlreadyExists if the table already exists
-  virtual expected<std::unique_ptr<Table>, Error> CreateTable(
+  virtual Result<std::unique_ptr<Table>> CreateTable(
       const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec,
       const std::string& location,
-      const std::map<std::string, std::string>& properties) = 0;
+      const std::unordered_map<std::string, std::string>& properties) = 0;
 
   /// \brief Update a table
   ///
   /// \param identifier a table identifier
   /// \param requirements a list of table requirements
   /// \param updates a list of table updates
   /// \return a Table instance or ErrorKind::kAlreadyExists if the table already exists
-  virtual expected<std::unique_ptr<Table>, Error> UpdateTable(
+  virtual Result<std::unique_ptr<Table>> UpdateTable(
       const TableIdentifier& identifier,
       const std::vector<std::unique_ptr<UpdateRequirement>>& requirements,
       const std::vector<std::unique_ptr<MetadataUpdate>>& updates) = 0;
@@ -84,10 +82,10 @@ class ICEBERG_EXPORT Catalog {
   /// \param properties a string map of table properties
   /// \return a Transaction to create the table or ErrorKind::kAlreadyExists if the table
   /// already exists
-  virtual expected<std::shared_ptr<Transaction>, Error> StageCreateTable(
+  virtual Result<std::shared_ptr<Transaction>> StageCreateTable(
       const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec,
       const std::string& location,
-      const std::map<std::string, std::string>& properties) = 0;
+      const std::unordered_map<std::string, std::string>& properties) = 0;
 
   /// \brief Check whether table exists
   ///
@@ -110,15 +108,15 @@ class ICEBERG_EXPORT Catalog {
   /// \param identifier a table identifier
   /// \return instance of Table implementation referred to by identifier or
   /// ErrorKind::kNoSuchTable if the table does not exist
-  virtual expected<std::shared_ptr<Table>, Error> LoadTable(
+  virtual Result<std::shared_ptr<Table>> LoadTable(
       const TableIdentifier& identifier) const = 0;
 
   /// \brief Register a table with the catalog if it does not exist
   ///
   /// \param identifier a table identifier
   /// \param metadata_file_location the location of a metadata file
   /// \return a Table instance or ErrorKind::kAlreadyExists if the table already exists
-  virtual expected<std::shared_ptr<Table>, Error> RegisterTable(
+  virtual Result<std::shared_ptr<Table>> RegisterTable(
       const TableIdentifier& identifier, const std::string& metadata_file_location) = 0;
 
   /// \brief Initialize a catalog given a custom name and a map of catalog properties
@@ -129,8 +127,9 @@ class ICEBERG_EXPORT Catalog {
   ///
   /// \param name a custom name for the catalog
   /// \param properties catalog properties
-  virtual void Initialize(const std::string& name,
-                          const std::map<std::string, std::string>& properties) = 0;
+  virtual void Initialize(
+      const std::string& name,
+      const std::unordered_map<std::string, std::string>& properties) = 0;
 
   /// \brief Instantiate a builder to either create a table or start a create/replace
   /// transaction
@@ -169,7 +168,7 @@ class ICEBERG_EXPORT Catalog {
     /// \param properties key/value properties
     /// \return this for method chaining
     virtual TableBuilder& WithProperties(
-        const std::map<std::string, std::string>& properties) = 0;
+        const std::unordered_map<std::string, std::string>& properties) = 0;
 
     /// \brief Adds a key/value property to the table
     ///

src/iceberg/file_io.h

@@ -23,9 +23,8 @@
 #include <string>
 #include <string_view>
 
-#include "iceberg/error.h"
-#include "iceberg/expected.h"
 #include "iceberg/iceberg_export.h"
+#include "iceberg/result.h"
 
 namespace iceberg {
 
@@ -50,8 +49,8 @@ class ICEBERG_EXPORT FileIO {
   /// the length to read, e.g. S3 `GetObject` has a Range parameter.
   /// \return The content of the file if the read succeeded, an error code if the read
   /// failed.
-  virtual expected<std::string, Error> ReadFile(const std::string& file_location,
-                                                std::optional<size_t> length) {
+  virtual Result<std::string> ReadFile(const std::string& file_location,
+                                       std::optional<size_t> length) {
     // We provide a default implementation to avoid Windows linker error LNK2019.
     return unexpected<Error>{
         {.kind = ErrorKind::kNotImplemented, .message = "ReadFile not implemented"}};
@@ -64,8 +63,7 @@ class ICEBERG_EXPORT FileIO {
   /// \param overwrite If true, overwrite the file if it exists. If false, fail if the
   /// file exists.
   /// \return void if the write succeeded, an error code if the write failed.
-  virtual expected<void, Error> WriteFile(const std::string& file_location,
-                                          std::string_view content) {
+  virtual Status WriteFile(const std::string& file_location, std::string_view content) {
     return unexpected<Error>{
         {.kind = ErrorKind::kNotImplemented, .message = "WriteFile not implemented"}};
   }
@@ -74,7 +72,7 @@ class ICEBERG_EXPORT FileIO {
   ///
   /// \param file_location The location of the file to delete.
   /// \return void if the delete succeeded, an error code if the delete failed.
-  virtual expected<void, Error> DeleteFile(const std::string& file_location) {
+  virtual Status DeleteFile(const std::string& file_location) {
     return unexpected<Error>{
         {.kind = ErrorKind::kNotImplemented, .message = "DeleteFile not implemented"}};
   }