Merge branch 'apache:main' into feat/literal2

Author: HeartLinked

.github/workflows/cpp-linter.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Run build
         run: |
           mkdir build && cd build
-          cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+          cmake .. -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
           cmake --build .
       - uses: cpp-linter/cpp-linter-action@f91c446a32ae3eb9f98fef8c9ed4c7cb613a4f8a
         id: linter

.github/workflows/sanitizer_test.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Configure and Build with ASAN & UBSAN
         run: |
           mkdir build && cd build
-          cmake .. -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON
+          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON
           cmake --build . --verbose
       - name: Run Tests
         working-directory: build

.gitignore

@@ -23,3 +23,6 @@ cmake-build-release/
 
 # intellij files
 .idea
+
+# vscode files
+.vscode

README.md

@@ -32,7 +32,7 @@ C++ implementation of [Apache Iceberg™](https://iceberg.apache.org/).
 
 ```bash
 cd iceberg-cpp
-cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/path/to/install -DICEBERG_BUILD_STATIC=ON -DICEBERG_BUILD_SHARED=ON
+cmake -S . -B build -G Ninja -DCMAKE_INSTALL_PREFIX=/path/to/install -DICEBERG_BUILD_STATIC=ON -DICEBERG_BUILD_SHARED=ON
 cmake --build build
 ctest --test-dir build --output-on-failure
 cmake --install build
@@ -43,7 +43,7 @@ cmake --install build
 #### Vendored Apache Arrow (default)
 
 ```bash
-cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/path/to/install -DICEBERG_BUILD_BUNDLE=ON
+cmake -S . -B build -G Ninja -DCMAKE_INSTALL_PREFIX=/path/to/install -DICEBERG_BUILD_BUNDLE=ON
 cmake --build build
 ctest --test-dir build --output-on-failure
 cmake --install build
@@ -52,7 +52,7 @@ cmake --install build
 #### Provided Apache Arrow
 
 ```bash
-cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/path/to/install -DCMAKE_PREFIX_PATH=/path/to/arrow -DICEBERG_BUILD_BUNDLE=ON
+cmake -S . -B build -G Ninja -DCMAKE_INSTALL_PREFIX=/path/to/install -DCMAKE_PREFIX_PATH=/path/to/arrow -DICEBERG_BUILD_BUNDLE=ON
 cmake --build build
 ctest --test-dir build --output-on-failure
 cmake --install build
@@ -64,14 +64,14 @@ After installing the core libraries, you can build the examples:
 
 ```bash
 cd iceberg-cpp/example
-cmake -S . -B build -DCMAKE_PREFIX_PATH=/path/to/install
+cmake -S . -B build -G Ninja -DCMAKE_PREFIX_PATH=/path/to/install
 cmake --build build
 ```
 
 If you are using provided Apache Arrow, you need to include `/path/to/arrow` in `CMAKE_PREFIX_PATH` as below.
 
 ```bash
-cmake -S . -B build -DCMAKE_PREFIX_PATH="/path/to/install;/path/to/arrow"
+cmake -S . -B build -G Ninja -DCMAKE_PREFIX_PATH="/path/to/install;/path/to/arrow"
 ```
 
 ## Contribute

ci/scripts/build_example.sh

@@ -30,6 +30,7 @@ is_windows() {
 }
 
 CMAKE_ARGS=(
+    "-G Ninja"
     "-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
 )
 

ci/scripts/build_iceberg.sh

@@ -30,6 +30,7 @@ is_windows() {
 }
 
 CMAKE_ARGS=(
+    "-G Ninja"
     "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
     "-DICEBERG_BUILD_STATIC=ON"
     "-DICEBERG_BUILD_SHARED=ON"

example/demo_example.cc

@@ -19,18 +19,63 @@
 
 #include <iostream>
 
+#include "iceberg/arrow/arrow_file_io.h"
 #include "iceberg/avro/avro_register.h"
-#include "iceberg/file_reader.h"
+#include "iceberg/catalog/in_memory_catalog.h"
 #include "iceberg/parquet/parquet_register.h"
+#include "iceberg/table.h"
+#include "iceberg/table_scan.h"
+
+int main(int argc, char** argv) {
+  if (argc != 4) {
+    std::cerr << "Usage: " << argv[0]
+              << " <warehouse_location> <table_name> <table_location>" << std::endl;
+    return 0;
+  }
+
+  const std::string warehouse_location = argv[1];
+  const std::string table_name = argv[2];
+  const std::string table_location = argv[3];
+  const std::unordered_map<std::string, std::string> properties;
 
-int main() {
   iceberg::avro::RegisterAll();
   iceberg::parquet::RegisterAll();
-  auto open_result = iceberg::ReaderFactoryRegistry::Open(
-      iceberg::FileFormatType::kAvro, {.path = "non-existing-file.avro"});
-  if (!open_result.has_value()) {
-    std::cerr << "Failed to open avro file" << std::endl;
+
+  auto catalog = iceberg::InMemoryCatalog::Make("test", iceberg::arrow::MakeLocalFileIO(),
+                                                warehouse_location, properties);
+
+  auto register_result = catalog->RegisterTable({.name = table_name}, table_location);
+  if (!register_result.has_value()) {
+    std::cerr << "Failed to register table: " << register_result.error().message
+              << std::endl;
     return 1;
   }
+
+  auto load_result = catalog->LoadTable({.name = table_name});
+  if (!load_result.has_value()) {
+    std::cerr << "Failed to load table: " << load_result.error().message << std::endl;
+    return 1;
+  }
+
+  auto table = std::move(load_result.value());
+  auto scan_result = table->NewScan()->Build();
+  if (!scan_result.has_value()) {
+    std::cerr << "Failed to build scan: " << scan_result.error().message << std::endl;
+    return 1;
+  }
+
+  auto scan = std::move(scan_result.value());
+  auto plan_result = scan->PlanFiles();
+  if (!plan_result.has_value()) {
+    std::cerr << "Failed to plan files: " << plan_result.error().message << std::endl;
+    return 1;
+  }
+
+  std::cout << "Scan tasks: " << std::endl;
+  auto scan_tasks = std::move(plan_result.value());
+  for (const auto& scan_task : scan_tasks) {
+    std::cout << " - " << scan_task->data_file()->file_path << std::endl;
+  }
+
   return 0;
 }

src/iceberg/CMakeLists.txt

@@ -47,6 +47,7 @@ set(ICEBERG_SOURCES
     type.cc
     manifest_reader.cc
     manifest_reader_internal.cc
+    manifest_writer.cc
     arrow_c_data_guard_internal.cc
     util/murmurhash3_internal.cc
     util/timepoint.cc
@@ -115,7 +116,8 @@ if(ICEBERG_BUILD_BUNDLE)
       parquet/parquet_data_util.cc
       parquet/parquet_reader.cc
       parquet/parquet_register.cc
-      parquet/parquet_schema_util.cc)
+      parquet/parquet_schema_util.cc
+      parquet/parquet_writer.cc)
 
   # Libraries to link with exported libiceberg_bundle.{so,a}.
   set(ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS)

src/iceberg/arrow/arrow_error_transform_internal.h

@@ -30,6 +30,8 @@ inline ErrorKind ToErrorKind(const ::arrow::Status& status) {
   switch (status.code()) {
     case ::arrow::StatusCode::IOError:
       return ErrorKind::kIOError;
+    case ::arrow::StatusCode::NotImplemented:
+      return ErrorKind::kNotImplemented;
     default:
       return ErrorKind::kUnknownError;
   }

src/iceberg/arrow/arrow_file_io.h

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "iceberg/file_io.h"
+#include "iceberg/iceberg_bundle_export.h"
+
+namespace iceberg::arrow {
+
+ICEBERG_BUNDLE_EXPORT std::unique_ptr<FileIO> MakeMockFileIO();
+
+ICEBERG_BUNDLE_EXPORT std::unique_ptr<FileIO> MakeLocalFileIO();
+
+}  // namespace iceberg::arrow