Merge pull request #3 from wgtmac/add_avro

Add iceberg_avro library

Author: zhjwpku

.github/workflows/test.yml

@@ -88,7 +88,7 @@ jobs:
         shell: cmd
         run: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
-          bash -c "ci/scripts/build_iceberg_windows.sh $(pwd)
+          bash -c "ci/scripts/build_iceberg.sh $(pwd)
       - name: Build Example
         shell: cmd
         run: |

ci/scripts/build_example.sh

@@ -25,10 +25,27 @@ build_dir=${1}/build
 mkdir ${build_dir}
 pushd ${build_dir}
 
-cmake \
-    -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \
-    ${source_dir}
-cmake --build .
+is_windows() {
+    [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]
+}
+
+CMAKE_ARGS=(
+    "-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
+)
+
+if is_windows; then
+    CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
+    CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
+else
+    CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
+fi
+
+cmake "${CMAKE_ARGS[@]}" ${source_dir}
+if is_windows; then
+  cmake --build . --config Release
+else
+  cmake --build .
+fi
 
 popd
 

ci/scripts/build_iceberg.sh

@@ -25,13 +25,31 @@ build_dir=${1}/build
 mkdir ${build_dir}
 pushd ${build_dir}
 
-cmake \
-    -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \
-    -DICEBERG_BUILD_STATIC=ON \
-    -DICEBERG_BUILD_SHARED=ON \
-    ${source_dir}
-cmake --build . --target install
-ctest --output-on-failure -C Debug
+is_windows() {
+    [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]
+}
+
+CMAKE_ARGS=(
+    "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
+    "-DICEBERG_BUILD_STATIC=ON"
+    "-DICEBERG_BUILD_SHARED=ON"
+)
+
+if is_windows; then
+    CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
+    CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
+else
+    CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
+fi
+
+cmake "${CMAKE_ARGS[@]}" ${source_dir}
+if is_windows; then
+  cmake --build . --config Release --target install
+  ctest --output-on-failure -C Release
+else
+  cmake --build . --target install
+  ctest --output-on-failure
+fi
 
 popd
 

ci/scripts/build_iceberg_windows.sh

@@ -147,8 +147,11 @@ function(resolve_avro_dependency)
 
   fetchcontent_declare(Avro
                        ${FC_DECLARE_COMMON_OPTIONS}
-                       GIT_REPOSITORY https://github.com/apache/avro.git
-                       GIT_TAG 1144cb7322bab4cd1c8bf330a9c504a0d4252b56
+                       # TODO: switch to upstream once the PR below is merged
+                       # https://github.com/apache/avro/pull/3299
+                       # Eventually, we should switch to Apache Avro 1.3.0.
+                       GIT_REPOSITORY https://github.com/wgtmac/avro.git
+                       GIT_TAG 0aa7adf87a9af6d472a3e9d5966c5e7f1d6baa7d
                        SOURCE_SUBDIR
                        lang/c++
                        FIND_PACKAGE_ARGS
@@ -159,33 +162,39 @@ function(resolve_avro_dependency)
   fetchcontent_makeavailable(Avro)
 
   if(avro_SOURCE_DIR)
-    if(NOT TARGET Avro::avro_static)
-      add_library(Avro::avro_static INTERFACE IMPORTED)
-      target_link_libraries(Avro::avro_static INTERFACE avrocpp_s)
-      set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avro")
+    if(NOT TARGET Avro::avrocpp_static)
+      add_library(Avro::avrocpp_static INTERFACE IMPORTED)
+      target_link_libraries(Avro::avrocpp_static INTERFACE avrocpp_s)
+      target_include_directories(Avro::avrocpp_static
+                                 INTERFACE ${avro_BINARY_DIR} ${avro_SOURCE_DIR}/lang/c++)
     endif()
 
-    if(NOT TARGET Avro::avro_shared)
-      add_library(Avro::avro_shared INTERFACE IMPORTED)
-      target_link_libraries(Avro::avro_shared INTERFACE avrocpp)
-      set_target_properties(avrocpp PROPERTIES OUTPUT_NAME "iceberg_vendored_avro")
-    endif()
+    set(AVRO_VENDORED TRUE)
+    set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp")
+    set_target_properties(avrocpp_s PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    install(TARGETS avrocpp_s
+            EXPORT iceberg_targets
+            RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
+            ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
+            LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
 
-    if(ICEBERG_BUILD_STATIC)
-      install(TARGETS avrocpp_s
-              EXPORT fmt-targets
-              RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
-              ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
-              LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
-    endif()
-    if(ICEBERG_BUILD_SHARED)
-      install(TARGETS avrocpp
-              EXPORT fmt-targets
-              RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
-              ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
-              LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
+    # TODO: add vendored ZLIB and Snappy support
+    find_package(Snappy CONFIG)
+    if(Snappy_FOUND)
+      list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Snappy)
     endif()
+    list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB)
+  else()
+    set(AVRO_VENDORED FALSE)
+    list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro)
   endif()
+
+  set(ICEBERG_SYSTEM_DEPENDENCIES
+      ${ICEBERG_SYSTEM_DEPENDENCIES}
+      PARENT_SCOPE)
+  set(AVRO_VENDORED
+      ${AVRO_VENDORED}
+      PARENT_SCOPE)
 endfunction()
 
 if(ICEBERG_AVRO)

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -26,5 +26,6 @@ find_package(Iceberg CONFIG REQUIRED)
 
 add_executable(demo_example demo_example.cc)
 
-target_link_libraries(demo_example PRIVATE Iceberg::iceberg_puffin_static
-                                           Iceberg::iceberg_arrow_static)
+target_link_libraries(demo_example
+                      PRIVATE Iceberg::iceberg_puffin_static
+                              Iceberg::iceberg_arrow_static Iceberg::iceberg_avro_static)

example/CMakeLists.txt

@@ -20,12 +20,14 @@
 #include <iostream>
 
 #include "iceberg/arrow/demo_arrow.h"
+#include "iceberg/avro/demo_avro.h"
 #include "iceberg/demo_table.h"
 #include "iceberg/puffin/demo_puffin.h"
 
 int main() {
   std::cout << iceberg::DemoTable().print() << std::endl;
   std::cout << iceberg::puffin::DemoPuffin().print() << std::endl;
   std::cout << iceberg::arrow::DemoArrow().print() << std::endl;
+  std::cout << iceberg::avro::DemoAvro().print() << std::endl;
   return 0;
 }

example/demo_example.cc

@@ -28,20 +28,39 @@ set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS)
 set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS)
 set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS)
 
-list(APPEND ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS
+list(APPEND
+     ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS
      "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
-     Avro::avro_static)
-list(APPEND ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
+)
+list(APPEND
+     ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS
      "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
-     Avro::avro_shared)
+     "$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
+)
+
+if(AVRO_VENDORED)
+  list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s)
+  list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s)
+else()
+  list(APPEND
+       ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS
+       "$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
+  )
+  list(APPEND
+       ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS
+       "$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
+  )
+endif()
+
 list(APPEND
      ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS
      "$<IF:$<TARGET_EXISTS:Iceberg::iceberg_static>,Iceberg::iceberg_static,Iceberg::iceberg_shared>"
-     Iceberg::avro_static)
+)
 list(APPEND
      ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS
      "$<IF:$<TARGET_EXISTS:Iceberg::iceberg_shared>,Iceberg::iceberg_shared,Iceberg::iceberg_static>"
-     Iceberg::avro_shared)
+)
 
 add_iceberg_lib(iceberg_avro
                 SOURCES

src/iceberg/avro/CMakeLists.txt

@@ -25,7 +25,7 @@
 #include "avro/ValidSchema.hh"
 #include "iceberg/demo_table.h"
 
-namespace iceberg {
+namespace iceberg::avro {
 
 std::string DemoAvro::print() const {
   std::string input =
@@ -42,11 +42,11 @@ std::string DemoAvro::print() const {
 }\n\
 ";
 
-  avro::ValidSchema schema = avro::compileJsonSchemaFromString(input);
+  ::avro::ValidSchema schema = ::avro::compileJsonSchemaFromString(input);
   std::ostringstream actual;
   schema.toJson(actual);
 
   return actual.str();
 }
 
-}  // namespace iceberg
+}  // namespace iceberg::avro

src/iceberg/avro/demo_avro.cc

@@ -24,7 +24,7 @@
 #include "iceberg/avro.h"
 #include "iceberg/avro/iceberg_avro_export.h"
 
-namespace iceberg {
+namespace iceberg::avro {
 
 class ICEBERG_AVRO_EXPORT DemoAvro : public Avro {
  public:
@@ -33,4 +33,4 @@ class ICEBERG_AVRO_EXPORT DemoAvro : public Avro {
   std::string print() const override;
 };
 
-}  // namespace iceberg
+}  // namespace iceberg::avro