From 73e64a7fa59f5c1765f7b192776ccc806a64290f Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sat, 18 Jan 2025 15:51:38 +0800 Subject: [PATCH 01/10] add libavrocpp_s This closes #17 The avro_unittest is added for testing if libavrocpp works, once we supported Manifest, this should be removed. Signed-off-by: Junwang Zhao --- .github/workflows/test.yml | 4 ++ .../IcebergThirdpartyToolchain.cmake | 41 +++++++++++++ src/iceberg/CMakeLists.txt | 7 ++- test/core/CMakeLists.txt | 6 ++ test/core/avro_unittest.cc | 58 +++++++++++++++++++ 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 test/core/avro_unittest.cc diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a471ae009..a0a90e160 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,6 +80,10 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + - name: Install ZLIB + shell: cmd + run: | + powershell -Command "(Invoke-WebRequest -Uri https://git.io/JnHTY -OutFile install_zlib.bat)"; ./install_zlib.bat; del install_zlib.bat - name: Build Iceberg shell: cmd run: | diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 1b00134b9..ea8099376 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -126,3 +126,44 @@ endfunction() if(ICEBERG_ARROW) resolve_arrow_dependency() endif() + +# ---------------------------------------------------------------------- +# Apache Avro + +function(resolve_avro_dependency) + prepare_fetchcontent() + + set(AVRO_USE_BOOST + OFF + CACHE BOOL "" FORCE) + + set(AVRO_BUILD_EXECUTABLES + OFF + CACHE BOOL "" FORCE) + + set(AVRO_BUILD_TESTS + OFF + CACHE BOOL "" FORCE) + + fetchcontent_declare(Avro + ${FC_DECLARE_COMMON_OPTIONS} + GIT_REPOSITORY https://github.com/apache/avro.git + GIT_TAG 1144cb7322bab4cd1c8bf330a9c504a0d4252b56 + SOURCE_SUBDIR + lang/c++ + FIND_PACKAGE_ARGS + NAMES + Avro + CONFIG) + + fetchcontent_makeavailable(Avro) + + if(avro_SOURCE_DIR) + if(NOT TARGET Avro::avro_static) + add_library(Avro::avro_static INTERFACE IMPORTED) + target_link_libraries(Avro::avro_static INTERFACE avrocpp_s) + endif() + endif() +endfunction() + +resolve_avro_dependency() diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 4bebe4e4e..333f1fdab 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -17,11 +17,16 @@ set(ICEBERG_SOURCES demo_table.cc) +set(ICEBERG_AVRO_STATIC_BUILD_LIBS) +list(APPEND ICEBERG_AVRO_STATIC_BUILD_LIBS Avro::avro_static) + add_iceberg_lib(iceberg SOURCES ${ICEBERG_SOURCES} PRIVATE_INCLUDES - ${ICEBERG_INCLUDES}) + ${ICEBERG_INCLUDES} + STATIC_LINK_LIBS + ${ICEBERG_AVRO_STATIC_BUILD_LIBS}) iceberg_install_all_headers(iceberg) diff --git a/test/core/CMakeLists.txt b/test/core/CMakeLists.txt index 551201779..89cfaacc3 100644 --- a/test/core/CMakeLists.txt +++ b/test/core/CMakeLists.txt @@ -20,3 +20,9 @@ target_sources(core_unittest PRIVATE core_unittest.cc) target_link_libraries(core_unittest PRIVATE iceberg_static GTest::gtest_main) target_include_directories(core_unittest PRIVATE "${ICEBERG_INCLUDES}") add_test(NAME core_unittest COMMAND core_unittest) + +add_executable(avro_unittest) +target_sources(avro_unittest PRIVATE avro_unittest.cc) +target_link_libraries(avro_unittest PRIVATE Avro::avro_static GTest::gtest_main) +target_include_directories(avro_unittest PRIVATE "${ICEBERG_INCLUDES}") +add_test(NAME avro_unittest COMMAND avro_unittest) diff --git a/test/core/avro_unittest.cc b/test/core/avro_unittest.cc new file mode 100644 index 000000000..fe4ed8fe3 --- /dev/null +++ b/test/core/avro_unittest.cc @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include "avro/Compiler.hh" +#include "avro/ValidSchema.hh" + +TEST(TableTest, TestTableCons) { + std::string input = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + std::string expected = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + + avro::ValidSchema schema = avro::compileJsonSchemaFromString(input); + std::ostringstream actual; + schema.toJson(actual); + + EXPECT_EQ(actual.str(), expected); +} From b195b861720453aec4c3a53e4bedad7b12fff951 Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Tue, 21 Jan 2025 19:10:27 +0800 Subject: [PATCH 02/10] install zlib using vcpkg Signed-off-by: Junwang Zhao --- .github/workflows/test.yml | 12 ++++----- ci/scripts/build_iceberg_windows.sh | 40 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 ci/scripts/build_iceberg_windows.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0a90e160..90f197c93 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,8 +72,8 @@ jobs: shell: bash run: ci/scripts/build_example.sh $(pwd)/example windows: - name: AMD64 Windows 2019 - runs-on: windows-2019 + name: AMD64 Windows 2022 + runs-on: windows-2022 timeout-minutes: 30 steps: - name: Checkout iceberg-cpp @@ -83,14 +83,14 @@ jobs: - name: Install ZLIB shell: cmd run: | - powershell -Command "(Invoke-WebRequest -Uri https://git.io/JnHTY -OutFile install_zlib.bat)"; ./install_zlib.bat; del install_zlib.bat + vcpkg install zlib:x64-windows - name: Build Iceberg shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_iceberg.sh $(pwd) + call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/build_iceberg_windows.sh $(pwd) - name: Build Example shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 bash -c "ci/scripts/build_example.sh $(pwd)/example diff --git a/ci/scripts/build_iceberg_windows.sh b/ci/scripts/build_iceberg_windows.sh new file mode 100644 index 000000000..0e9ba67d1 --- /dev/null +++ b/ci/scripts/build_iceberg_windows.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +source_dir=${1} +build_dir=${1}/build + +mkdir ${build_dir} +pushd ${build_dir} + +cmake \ + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ + -DICEBERG_BUILD_STATIC=ON \ + -DICEBERG_BUILD_SHARED=ON \ + -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake \ + ${source_dir} +cmake --build . --target install +ctest --output-on-failure -C Debug + +popd + +# clean up between builds +rm -rf ${build_dir} From 32daed6becbb57c001ff2a225a4c290663eb8e3b Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Tue, 21 Jan 2025 21:04:41 +0800 Subject: [PATCH 03/10] add iceberg_avro Signed-off-by: Junwang Zhao --- CMakeLists.txt | 1 + .../IcebergThirdpartyToolchain.cmake | 26 +++++++- src/iceberg/CMakeLists.txt | 8 +-- src/iceberg/avro.h | 34 ++++++++++ src/iceberg/avro/CMakeLists.txt | 66 +++++++++++++++++++ src/iceberg/avro/demo_avro.cc | 52 +++++++++++++++ src/iceberg/avro/demo_avro.h | 36 ++++++++++ test/core/CMakeLists.txt | 12 ++-- test/core/avro_unittest.cc | 26 ++------ 9 files changed, 227 insertions(+), 34 deletions(-) create mode 100644 src/iceberg/avro.h create mode 100644 src/iceberg/avro/CMakeLists.txt create mode 100644 src/iceberg/avro/demo_avro.cc create mode 100644 src/iceberg/avro/demo_avro.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c2968722..9b52e30d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ option(ICEBERG_BUILD_STATIC "Build static library" ON) option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) option(ICEBERG_ARROW "Build Arrow" ON) +option(ICEBERG_AVRO "Build Avro" ON) include(GNUInstallDirs) include(FetchContent) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index ea8099376..4c956f98e 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -162,8 +162,32 @@ function(resolve_avro_dependency) if(NOT TARGET Avro::avro_static) add_library(Avro::avro_static INTERFACE IMPORTED) target_link_libraries(Avro::avro_static INTERFACE avrocpp_s) + set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avro") + endif() + + if(NOT TARGET Avro::avro_shared) + add_library(Avro::avro_shared INTERFACE IMPORTED) + target_link_libraries(Avro::avro_shared INTERFACE avrocpp) + set_target_properties(avrocpp PROPERTIES OUTPUT_NAME "iceberg_vendored_avro") + endif() + + if(ICEBERG_BUILD_STATIC) + install(TARGETS avrocpp_s + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") + endif() + if(ICEBERG_BUILD_SHARED) + install(TARGETS avrocpp + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") endif() endif() endfunction() -resolve_avro_dependency() +if(ICEBERG_AVRO) + resolve_avro_dependency() +endif() diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 333f1fdab..24563a52b 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -17,16 +17,11 @@ set(ICEBERG_SOURCES demo_table.cc) -set(ICEBERG_AVRO_STATIC_BUILD_LIBS) -list(APPEND ICEBERG_AVRO_STATIC_BUILD_LIBS Avro::avro_static) - add_iceberg_lib(iceberg SOURCES ${ICEBERG_SOURCES} PRIVATE_INCLUDES - ${ICEBERG_INCLUDES} - STATIC_LINK_LIBS - ${ICEBERG_AVRO_STATIC_BUILD_LIBS}) + ${ICEBERG_INCLUDES}) iceberg_install_all_headers(iceberg) @@ -34,6 +29,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_export.h DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg) add_subdirectory(arrow) +add_subdirectory(avro) add_subdirectory(puffin) iceberg_install_cmake_package(Iceberg iceberg_targets) diff --git a/src/iceberg/avro.h b/src/iceberg/avro.h new file mode 100644 index 000000000..9bd016017 --- /dev/null +++ b/src/iceberg/avro.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/iceberg_export.h" + +namespace iceberg { + +class ICEBERG_EXPORT Avro { + public: + virtual ~Avro() = default; + virtual std::string print() const = 0; +}; + +} // namespace iceberg diff --git a/src/iceberg/avro/CMakeLists.txt b/src/iceberg/avro/CMakeLists.txt new file mode 100644 index 000000000..89ae8a8a3 --- /dev/null +++ b/src/iceberg/avro/CMakeLists.txt @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(NOT ICEBERG_AVRO) + return() +endif() + +set(ICEBERG_AVRO_SOURCES demo_avro.cc) +set(ICEBERG_AVRO_INCLUDES "${ICEBERG_INCLUDES}") + +# Libraries to link with exported libiceberg_avro.{so,a}. +set(ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS) +set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS) +set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS) +set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS) + +list(APPEND ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>" + Avro::avro_static) +list(APPEND ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>" + Avro::avro_shared) +list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avro_static) +list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avro_shared) + +list(APPEND + ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS + "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" +) +list(APPEND + ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS + "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" +) + +add_iceberg_lib(iceberg_avro + SOURCES + ${ICEBERG_AVRO_SOURCES} + PRIVATE_INCLUDES + ${ICEBERG_AVRO_INCLUDES} + SHARED_LINK_LIBS + ${ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS} + STATIC_LINK_LIBS + ${ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS} + STATIC_INSTALL_INTERFACE_LIBS + ${ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_INSTALL_INTERFACE_LIBS + ${ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS}) + +iceberg_install_all_headers(iceberg/avro) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_avro_export.h + DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg/avro) diff --git a/src/iceberg/avro/demo_avro.cc b/src/iceberg/avro/demo_avro.cc new file mode 100644 index 000000000..ee5d8bde6 --- /dev/null +++ b/src/iceberg/avro/demo_avro.cc @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/avro/demo_avro.h" + +#include + +#include "avro/Compiler.hh" +#include "avro/ValidSchema.hh" +#include "iceberg/demo_table.h" + +namespace iceberg { + +std::string DemoAvro::print() const { + std::string input = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + + avro::ValidSchema schema = avro::compileJsonSchemaFromString(input); + std::ostringstream actual; + schema.toJson(actual); + + return actual.str(); +} + +} // namespace iceberg diff --git a/src/iceberg/avro/demo_avro.h b/src/iceberg/avro/demo_avro.h new file mode 100644 index 000000000..b6a3375b0 --- /dev/null +++ b/src/iceberg/avro/demo_avro.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/avro.h" +#include "iceberg/avro/iceberg_avro_export.h" + +namespace iceberg { + +class ICEBERG_AVRO_EXPORT DemoAvro : public Avro { + public: + DemoAvro() = default; + ~DemoAvro() override = default; + std::string print() const override; +}; + +} // namespace iceberg diff --git a/test/core/CMakeLists.txt b/test/core/CMakeLists.txt index 89cfaacc3..e6280edb3 100644 --- a/test/core/CMakeLists.txt +++ b/test/core/CMakeLists.txt @@ -21,8 +21,10 @@ target_link_libraries(core_unittest PRIVATE iceberg_static GTest::gtest_main) target_include_directories(core_unittest PRIVATE "${ICEBERG_INCLUDES}") add_test(NAME core_unittest COMMAND core_unittest) -add_executable(avro_unittest) -target_sources(avro_unittest PRIVATE avro_unittest.cc) -target_link_libraries(avro_unittest PRIVATE Avro::avro_static GTest::gtest_main) -target_include_directories(avro_unittest PRIVATE "${ICEBERG_INCLUDES}") -add_test(NAME avro_unittest COMMAND avro_unittest) +if(ICEBERG_AVRO) + add_executable(avro_unittest) + target_sources(avro_unittest PRIVATE avro_unittest.cc) + target_link_libraries(avro_unittest PRIVATE iceberg_avro_static GTest::gtest_main) + target_include_directories(avro_unittest PRIVATE "${ICEBERG_INCLUDES}") + add_test(NAME avro_unittest COMMAND avro_unittest) +endif() diff --git a/test/core/avro_unittest.cc b/test/core/avro_unittest.cc index fe4ed8fe3..d5e4e00f2 100644 --- a/test/core/avro_unittest.cc +++ b/test/core/avro_unittest.cc @@ -18,24 +18,9 @@ */ #include +#include -#include "avro/Compiler.hh" -#include "avro/ValidSchema.hh" - -TEST(TableTest, TestTableCons) { - std::string input = - "{\n\ - \"type\": \"record\",\n\ - \"name\": \"testrecord\",\n\ - \"fields\": [\n\ - {\n\ - \"name\": \"testbytes\",\n\ - \"type\": \"bytes\",\n\ - \"default\": \"\"\n\ - }\n\ - ]\n\ -}\n\ -"; +TEST(AVROTest, TestDemoAvro) { std::string expected = "{\n\ \"type\": \"record\",\n\ @@ -50,9 +35,6 @@ TEST(TableTest, TestTableCons) { }\n\ "; - avro::ValidSchema schema = avro::compileJsonSchemaFromString(input); - std::ostringstream actual; - schema.toJson(actual); - - EXPECT_EQ(actual.str(), expected); + auto avro = iceberg::DemoAvro(); + EXPECT_EQ(avro.print(), expected); } From 9358d17d006b6ef8d4620cd85d81a53c3180d37f Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Tue, 21 Jan 2025 23:36:50 +0800 Subject: [PATCH 04/10] make example build Signed-off-by: Junwang Zhao --- cmake_modules/IcebergThirdpartyToolchain.cmake | 4 ++-- src/iceberg/avro/CMakeLists.txt | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 4c956f98e..aa778761d 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -173,14 +173,14 @@ function(resolve_avro_dependency) if(ICEBERG_BUILD_STATIC) install(TARGETS avrocpp_s - EXPORT iceberg_targets + EXPORT fmt-targets RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") endif() if(ICEBERG_BUILD_SHARED) install(TARGETS avrocpp - EXPORT iceberg_targets + EXPORT fmt-targets RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") diff --git a/src/iceberg/avro/CMakeLists.txt b/src/iceberg/avro/CMakeLists.txt index 89ae8a8a3..a728fe32a 100644 --- a/src/iceberg/avro/CMakeLists.txt +++ b/src/iceberg/avro/CMakeLists.txt @@ -34,17 +34,14 @@ list(APPEND ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS list(APPEND ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS "$,iceberg_shared,iceberg_static>" Avro::avro_shared) -list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avro_static) -list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avro_shared) - list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" -) + Iceberg::avro_static) list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" -) + Iceberg::avro_shared) add_iceberg_lib(iceberg_avro SOURCES From 923bf15b679d3d278d93e300b26d4b3a963882a3 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Thu, 23 Jan 2025 17:28:57 +0800 Subject: [PATCH 05/10] Adopt change from https://github.com/apache/avro/pull/3299 --- .../IcebergThirdpartyToolchain.cmake | 54 ++++++++++--------- example/CMakeLists.txt | 5 +- example/demo_example.cc | 2 + src/iceberg/avro/CMakeLists.txt | 31 ++++++++--- src/iceberg/avro/demo_avro.cc | 6 +-- src/iceberg/avro/demo_avro.h | 4 +- src/iceberg/demo_table.cc | 1 + test/core/avro_unittest.cc | 2 +- 8 files changed, 66 insertions(+), 39 deletions(-) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index aa778761d..3f846145b 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -147,8 +147,11 @@ function(resolve_avro_dependency) fetchcontent_declare(Avro ${FC_DECLARE_COMMON_OPTIONS} - GIT_REPOSITORY https://github.com/apache/avro.git - GIT_TAG 1144cb7322bab4cd1c8bf330a9c504a0d4252b56 + # TODO: switch to upstream once the PR below is merged + # https://github.com/apache/avro/pull/3299 + # Eventually, we should switch to Apache Avro 1.3.0. + GIT_REPOSITORY https://github.com/wgtmac/avro.git + GIT_TAG 0aa7adf87a9af6d472a3e9d5966c5e7f1d6baa7d SOURCE_SUBDIR lang/c++ FIND_PACKAGE_ARGS @@ -159,33 +162,34 @@ function(resolve_avro_dependency) fetchcontent_makeavailable(Avro) if(avro_SOURCE_DIR) - if(NOT TARGET Avro::avro_static) - add_library(Avro::avro_static INTERFACE IMPORTED) - target_link_libraries(Avro::avro_static INTERFACE avrocpp_s) - set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avro") + if(NOT TARGET Avro::avrocpp_static) + add_library(Avro::avrocpp_static INTERFACE IMPORTED) + target_link_libraries(Avro::avrocpp_static INTERFACE avrocpp_s) + target_include_directories(Avro::avrocpp_static + INTERFACE ${avro_BINARY_DIR} ${avro_SOURCE_DIR}/lang/c++) endif() - if(NOT TARGET Avro::avro_shared) - add_library(Avro::avro_shared INTERFACE IMPORTED) - target_link_libraries(Avro::avro_shared INTERFACE avrocpp) - set_target_properties(avrocpp PROPERTIES OUTPUT_NAME "iceberg_vendored_avro") - endif() + set(AVRO_VENDORED TRUE) + set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp") + install(TARGETS avrocpp_s + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") - if(ICEBERG_BUILD_STATIC) - install(TARGETS avrocpp_s - EXPORT fmt-targets - RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" - ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" - LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") - endif() - if(ICEBERG_BUILD_SHARED) - install(TARGETS avrocpp - EXPORT fmt-targets - RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" - ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" - LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") - endif() + # TODO: add vendored ZLIB and Snappy support + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB Snappy) + else() + set(AVRO_VENDORED FALSE) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro) endif() + + set(ICEBERG_SYSTEM_DEPENDENCIES + ${ICEBERG_SYSTEM_DEPENDENCIES} + PARENT_SCOPE) + set(AVRO_VENDORED + ${AVRO_VENDORED} + PARENT_SCOPE) endfunction() if(ICEBERG_AVRO) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 4da20ca3d..c2c7a9ccf 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -26,5 +26,6 @@ find_package(Iceberg CONFIG REQUIRED) add_executable(demo_example demo_example.cc) -target_link_libraries(demo_example PRIVATE Iceberg::iceberg_puffin_static - Iceberg::iceberg_arrow_static) +target_link_libraries(demo_example + PRIVATE Iceberg::iceberg_puffin_static + Iceberg::iceberg_arrow_static Iceberg::iceberg_avro_static) diff --git a/example/demo_example.cc b/example/demo_example.cc index f4801f10b..f584339ed 100644 --- a/example/demo_example.cc +++ b/example/demo_example.cc @@ -20,6 +20,7 @@ #include #include "iceberg/arrow/demo_arrow.h" +#include "iceberg/avro/demo_avro.h" #include "iceberg/demo_table.h" #include "iceberg/puffin/demo_puffin.h" @@ -27,5 +28,6 @@ int main() { std::cout << iceberg::DemoTable().print() << std::endl; std::cout << iceberg::puffin::DemoPuffin().print() << std::endl; std::cout << iceberg::arrow::DemoArrow().print() << std::endl; + std::cout << iceberg::avro::DemoAvro().print() << std::endl; return 0; } diff --git a/src/iceberg/avro/CMakeLists.txt b/src/iceberg/avro/CMakeLists.txt index a728fe32a..4313a107b 100644 --- a/src/iceberg/avro/CMakeLists.txt +++ b/src/iceberg/avro/CMakeLists.txt @@ -28,20 +28,39 @@ set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS) set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS) set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS) -list(APPEND ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS +list(APPEND + ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS "$,iceberg_static,iceberg_shared>" - Avro::avro_static) -list(APPEND ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS + "$,Avro::avrocpp_static,Avro::avrocpp_shared>" +) +list(APPEND + ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS "$,iceberg_shared,iceberg_static>" - Avro::avro_shared) + "$,Avro::avrocpp_shared,Avro::avrocpp_static>" +) + +if(ARROW_VENDORED) + list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) + list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) +else() + list(APPEND + ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS + "$,Avro::avrocpp_static,Avro::avrocpp_shared>" + ) + list(APPEND + ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS + "$,Avro::avrocpp_shared,Avro::avrocpp_static>" + ) +endif() + list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" - Iceberg::avro_static) +) list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" - Iceberg::avro_shared) +) add_iceberg_lib(iceberg_avro SOURCES diff --git a/src/iceberg/avro/demo_avro.cc b/src/iceberg/avro/demo_avro.cc index ee5d8bde6..b4bf00ef3 100644 --- a/src/iceberg/avro/demo_avro.cc +++ b/src/iceberg/avro/demo_avro.cc @@ -25,7 +25,7 @@ #include "avro/ValidSchema.hh" #include "iceberg/demo_table.h" -namespace iceberg { +namespace iceberg::avro { std::string DemoAvro::print() const { std::string input = @@ -42,11 +42,11 @@ std::string DemoAvro::print() const { }\n\ "; - avro::ValidSchema schema = avro::compileJsonSchemaFromString(input); + ::avro::ValidSchema schema = ::avro::compileJsonSchemaFromString(input); std::ostringstream actual; schema.toJson(actual); return actual.str(); } -} // namespace iceberg +} // namespace iceberg::avro diff --git a/src/iceberg/avro/demo_avro.h b/src/iceberg/avro/demo_avro.h index b6a3375b0..6abf969a3 100644 --- a/src/iceberg/avro/demo_avro.h +++ b/src/iceberg/avro/demo_avro.h @@ -24,7 +24,7 @@ #include "iceberg/avro.h" #include "iceberg/avro/iceberg_avro_export.h" -namespace iceberg { +namespace iceberg::avro { class ICEBERG_AVRO_EXPORT DemoAvro : public Avro { public: @@ -33,4 +33,4 @@ class ICEBERG_AVRO_EXPORT DemoAvro : public Avro { std::string print() const override; }; -} // namespace iceberg +} // namespace iceberg::avro diff --git a/src/iceberg/demo_table.cc b/src/iceberg/demo_table.cc index 29879b807..9e46bdadc 100644 --- a/src/iceberg/demo_table.cc +++ b/src/iceberg/demo_table.cc @@ -19,6 +19,7 @@ #include "iceberg/demo_table.h" +#include "iceberg/avro.h" // include to export symbols #include "iceberg/puffin.h" namespace iceberg { diff --git a/test/core/avro_unittest.cc b/test/core/avro_unittest.cc index d5e4e00f2..5bdcfca24 100644 --- a/test/core/avro_unittest.cc +++ b/test/core/avro_unittest.cc @@ -35,6 +35,6 @@ TEST(AVROTest, TestDemoAvro) { }\n\ "; - auto avro = iceberg::DemoAvro(); + auto avro = iceberg::avro::DemoAvro(); EXPECT_EQ(avro.print(), expected); } From be6f8d2df0440bba259c9e30d40ad1a03e332d3f Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Thu, 23 Jan 2025 17:45:12 +0800 Subject: [PATCH 06/10] fix ci --- .github/workflows/test.yml | 2 +- ci/scripts/build_example.sh | 19 +++++++-- ci/scripts/build_iceberg.sh | 23 ++++++++--- ci/scripts/build_iceberg_windows.sh | 40 ------------------- .../IcebergThirdpartyToolchain.cmake | 7 +++- 5 files changed, 39 insertions(+), 52 deletions(-) delete mode 100644 ci/scripts/build_iceberg_windows.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90f197c93..360431445 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_iceberg_windows.sh $(pwd) + bash -c "ci/scripts/build_iceberg.sh $(pwd) - name: Build Example shell: cmd run: | diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index a09f9c998..cc9e78ef8 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -25,10 +25,21 @@ build_dir=${1}/build mkdir ${build_dir} pushd ${build_dir} -cmake \ - -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ - ${source_dir} -cmake --build . +CMAKE_ARGS=( + "-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" + "-DCMAKE_BUILD_TYPE=Debug" +) + +BUILD_ARGS=() + +# Add Windows-specific toolchain file if on Windows +if [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]; then + CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") + BUILD_ARGS+=("--config Debug") +fi + +cmake "${CMAKE_ARGS[@]}" ${source_dir} +cmake --build . "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" popd diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index d08e0b263..d83bebe69 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -25,12 +25,23 @@ build_dir=${1}/build mkdir ${build_dir} pushd ${build_dir} -cmake \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ - -DICEBERG_BUILD_STATIC=ON \ - -DICEBERG_BUILD_SHARED=ON \ - ${source_dir} -cmake --build . --target install +CMAKE_ARGS=( + "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" + "-DICEBERG_BUILD_STATIC=ON" + "-DICEBERG_BUILD_SHARED=ON" + "-DCMAKE_BUILD_TYPE=Debug" +) + +BUILD_ARGS=() + +# Add Windows-specific toolchain file if on Windows +if [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]; then + CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") + BUILD_ARGS+=("--config Debug") +fi + +cmake "${CMAKE_ARGS[@]}" ${source_dir} +cmake --build . "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" --target install ctest --output-on-failure -C Debug popd diff --git a/ci/scripts/build_iceberg_windows.sh b/ci/scripts/build_iceberg_windows.sh deleted file mode 100644 index 0e9ba67d1..000000000 --- a/ci/scripts/build_iceberg_windows.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -source_dir=${1} -build_dir=${1}/build - -mkdir ${build_dir} -pushd ${build_dir} - -cmake \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ - -DICEBERG_BUILD_STATIC=ON \ - -DICEBERG_BUILD_SHARED=ON \ - -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake \ - ${source_dir} -cmake --build . --target install -ctest --output-on-failure -C Debug - -popd - -# clean up between builds -rm -rf ${build_dir} diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 3f846145b..83deb190f 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -171,6 +171,7 @@ function(resolve_avro_dependency) set(AVRO_VENDORED TRUE) set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp") + set_target_properties(avrocpp_s PROPERTIES POSITION_INDEPENDENT_CODE ON) install(TARGETS avrocpp_s EXPORT iceberg_targets RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" @@ -178,7 +179,11 @@ function(resolve_avro_dependency) LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") # TODO: add vendored ZLIB and Snappy support - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB Snappy) + find_package(Snappy CONFIG) + if(Snappy_FOUND) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Snappy) + endif() + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB) else() set(AVRO_VENDORED FALSE) list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro) From e2c581bf08aaa345d5f3d9b900601ff0cab606b4 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Thu, 23 Jan 2025 23:49:04 +0800 Subject: [PATCH 07/10] rollback windows --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 360431445..b389ed945 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,8 +72,8 @@ jobs: shell: bash run: ci/scripts/build_example.sh $(pwd)/example windows: - name: AMD64 Windows 2022 - runs-on: windows-2022 + name: AMD64 Windows 2019 + runs-on: windows-2019 timeout-minutes: 30 steps: - name: Checkout iceberg-cpp From e3ee8df83eaecdcdef0e97e57caf8cb6684521df Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Fri, 24 Jan 2025 00:00:26 +0800 Subject: [PATCH 08/10] fix windows --- .github/workflows/test.yml | 4 ++-- ci/scripts/build_example.sh | 20 +++++++++++++------- ci/scripts/build_iceberg.sh | 23 +++++++++++++++-------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b389ed945..360431445 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,8 +72,8 @@ jobs: shell: bash run: ci/scripts/build_example.sh $(pwd)/example windows: - name: AMD64 Windows 2019 - runs-on: windows-2019 + name: AMD64 Windows 2022 + runs-on: windows-2022 timeout-minutes: 30 steps: - name: Checkout iceberg-cpp diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index cc9e78ef8..eb33e12da 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -25,21 +25,27 @@ build_dir=${1}/build mkdir ${build_dir} pushd ${build_dir} +is_windows() { + [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]] +} + CMAKE_ARGS=( "-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" - "-DCMAKE_BUILD_TYPE=Debug" ) -BUILD_ARGS=() - -# Add Windows-specific toolchain file if on Windows -if [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]; then +if is_windows; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - BUILD_ARGS+=("--config Debug") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") +else + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") fi cmake "${CMAKE_ARGS[@]}" ${source_dir} -cmake --build . "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" +if is_windows; then + cmake --build . --config Release +else + cmake --build . +fi popd diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index d83bebe69..4351b0f40 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -25,24 +25,31 @@ build_dir=${1}/build mkdir ${build_dir} pushd ${build_dir} +is_windows() { + [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]] +} + CMAKE_ARGS=( "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" "-DICEBERG_BUILD_STATIC=ON" "-DICEBERG_BUILD_SHARED=ON" - "-DCMAKE_BUILD_TYPE=Debug" ) -BUILD_ARGS=() - -# Add Windows-specific toolchain file if on Windows -if [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]; then +if is_windows; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - BUILD_ARGS+=("--config Debug") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") +else + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") fi cmake "${CMAKE_ARGS[@]}" ${source_dir} -cmake --build . "${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"}" --target install -ctest --output-on-failure -C Debug +if is_windows; then + cmake --build . --config Release --target install + ctest --output-on-failure -C Release +else + cmake --build . --target install + ctest --output-on-failure +fi popd From 79cbf7bbc965d08f0b18b055c8457688577e4b74 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Fri, 24 Jan 2025 10:01:18 +0800 Subject: [PATCH 09/10] fix AVRO_VENDORED --- src/iceberg/avro/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/avro/CMakeLists.txt b/src/iceberg/avro/CMakeLists.txt index 4313a107b..6a17eb500 100644 --- a/src/iceberg/avro/CMakeLists.txt +++ b/src/iceberg/avro/CMakeLists.txt @@ -39,7 +39,7 @@ list(APPEND "$,Avro::avrocpp_shared,Avro::avrocpp_static>" ) -if(ARROW_VENDORED) +if(AVRO_VENDORED) list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) else() From 0fd88c169d94095aa9422366f5583c86a481dcc1 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Fri, 24 Jan 2025 10:39:00 +0800 Subject: [PATCH 10/10] minor fix --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 360431445..502941462 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,10 +87,10 @@ jobs: - name: Build Iceberg shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_iceberg.sh $(pwd) + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/build_iceberg.sh $(pwd)" - name: Build Example shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_example.sh $(pwd)/example + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/build_example.sh $(pwd)/example"