diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a471ae009..90f197c93 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,21 +72,25 @@ jobs: shell: bash run: ci/scripts/build_example.sh $(pwd)/example windows: - name: AMD64 Windows 2019 - runs-on: windows-2019 + name: AMD64 Windows 2022 + runs-on: windows-2022 timeout-minutes: 30 steps: - name: Checkout iceberg-cpp uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + - name: Install ZLIB + shell: cmd + run: | + vcpkg install zlib:x64-windows - name: Build Iceberg shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_iceberg.sh $(pwd) + call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/build_iceberg_windows.sh $(pwd) - name: Build Example shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 bash -c "ci/scripts/build_example.sh $(pwd)/example diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c2968722..9b52e30d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ option(ICEBERG_BUILD_STATIC "Build static library" ON) option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) option(ICEBERG_ARROW "Build Arrow" ON) +option(ICEBERG_AVRO "Build Avro" ON) include(GNUInstallDirs) include(FetchContent) diff --git a/ci/scripts/build_iceberg_windows.sh b/ci/scripts/build_iceberg_windows.sh new file mode 100644 index 000000000..0e9ba67d1 --- /dev/null +++ b/ci/scripts/build_iceberg_windows.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +source_dir=${1} +build_dir=${1}/build + +mkdir ${build_dir} +pushd ${build_dir} + +cmake \ + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ + -DICEBERG_BUILD_STATIC=ON \ + -DICEBERG_BUILD_SHARED=ON \ + -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake \ + ${source_dir} +cmake --build . --target install +ctest --output-on-failure -C Debug + +popd + +# clean up between builds +rm -rf ${build_dir} diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 1b00134b9..3f846145b 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -126,3 +126,72 @@ endfunction() if(ICEBERG_ARROW) resolve_arrow_dependency() endif() + +# ---------------------------------------------------------------------- +# Apache Avro + +function(resolve_avro_dependency) + prepare_fetchcontent() + + set(AVRO_USE_BOOST + OFF + CACHE BOOL "" FORCE) + + set(AVRO_BUILD_EXECUTABLES + OFF + CACHE BOOL "" FORCE) + + set(AVRO_BUILD_TESTS + OFF + CACHE BOOL "" FORCE) + + fetchcontent_declare(Avro + ${FC_DECLARE_COMMON_OPTIONS} + # TODO: switch to upstream once the PR below is merged + # https://github.com/apache/avro/pull/3299 + # Eventually, we should switch to Apache Avro 1.3.0. + GIT_REPOSITORY https://github.com/wgtmac/avro.git + GIT_TAG 0aa7adf87a9af6d472a3e9d5966c5e7f1d6baa7d + SOURCE_SUBDIR + lang/c++ + FIND_PACKAGE_ARGS + NAMES + Avro + CONFIG) + + fetchcontent_makeavailable(Avro) + + if(avro_SOURCE_DIR) + if(NOT TARGET Avro::avrocpp_static) + add_library(Avro::avrocpp_static INTERFACE IMPORTED) + target_link_libraries(Avro::avrocpp_static INTERFACE avrocpp_s) + target_include_directories(Avro::avrocpp_static + INTERFACE ${avro_BINARY_DIR} ${avro_SOURCE_DIR}/lang/c++) + endif() + + set(AVRO_VENDORED TRUE) + set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp") + install(TARGETS avrocpp_s + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") + + # TODO: add vendored ZLIB and Snappy support + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB Snappy) + else() + set(AVRO_VENDORED FALSE) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro) + endif() + + set(ICEBERG_SYSTEM_DEPENDENCIES + ${ICEBERG_SYSTEM_DEPENDENCIES} + PARENT_SCOPE) + set(AVRO_VENDORED + ${AVRO_VENDORED} + PARENT_SCOPE) +endfunction() + +if(ICEBERG_AVRO) + resolve_avro_dependency() +endif() diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 4da20ca3d..c2c7a9ccf 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -26,5 +26,6 @@ find_package(Iceberg CONFIG REQUIRED) add_executable(demo_example demo_example.cc) -target_link_libraries(demo_example PRIVATE Iceberg::iceberg_puffin_static - Iceberg::iceberg_arrow_static) +target_link_libraries(demo_example + PRIVATE Iceberg::iceberg_puffin_static + Iceberg::iceberg_arrow_static Iceberg::iceberg_avro_static) diff --git a/example/demo_example.cc b/example/demo_example.cc index f4801f10b..f584339ed 100644 --- a/example/demo_example.cc +++ b/example/demo_example.cc @@ -20,6 +20,7 @@ #include #include "iceberg/arrow/demo_arrow.h" +#include "iceberg/avro/demo_avro.h" #include "iceberg/demo_table.h" #include "iceberg/puffin/demo_puffin.h" @@ -27,5 +28,6 @@ int main() { std::cout << iceberg::DemoTable().print() << std::endl; std::cout << iceberg::puffin::DemoPuffin().print() << std::endl; std::cout << iceberg::arrow::DemoArrow().print() << std::endl; + std::cout << iceberg::avro::DemoAvro().print() << std::endl; return 0; } diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 4bebe4e4e..24563a52b 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -29,6 +29,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_export.h DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg) add_subdirectory(arrow) +add_subdirectory(avro) add_subdirectory(puffin) iceberg_install_cmake_package(Iceberg iceberg_targets) diff --git a/src/iceberg/avro.h b/src/iceberg/avro.h new file mode 100644 index 000000000..9bd016017 --- /dev/null +++ b/src/iceberg/avro.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/iceberg_export.h" + +namespace iceberg { + +class ICEBERG_EXPORT Avro { + public: + virtual ~Avro() = default; + virtual std::string print() const = 0; +}; + +} // namespace iceberg diff --git a/src/iceberg/avro/CMakeLists.txt b/src/iceberg/avro/CMakeLists.txt new file mode 100644 index 000000000..4313a107b --- /dev/null +++ b/src/iceberg/avro/CMakeLists.txt @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(NOT ICEBERG_AVRO) + return() +endif() + +set(ICEBERG_AVRO_SOURCES demo_avro.cc) +set(ICEBERG_AVRO_INCLUDES "${ICEBERG_INCLUDES}") + +# Libraries to link with exported libiceberg_avro.{so,a}. +set(ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS) +set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS) +set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS) +set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS) + +list(APPEND + ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>" + "$,Avro::avrocpp_static,Avro::avrocpp_shared>" +) +list(APPEND + ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>" + "$,Avro::avrocpp_shared,Avro::avrocpp_static>" +) + +if(ARROW_VENDORED) + list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) + list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) +else() + list(APPEND + ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS + "$,Avro::avrocpp_static,Avro::avrocpp_shared>" + ) + list(APPEND + ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS + "$,Avro::avrocpp_shared,Avro::avrocpp_static>" + ) +endif() + +list(APPEND + ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS + "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" +) +list(APPEND + ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS + "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" +) + +add_iceberg_lib(iceberg_avro + SOURCES + ${ICEBERG_AVRO_SOURCES} + PRIVATE_INCLUDES + ${ICEBERG_AVRO_INCLUDES} + SHARED_LINK_LIBS + ${ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS} + STATIC_LINK_LIBS + ${ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS} + STATIC_INSTALL_INTERFACE_LIBS + ${ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_INSTALL_INTERFACE_LIBS + ${ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS}) + +iceberg_install_all_headers(iceberg/avro) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_avro_export.h + DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg/avro) diff --git a/src/iceberg/avro/demo_avro.cc b/src/iceberg/avro/demo_avro.cc new file mode 100644 index 000000000..b4bf00ef3 --- /dev/null +++ b/src/iceberg/avro/demo_avro.cc @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/avro/demo_avro.h" + +#include + +#include "avro/Compiler.hh" +#include "avro/ValidSchema.hh" +#include "iceberg/demo_table.h" + +namespace iceberg::avro { + +std::string DemoAvro::print() const { + std::string input = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + + ::avro::ValidSchema schema = ::avro::compileJsonSchemaFromString(input); + std::ostringstream actual; + schema.toJson(actual); + + return actual.str(); +} + +} // namespace iceberg::avro diff --git a/src/iceberg/avro/demo_avro.h b/src/iceberg/avro/demo_avro.h new file mode 100644 index 000000000..6abf969a3 --- /dev/null +++ b/src/iceberg/avro/demo_avro.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/avro.h" +#include "iceberg/avro/iceberg_avro_export.h" + +namespace iceberg::avro { + +class ICEBERG_AVRO_EXPORT DemoAvro : public Avro { + public: + DemoAvro() = default; + ~DemoAvro() override = default; + std::string print() const override; +}; + +} // namespace iceberg::avro diff --git a/src/iceberg/demo_table.cc b/src/iceberg/demo_table.cc index 29879b807..9e46bdadc 100644 --- a/src/iceberg/demo_table.cc +++ b/src/iceberg/demo_table.cc @@ -19,6 +19,7 @@ #include "iceberg/demo_table.h" +#include "iceberg/avro.h" // include to export symbols #include "iceberg/puffin.h" namespace iceberg { diff --git a/test/core/CMakeLists.txt b/test/core/CMakeLists.txt index 551201779..e6280edb3 100644 --- a/test/core/CMakeLists.txt +++ b/test/core/CMakeLists.txt @@ -20,3 +20,11 @@ target_sources(core_unittest PRIVATE core_unittest.cc) target_link_libraries(core_unittest PRIVATE iceberg_static GTest::gtest_main) target_include_directories(core_unittest PRIVATE "${ICEBERG_INCLUDES}") add_test(NAME core_unittest COMMAND core_unittest) + +if(ICEBERG_AVRO) + add_executable(avro_unittest) + target_sources(avro_unittest PRIVATE avro_unittest.cc) + target_link_libraries(avro_unittest PRIVATE iceberg_avro_static GTest::gtest_main) + target_include_directories(avro_unittest PRIVATE "${ICEBERG_INCLUDES}") + add_test(NAME avro_unittest COMMAND avro_unittest) +endif() diff --git a/test/core/avro_unittest.cc b/test/core/avro_unittest.cc new file mode 100644 index 000000000..5bdcfca24 --- /dev/null +++ b/test/core/avro_unittest.cc @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +TEST(AVROTest, TestDemoAvro) { + std::string expected = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + + auto avro = iceberg::avro::DemoAvro(); + EXPECT_EQ(avro.print(), expected); +}