Add avro

.github/workflows/test.yml

@@ -72,21 +72,25 @@ jobs:
         shell: bash
         run: ci/scripts/build_example.sh $(pwd)/example
   windows:
-    name: AMD64 Windows 2019
-    runs-on: windows-2019
+    name: AMD64 Windows 2022
+    runs-on: windows-2022
     timeout-minutes: 30
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
+      - name: Install ZLIB
+        shell: cmd
+        run: |
+          vcpkg install zlib:x64-windows
       - name: Build Iceberg
         shell: cmd
         run: |
-          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
-          bash -c "ci/scripts/build_iceberg.sh $(pwd)
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          bash -c "ci/scripts/build_iceberg_windows.sh $(pwd)
       - name: Build Example
         shell: cmd
         run: |
-          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
           bash -c "ci/scripts/build_example.sh $(pwd)/example

CMakeLists.txt

@@ -37,6 +37,7 @@ option(ICEBERG_BUILD_STATIC "Build static library" ON)
 option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
 option(ICEBERG_BUILD_TESTS "Build tests" ON)
 option(ICEBERG_ARROW "Build Arrow" ON)
+option(ICEBERG_AVRO "Build Avro" ON)
 
 include(GNUInstallDirs)
 include(FetchContent)

ci/scripts/build_iceberg_windows.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+source_dir=${1}
+build_dir=${1}/build
+
+mkdir ${build_dir}
+pushd ${build_dir}
+
+cmake \
+    -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \
+    -DICEBERG_BUILD_STATIC=ON \
+    -DICEBERG_BUILD_SHARED=ON \
+    -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake \
+    ${source_dir}
+cmake --build . --target install
+ctest --output-on-failure -C Debug
+
+popd
+
+# clean up between builds
+rm -rf ${build_dir}

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -126,3 +126,72 @@ endfunction()
 if(ICEBERG_ARROW)
   resolve_arrow_dependency()
 endif()
+
+# ----------------------------------------------------------------------
+# Apache Avro
+
+function(resolve_avro_dependency)
+  prepare_fetchcontent()
+
+  set(AVRO_USE_BOOST
+      OFF
+      CACHE BOOL "" FORCE)
+
+  set(AVRO_BUILD_EXECUTABLES
+      OFF
+      CACHE BOOL "" FORCE)
+
+  set(AVRO_BUILD_TESTS
+      OFF
+      CACHE BOOL "" FORCE)
+
+  fetchcontent_declare(Avro
+                       ${FC_DECLARE_COMMON_OPTIONS}
+                       # TODO: switch to upstream once the PR below is merged
+                       # https://github.com/apache/avro/pull/3299
+                       # Eventually, we should switch to Apache Avro 1.3.0.
+                       GIT_REPOSITORY https://github.com/wgtmac/avro.git
+                       GIT_TAG 0aa7adf87a9af6d472a3e9d5966c5e7f1d6baa7d
+                       SOURCE_SUBDIR
+                       lang/c++
+                       FIND_PACKAGE_ARGS
+                       NAMES
+                       Avro
+                       CONFIG)
+
+  fetchcontent_makeavailable(Avro)
+
+  if(avro_SOURCE_DIR)
+    if(NOT TARGET Avro::avrocpp_static)
+      add_library(Avro::avrocpp_static INTERFACE IMPORTED)
+      target_link_libraries(Avro::avrocpp_static INTERFACE avrocpp_s)
+      target_include_directories(Avro::avrocpp_static
+                                 INTERFACE ${avro_BINARY_DIR} ${avro_SOURCE_DIR}/lang/c++)
+    endif()
+
+    set(AVRO_VENDORED TRUE)
+    set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp")
+    install(TARGETS avrocpp_s
+            EXPORT iceberg_targets
+            RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
+            ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
+            LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
+
+    # TODO: add vendored ZLIB and Snappy support
+    list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB Snappy)
+  else()
+    set(AVRO_VENDORED FALSE)
+    list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro)
+  endif()
+
+  set(ICEBERG_SYSTEM_DEPENDENCIES
+      ${ICEBERG_SYSTEM_DEPENDENCIES}
+      PARENT_SCOPE)
+  set(AVRO_VENDORED
+      ${AVRO_VENDORED}
+      PARENT_SCOPE)
+endfunction()
+
+if(ICEBERG_AVRO)
+  resolve_avro_dependency()
+endif()

example/CMakeLists.txt

@@ -26,5 +26,6 @@ find_package(Iceberg CONFIG REQUIRED)
 
 add_executable(demo_example demo_example.cc)
 
-target_link_libraries(demo_example PRIVATE Iceberg::iceberg_puffin_static
-                                           Iceberg::iceberg_arrow_static)
+target_link_libraries(demo_example
+                      PRIVATE Iceberg::iceberg_puffin_static
+                              Iceberg::iceberg_arrow_static Iceberg::iceberg_avro_static)

example/demo_example.cc

@@ -20,12 +20,14 @@
 #include <iostream>
 
 #include "iceberg/arrow/demo_arrow.h"
+#include "iceberg/avro/demo_avro.h"
 #include "iceberg/demo_table.h"
 #include "iceberg/puffin/demo_puffin.h"
 
 int main() {
   std::cout << iceberg::DemoTable().print() << std::endl;
   std::cout << iceberg::puffin::DemoPuffin().print() << std::endl;
   std::cout << iceberg::arrow::DemoArrow().print() << std::endl;
+  std::cout << iceberg::avro::DemoAvro().print() << std::endl;
   return 0;
 }

src/iceberg/CMakeLists.txt

@@ -29,6 +29,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_export.h
         DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg)
 
 add_subdirectory(arrow)
+add_subdirectory(avro)
 add_subdirectory(puffin)
 
 iceberg_install_cmake_package(Iceberg iceberg_targets)

src/iceberg/avro.h

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "iceberg/iceberg_export.h"
+
+namespace iceberg {
+
+class ICEBERG_EXPORT Avro {
+ public:
+  virtual ~Avro() = default;
+  virtual std::string print() const = 0;
+};
+
+}  // namespace iceberg

src/iceberg/avro/CMakeLists.txt

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(NOT ICEBERG_AVRO)
+  return()
+endif()
+
+set(ICEBERG_AVRO_SOURCES demo_avro.cc)
+set(ICEBERG_AVRO_INCLUDES "${ICEBERG_INCLUDES}")
+
+# Libraries to link with exported libiceberg_avro.{so,a}.
+set(ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS)
+set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS)
+set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS)
+set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS)
+
+list(APPEND
+     ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
+     "$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
+)
+list(APPEND
+     ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
+     "$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
+)
+
+if(ARROW_VENDORED)
+  list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s)
+  list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s)
+else()
+  list(APPEND
+       ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS
+       "$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
+  )
+  list(APPEND
+       ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS
+       "$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
+  )
+endif()
+
+list(APPEND
+     ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:Iceberg::iceberg_static>,Iceberg::iceberg_static,Iceberg::iceberg_shared>"
+)
+list(APPEND
+     ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:Iceberg::iceberg_shared>,Iceberg::iceberg_shared,Iceberg::iceberg_static>"
+)
+
+add_iceberg_lib(iceberg_avro
+                SOURCES
+                ${ICEBERG_AVRO_SOURCES}
+                PRIVATE_INCLUDES
+                ${ICEBERG_AVRO_INCLUDES}
+                SHARED_LINK_LIBS
+                ${ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS}
+                STATIC_LINK_LIBS
+                ${ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS}
+                STATIC_INSTALL_INTERFACE_LIBS
+                ${ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS}
+                SHARED_INSTALL_INTERFACE_LIBS
+                ${ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS})
+
+iceberg_install_all_headers(iceberg/avro)
+
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_avro_export.h
+        DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg/avro)

src/iceberg/avro/demo_avro.cc

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/avro/demo_avro.h"
+
+#include <sstream>
+
+#include "avro/Compiler.hh"
+#include "avro/ValidSchema.hh"
+#include "iceberg/demo_table.h"
+
+namespace iceberg::avro {
+
+std::string DemoAvro::print() const {
+  std::string input =
+      "{\n\
+    \"type\": \"record\",\n\
+    \"name\": \"testrecord\",\n\
+    \"fields\": [\n\
+        {\n\
+            \"name\": \"testbytes\",\n\
+            \"type\": \"bytes\",\n\
+            \"default\": \"\"\n\
+        }\n\
+    ]\n\
+}\n\
+";
+
+  ::avro::ValidSchema schema = ::avro::compileJsonSchemaFromString(input);
+  std::ostringstream actual;
+  schema.toJson(actual);
+
+  return actual.str();
+}
+
+}  // namespace iceberg::avro

src/iceberg/avro/demo_avro.h

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "iceberg/avro.h"
+#include "iceberg/avro/iceberg_avro_export.h"
+
+namespace iceberg::avro {
+
+class ICEBERG_AVRO_EXPORT DemoAvro : public Avro {
+ public:
+  DemoAvro() = default;
+  ~DemoAvro() override = default;
+  std::string print() const override;
+};
+
+}  // namespace iceberg::avro