test: add rest catalog integration test (apache#361)

Add integration testing infrastructure for the REST Catalog:
- Docker Compose setup (`docker-compose.yml`) for running a real REST catalog server
- Test utilities for process management (`cmd_util.cc/h`) - run cli commands
- Docker Compose control utilities (`docker_compose_util.cc/h`) managing containerized services
- Modified GitHub CI to support integration tests.
- These test cases (cmd / docker compose related) only work on Linux, so added conditional compilation flags for Linux-specific test utilities.

Author: HeartLinked

.github/workflows/test.yml

@@ -54,7 +54,7 @@ jobs:
         env:
           CC: gcc-14
           CXX: g++-14
-        run: ci/scripts/build_iceberg.sh $(pwd)
+        run: ci/scripts/build_iceberg.sh $(pwd) ON
       - name: Build Example
         shell: bash
         env:
@@ -110,6 +110,7 @@ jobs:
             runs-on: ubuntu-24.04
             CC: gcc-14
             CXX: g++-14
+            meson-setup-args: -Drest_integration_test=enabled
           - title: AMD64 Windows 2025
             runs-on: windows-2025
             meson-setup-args: --vsenv

CMakeLists.txt

@@ -42,6 +42,7 @@ option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
 option(ICEBERG_BUILD_TESTS "Build tests" ON)
 option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON)
 option(ICEBERG_BUILD_REST "Build rest catalog client" ON)
+option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF)
 option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
 option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
 
@@ -60,6 +61,11 @@ else()
   set(MSVC_TOOLCHAIN FALSE)
 endif()
 
+if(ICEBERG_BUILD_REST_INTEGRATION_TESTS AND WIN32)
+  set(ICEBERG_BUILD_REST_INTEGRATION_TESTS OFF)
+  message(WARNING "Cannot build rest integration test on Windows, turning it off.")
+endif()
+
 include(CMakeParseArguments)
 include(IcebergBuildUtils)
 include(IcebergSanitizer)

ci/scripts/build_iceberg.sh

@@ -21,6 +21,7 @@ set -eux
 
 source_dir=${1}
 build_dir=${1}/build
+build_rest_integration_test=${2:-OFF}
 
 mkdir ${build_dir}
 pushd ${build_dir}
@@ -34,6 +35,7 @@ CMAKE_ARGS=(
     "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
     "-DICEBERG_BUILD_STATIC=ON"
     "-DICEBERG_BUILD_SHARED=ON"
+    "-DICEBERG_BUILD_REST_INTEGRATION_TESTS=${build_rest_integration_test}"
 )
 
 if is_windows; then

meson.options

@@ -36,4 +36,12 @@ option(
     description: 'Build rest catalog client',
     value: 'enabled',
 )
+
+option(
+    'rest_integration_test',
+    type: 'feature',
+    description: 'Build integration test for rest catalog',
+    value: 'disabled',
+)
+
 option('tests', type: 'feature', description: 'Build tests', value: 'enabled')

src/iceberg/test/CMakeLists.txt

@@ -15,23 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
-fetchcontent_declare(cpp-httplib
-                     GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git
-                     GIT_TAG 89c932f313c6437c38f2982869beacc89c2f2246 #release-0.26.0
-)
-
 fetchcontent_declare(googletest
                      GIT_REPOSITORY https://github.com/google/googletest.git
                      GIT_TAG b514bdc898e2951020cbdca1304b75f5950d1f59 # release-1.15.2
                      FIND_PACKAGE_ARGS
                      NAMES
                      GTest)
 
-if(ICEBERG_BUILD_REST)
-  fetchcontent_makeavailable(cpp-httplib googletest)
-else()
-  fetchcontent_makeavailable(googletest)
-endif()
+fetchcontent_makeavailable(googletest)
 
 set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources")
 
@@ -53,11 +44,9 @@ function(add_iceberg_test test_name)
   target_sources(${test_name} PRIVATE ${ARG_SOURCES})
 
   if(ARG_USE_BUNDLE)
-    target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gtest_main
-                                               GTest::gmock)
+    target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gmock_main)
   else()
-    target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gtest_main
-                                               GTest::gmock)
+    target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gmock_main)
   endif()
 
   add_test(NAME ${test_name} COMMAND ${test_name})
@@ -173,16 +162,18 @@ if(ICEBERG_BUILD_REST)
     add_executable(${test_name})
     target_include_directories(${test_name} PRIVATE "${CMAKE_BINARY_DIR}/iceberg/test/")
     target_sources(${test_name} PRIVATE ${ARG_SOURCES})
-    target_link_libraries(${test_name} PRIVATE GTest::gtest_main GTest::gmock
-                                               iceberg_rest_static)
+    target_link_libraries(${test_name} PRIVATE GTest::gmock_main iceberg_rest_static)
     add_test(NAME ${test_name} COMMAND ${test_name})
   endfunction()
 
-  add_rest_iceberg_test(rest_catalog_test
-                        SOURCES
-                        rest_catalog_test.cc
-                        rest_json_internal_test.cc
+  add_rest_iceberg_test(rest_catalog_test SOURCES rest_json_internal_test.cc
                         rest_util_test.cc)
 
-  target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR})
+  if(ICEBERG_BUILD_REST_INTEGRATION_TESTS)
+    add_rest_iceberg_test(rest_catalog_integration_test
+                          SOURCES
+                          rest_catalog_test.cc
+                          util/cmd_util.cc
+                          util/docker_compose_util.cc)
+  endif()
 endif()

src/iceberg/test/meson.build

@@ -91,17 +91,28 @@ iceberg_tests = {
 }
 
 if get_option('rest').enabled()
-    cpp_httplib_dep = dependency('cpp-httplib')
     iceberg_tests += {
         'rest_catalog_test': {
-            'sources': files(
-                'rest_catalog_test.cc',
-                'rest_json_internal_test.cc',
-                'rest_util_test.cc',
-            ),
-            'dependencies': [iceberg_rest_dep, cpp_httplib_dep],
+            'sources': files('rest_json_internal_test.cc', 'rest_util_test.cc'),
+            'dependencies': [iceberg_rest_dep],
         },
     }
+    if get_option('rest_integration_test').enabled()
+        if host_machine.system() == 'windows'
+            warning('Cannot build rest integration test on Windows, skipping.')
+        else
+            iceberg_tests += {
+                'rest_integration_test': {
+                    'sources': files(
+                        'rest_catalog_test.cc',
+                        'util/cmd_util.cc',
+                        'util/docker_compose_util.cc',
+                    ),
+                    'dependencies': [iceberg_rest_dep],
+                },
+            }
+        endif
+    endif
 endif
 
 foreach test_name, values : iceberg_tests

subprojects/cpp-httplib.wrap …/iceberg-rest-fixture/docker-compose.ymlsubprojects/cpp-httplib.wrap renamed to src/iceberg/test/resources/iceberg-rest-fixture/docker-compose.yml subprojects/cpp-httplib.wrap renamed to src/iceberg/test/resources/iceberg-rest-fixture/docker-compose.yml

@@ -15,13 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-[wrap-file]
-directory = cpp-httplib-0.26.0
-source_url = https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.26.0.tar.gz
-source_filename = cpp-httplib-0.26.0.tar.gz
-source_hash = a66f908f50ccb119769adce44fe1eac75f81b6ffab7c4ac0211bb663ffeb2688
-source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/cpp-httplib_0.26.0-1/cpp-httplib-0.26.0.tar.gz
-wrapdb_version = 0.26.0-1
-
-[provide]
-dependency_names = cpp-httplib
+services:
+  rest:
+    image: apache/iceberg-rest-fixture:latest
+    environment:
+      - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog
+      - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory
+      - CATALOG_WAREHOUSE=file:///tmp/iceberg_warehouse
+    ports:
+      - "8181:8181"