Skip to content

Commit 777ea8a

Browse files
Alex-PLACETAlex-PLACET
committed
Add integration tests
1 parent 5ad8ebb commit 777ea8a

File tree

8 files changed

+920
-0
lines changed

8 files changed

+920
-0
lines changed

‎.github/workflows/integration_tests.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Integration tests
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
push:
7+
branches: [main]
8+
9+
jobs:
10+
build_integration_container_and_run_tests:
11+
runs-on: ubuntu-22.04
12+
steps:
13+
- name: Checkout code
14+
uses: actions/checkout@v2
15+
16+
- name: Run sccache-cache
17+
uses: mozilla-actions/[email protected]
18+
19+
- name: Install dependencies
20+
run: |
21+
sudo apt-get update
22+
sudo apt-get install -y libpthread-stubs0-dev libboost-thread-dev doctest-dev
23+
24+
- name: Install specific version of tzdata
25+
run: sudo apt-get install tzdata
26+
27+
- name: Configure using CMake
28+
run: |
29+
cmake -G Ninja \
30+
-Bbuild \
31+
-DCMAKE_BUILD_TYPE:STRING=RELEASE \
32+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
33+
-DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
34+
-DSPARROW_IPC_BUILD_SHARED=ON
35+
36+
- name: Build file_to_stream target
37+
working-directory: build
38+
run: cmake --build . --config Release --target file_to_stream
39+
40+
- name: Build stream_to_file target
41+
working-directory: build
42+
run: cmake --build . --config Release --target stream_to_file
43+
44+
- name: Build Docker image
45+
run: docker build -t sparrow/integration-tests -f ci/docker/integration.dockerfile .
46+
47+
- name: Run Integration tests
48+
run: |
49+
docker run --rm \
50+
-e ARCHERY_INTEGRATION_WITH_EXTERNAL_LIBRARY=/workspace/build/bin/RELEASE/ \
51+
-e ARCHERY_INTEGRATION_EXTERNAL_LIBRARY_IPC_PRODUCER=true \
52+
-e ARCHERY_INTEGRATION_EXTERNAL_LIBRARY_IPC_CONSUMER=true \
53+
-v ${{ github.workspace }}:/workspace \
54+
-w /arrow-integration sparrow/integration-tests \
55+
"/arrow-integration/ci/scripts/integration_arrow.sh /arrow-integration /build"

‎CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ MESSAGE(STATUS "🔧 Build docs: ${SPARROW_IPC_BUILD_DOCS}")
8585
OPTION(SPARROW_IPC_BUILD_EXAMPLES "Build sparrow-ipc examples" OFF)
8686
MESSAGE(STATUS "🔧 Build examples: ${SPARROW_IPC_BUILD_EXAMPLES}")
8787

88+
OPTION(SPARROW_IPC_BUILD_INTEGRATION_TESTS "Build sparrow-ipc integration tests" OFF)
89+
MESSAGE(STATUS "🔧 Build integration tests: ${SPARROW_IPC_BUILD_INTEGRATION_TESTS}")
90+
8891
# Code coverage
8992
# =============
9093
OPTION(SPARROW_IPC_ENABLE_COVERAGE "Enable sparrow-ipc test coverage" OFF)
@@ -284,6 +287,13 @@ if(SPARROW_IPC_BUILD_EXAMPLES)
284287
add_subdirectory(examples)
285288
endif()
286289

290+
# Integration tests
291+
# =================
292+
if(SPARROW_IPC_BUILD_INTEGRATION_TESTS)
293+
message(STATUS "🔨 Create integration tests targets")
294+
add_subdirectory(integration_tests)
295+
endif()
296+
287297
# Installation
288298
# ============
289299
include(GNUInstallDirs)

‎ci/docker/integration.dockerfile‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
FROM apache/arrow-dev:amd64-conda-integration
19+
20+
ENV ARROW_USE_CCACHE=OFF \
21+
ARROW_CPP_EXE_PATH=/build/cpp/debug \
22+
BUILD_DOCS_CPP=OFF \
23+
ARROW_INTEGRATION_CPP=ON \
24+
ARROW_INTEGRATION_CSHARP=OFF \
25+
ARROW_INTEGRATION_GO=OFF \
26+
ARROW_INTEGRATION_JAVA=OFF \
27+
ARROW_INTEGRATION_JS=OFF \
28+
ARCHERY_INTEGRATION_WITH_NANOARROW="0" \
29+
ARCHERY_INTEGRATION_WITH_RUST="0"
30+
31+
RUN apt update
32+
33+
RUN apt install build-essential git -y
34+
35+
# Clone the arrow monorepo // TODO: change to the official repo
36+
RUN git clone --depth 1 --branch archery_supports_external_libraries https://github.com/Alex-PLACET/arrow.git /arrow-integration --recurse-submodules
37+
38+
# Build all the integrations
39+
RUN conda run --no-capture-output \
40+
/arrow-integration/ci/scripts/integration_arrow_build.sh \
41+
/arrow-integration \
42+
/build

‎integration_tests/CMakeLists.txt‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
cmake_minimum_required(VERSION 3.28)
2+
3+
# Create executable for file_to_stream integration test
4+
add_executable(file_to_stream file_to_stream.cpp)
5+
6+
target_link_libraries(file_to_stream
7+
PRIVATE
8+
sparrow-ipc
9+
sparrow::sparrow
10+
sparrow::json_reader
11+
)
12+
13+
set_target_properties(file_to_stream
14+
PROPERTIES
15+
CXX_STANDARD 20
16+
CXX_STANDARD_REQUIRED ON
17+
CXX_EXTENSIONS OFF
18+
)
19+
20+
target_include_directories(file_to_stream
21+
PRIVATE
22+
${CMAKE_SOURCE_DIR}/include
23+
${CMAKE_BINARY_DIR}/generated
24+
)
25+
26+
add_dependencies(file_to_stream generate_flatbuffers_headers)
27+
28+
# Create executable for stream_to_file integration test
29+
add_executable(stream_to_file stream_to_file.cpp)
30+
31+
target_link_libraries(stream_to_file
32+
PRIVATE
33+
sparrow-ipc
34+
sparrow::sparrow
35+
)
36+
37+
set_target_properties(stream_to_file
38+
PROPERTIES
39+
CXX_STANDARD 20
40+
CXX_STANDARD_REQUIRED ON
41+
CXX_EXTENSIONS OFF
42+
)
43+
44+
target_include_directories(stream_to_file
45+
PRIVATE
46+
${CMAKE_SOURCE_DIR}/include
47+
${CMAKE_BINARY_DIR}/generated
48+
)
49+
50+
add_dependencies(stream_to_file generate_flatbuffers_headers)
51+
52+
# Create test executable for integration tools
53+
add_executable(test_integration_tools main.cpp test_integration_tools.cpp)
54+
55+
target_link_libraries(test_integration_tools
56+
PRIVATE
57+
sparrow-ipc
58+
sparrow::sparrow
59+
sparrow::json_reader
60+
doctest::doctest
61+
arrow-testing-data
62+
)
63+
64+
target_compile_definitions(test_integration_tools
65+
PRIVATE
66+
INTEGRATION_TOOLS_DIR="${CMAKE_CURRENT_BINARY_DIR}"
67+
)
68+
69+
set_target_properties(test_integration_tools
70+
PROPERTIES
71+
CXX_STANDARD 20
72+
CXX_STANDARD_REQUIRED ON
73+
CXX_EXTENSIONS OFF
74+
)
75+
76+
target_include_directories(test_integration_tools
77+
PRIVATE
78+
${CMAKE_SOURCE_DIR}/include
79+
${CMAKE_BINARY_DIR}/generated
80+
)
81+
82+
add_dependencies(test_integration_tools generate_flatbuffers_headers file_to_stream stream_to_file)
83+
84+
# Register with CTest
85+
enable_testing()
86+
add_test(NAME integration_tools_test COMMAND test_integration_tools)
87+
88+
# On Windows, copy required DLLs
89+
if(WIN32)
90+
add_custom_command(
91+
TARGET file_to_stream POST_BUILD
92+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
93+
"$<TARGET_FILE:sparrow::sparrow>"
94+
"$<TARGET_FILE_DIR:file_to_stream>"
95+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
96+
"$<TARGET_FILE:sparrow-ipc>"
97+
"$<TARGET_FILE_DIR:file_to_stream>"
98+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
99+
"$<TARGET_FILE:sparrow::json_reader>"
100+
"$<TARGET_FILE_DIR:file_to_stream>"
101+
COMMENT "Copying DLLs to file_to_stream executable directory"
102+
)
103+
104+
add_custom_command(
105+
TARGET stream_to_file POST_BUILD
106+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
107+
"$<TARGET_FILE:sparrow::sparrow>"
108+
"$<TARGET_FILE_DIR:stream_to_file>"
109+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
110+
"$<TARGET_FILE:sparrow-ipc>"
111+
"$<TARGET_FILE_DIR:stream_to_file>"
112+
COMMENT "Copying DLLs to stream_to_file executable directory"
113+
)
114+
115+
add_custom_command(
116+
TARGET test_integration_tools POST_BUILD
117+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
118+
"$<TARGET_FILE:sparrow::sparrow>"
119+
"$<TARGET_FILE_DIR:test_integration_tools>"
120+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
121+
"$<TARGET_FILE:sparrow-ipc>"
122+
"$<TARGET_FILE_DIR:test_integration_tools>"
123+
COMMAND ${CMAKE_COMMAND} -E copy_if_different
124+
"$<TARGET_FILE:sparrow::json_reader>"
125+
"$<TARGET_FILE_DIR:test_integration_tools>"
126+
COMMENT "Copying DLLs to test_integration_tools executable directory"
127+
)
128+
endif()
129+
130+
set_target_properties(file_to_stream stream_to_file test_integration_tools PROPERTIES FOLDER "Integration Tests")

‎integration_tests/file_to_stream.cpp‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#include <cstdlib>
2+
#include <filesystem>
3+
#include <fstream>
4+
#include <iostream>
5+
#include <vector>
6+
7+
#include <nlohmann/json.hpp>
8+
#include <sparrow/record_batch.hpp>
9+
10+
#include "sparrow/json_reader/json_parser.hpp"
11+
12+
#include <sparrow_ipc/memory_output_stream.hpp>
13+
#include <sparrow_ipc/serializer.hpp>
14+
15+
/**
16+
* @brief Reads a JSON file containing record batches and outputs the serialized Arrow IPC stream to stdout.
17+
*
18+
* This program takes a JSON file path as a command-line argument, parses the record batches
19+
* from the JSON data, serializes them into Arrow IPC stream format, and writes the binary
20+
* stream to stdout. The output can be redirected to a file or piped to another program.
21+
*
22+
* Usage: file_to_stream <json_file_path>
23+
*
24+
* @param argc Number of command-line arguments
25+
* @param argv Array of command-line arguments
26+
* @return EXIT_SUCCESS on success, EXIT_FAILURE on error
27+
*/
28+
int main(int argc, char* argv[])
29+
{
30+
// Check command-line arguments
31+
if (argc != 2)
32+
{
33+
std::cerr << "Usage: " << argv[0] << " <json_file_path>\n";
34+
std::cerr << "Reads a JSON file and outputs the serialized Arrow IPC stream to stdout.\n";
35+
return EXIT_FAILURE;
36+
}
37+
38+
const std::filesystem::path json_path(argv[1]);
39+
40+
try
41+
{
42+
// Check if the JSON file exists
43+
if (!std::filesystem::exists(json_path))
44+
{
45+
std::cerr << "Error: File not found: " << json_path << "\n";
46+
return EXIT_FAILURE;
47+
}
48+
49+
// Open and parse the JSON file
50+
std::ifstream json_file(json_path);
51+
if (!json_file.is_open())
52+
{
53+
std::cerr << "Error: Could not open file: " << json_path << "\n";
54+
return EXIT_FAILURE;
55+
}
56+
57+
nlohmann::json json_data;
58+
try
59+
{
60+
json_data = nlohmann::json::parse(json_file);
61+
}
62+
catch (const nlohmann::json::parse_error& e)
63+
{
64+
std::cerr << "Error: Failed to parse JSON file: " << e.what() << "\n";
65+
return EXIT_FAILURE;
66+
}
67+
json_file.close();
68+
69+
// Get the number of batches
70+
if (!json_data.contains("batches") || !json_data["batches"].is_array())
71+
{
72+
std::cerr << "Error: JSON file does not contain a 'batches' array.\n";
73+
return EXIT_FAILURE;
74+
}
75+
76+
const size_t num_batches = json_data["batches"].size();
77+
78+
// Parse all record batches from JSON
79+
std::vector<sparrow::record_batch> record_batches;
80+
record_batches.reserve(num_batches);
81+
82+
for (size_t batch_idx = 0; batch_idx < num_batches; ++batch_idx)
83+
{
84+
try
85+
{
86+
record_batches.emplace_back(
87+
sparrow::json_reader::build_record_batch_from_json(json_data, batch_idx)
88+
);
89+
}
90+
catch (const std::exception& e)
91+
{
92+
std::cerr << "Error: Failed to build record batch " << batch_idx << ": " << e.what()
93+
<< "\n";
94+
return EXIT_FAILURE;
95+
}
96+
}
97+
98+
// Serialize record batches to Arrow IPC stream format
99+
std::vector<uint8_t> stream_data;
100+
sparrow_ipc::memory_output_stream stream(stream_data);
101+
sparrow_ipc::serializer serializer(stream);
102+
103+
serializer << record_batches << sparrow_ipc::end_stream;
104+
105+
// Write the binary stream to stdout
106+
std::cout.write(reinterpret_cast<const char*>(stream_data.data()), stream_data.size());
107+
std::cout.flush();
108+
109+
return EXIT_SUCCESS;
110+
}
111+
catch (const std::exception& e)
112+
{
113+
std::cerr << "Error: " << e.what() << "\n";
114+
return EXIT_FAILURE;
115+
}
116+
}

‎integration_tests/main.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
2+
#include "doctest/doctest.h"

0 commit comments

Comments
 (0)