Skip to content

Commit 6d85f52

Browse files
Hind-MHind-M
authored
Generate headers from schema files (#1)
* Generate headers with flatc from apache arrow schemas * Link sparrow * Add dev environment file * Ignore all build dirs * Code review: put fbs schemas and generated headers in build * Add test and more adjustments * Add linux workflow * Change workflow trigger
1 parent 77065fa commit 6d85f52

File tree

7 files changed

+186
-0
lines changed

7 files changed

+186
-0
lines changed

‎.github/workflows/linux.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Build and test sparrow-ipc
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
push:
7+
branches: [main]
8+
9+
defaults:
10+
run:
11+
shell: bash -l -eo pipefail {0}
12+
13+
jobs:
14+
build:
15+
runs-on: ubuntu-latest
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v4
19+
- name: Create build environment
20+
uses: mamba-org/setup-micromamba@v2
21+
with:
22+
environment-file: ./environment-dev.yml
23+
environment-name: build_env
24+
cache-environment: true
25+
- name: Build sparrow-ipc
26+
run: |
27+
cmake -B build/ -G Ninja \
28+
-DCMAKE_BUILD_TYPE=Release \
29+
-DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
30+
-DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
31+
-DBUILD_TESTS=ON
32+
cmake --build build/ --parallel
33+
- name: Run tests
34+
run: |
35+
cd build
36+
ctest --output-on-failure

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,6 @@
3030
*.exe
3131
*.out
3232
*.app
33+
34+
# Build directories
35+
/build*/

‎CMakeLists.txt‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
cmake_minimum_required(VERSION 3.28)
2+
3+
project(sparrow-ipc CXX)
4+
5+
set(CMAKE_CXX_STANDARD 20)
6+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
7+
set(CMAKE_CXX_SCAN_FOR_MODULES OFF)
8+
9+
# Build options
10+
# =============
11+
OPTION(BUILD_TESTS "Build sparrow-ipc test suite" OFF)
12+
MESSAGE(STATUS "🔧 Build tests: ${BUILD_TESTS}")
13+
14+
set(SCHEMA_DIR ${CMAKE_BINARY_DIR}/format)
15+
set(FLATBUFFERS_GENERATED_DIR ${CMAKE_BINARY_DIR}/generated)
16+
17+
find_program(FLATC_EXECUTABLE flatc)
18+
19+
if(NOT FLATC_EXECUTABLE)
20+
message(FATAL_ERROR "flatc not found. Please install Flatbuffers.")
21+
endif()
22+
23+
# Fetch schemas from apache arrow
24+
set(SCHEMA_URLS
25+
"https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/File.fbs"
26+
"https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/Message.fbs"
27+
"https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/Schema.fbs"
28+
"https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/Tensor.fbs"
29+
"https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/SparseTensor.fbs"
30+
# TODO what about feather.fbs?
31+
)
32+
33+
file(MAKE_DIRECTORY ${SCHEMA_DIR})
34+
35+
# Download schemas
36+
set(FLATBUFFERS_SCHEMAS "")
37+
foreach(url IN LISTS SCHEMA_URLS)
38+
get_filename_component(filename ${url} NAME)
39+
message(STATUS "Downloading schema: ${url}")
40+
file(DOWNLOAD ${url} ${SCHEMA_DIR}/${filename}
41+
STATUS status
42+
SHOW_PROGRESS)
43+
list(APPEND FLATBUFFERS_SCHEMAS ${SCHEMA_DIR}/${filename})
44+
endforeach()
45+
46+
# Generate Flatbuffers C++ headers from the schemas
47+
file(MAKE_DIRECTORY ${FLATBUFFERS_GENERATED_DIR})
48+
49+
# Generate output files list
50+
set(FLATBUFFERS_GENERATED_HEADERS "")
51+
foreach(fbs_file IN LISTS FLATBUFFERS_SCHEMAS)
52+
# Generate the corresponding header file name
53+
get_filename_component(header_name ${fbs_file} NAME_WE)
54+
list(APPEND FLATBUFFERS_GENERATED_HEADERS "${FLATBUFFERS_GENERATED_DIR}/${header_name}_generated.h")
55+
endforeach()
56+
57+
add_custom_command(
58+
OUTPUT ${FLATBUFFERS_GENERATED_HEADERS}
59+
COMMAND ${FLATC_EXECUTABLE} --cpp -o ${FLATBUFFERS_GENERATED_DIR} --cpp-std c++17 --scoped-enums ${FLATBUFFERS_SCHEMAS}
60+
DEPENDS ${FLATBUFFERS_SCHEMAS}
61+
COMMENT "Generating FlatBuffers C++ headers from schemas"
62+
)
63+
64+
add_custom_target(generate_flatbuffers_headers
65+
DEPENDS ${FLATBUFFERS_GENERATED_HEADERS}
66+
)
67+
68+
# Interface target for generated headers
69+
add_library(flatbuffers_interface INTERFACE)
70+
target_include_directories(flatbuffers_interface INTERFACE ${FLATBUFFERS_GENERATED_DIR})
71+
add_dependencies(flatbuffers_interface generate_flatbuffers_headers)
72+
73+
find_package(FlatBuffers CONFIG REQUIRED)
74+
find_package(sparrow CONFIG REQUIRED)
75+
76+
# TODO Handle shared/static build later (after more code is available)
77+
add_library(sparrow-ipc STATIC src/sparrow-ipc.cpp)
78+
target_link_libraries(sparrow-ipc PRIVATE flatbuffers_interface flatbuffers::flatbuffers sparrow)
79+
80+
add_dependencies(sparrow-ipc generate_flatbuffers_headers)
81+
82+
if(BUILD_TESTS)
83+
message(STATUS "🧪 Create tests targets")
84+
enable_testing()
85+
add_subdirectory(tests)
86+
endif()

‎environment-dev.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
name: sparrow-ipc
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
# Build dependencies
6+
- cmake
7+
- make
8+
- ninja
9+
- cxx-compiler
10+
# Libraries dependencies
11+
- flatbuffers
12+
- sparrow
13+
# Tests
14+
- doctest

‎src/sparrow-ipc.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#include "sparrow/sparrow.hpp"
2+
3+
#include "../generated/Schema_generated.h"
4+

‎tests/CMakeLists.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
cmake_minimum_required(VERSION 3.28)
2+
3+
find_package(doctest CONFIG REQUIRED)
4+
5+
set(test_target "test_sparrow_ipc_lib")
6+
7+
add_executable(${test_target} test.cpp)
8+
target_link_libraries(${test_target}
9+
PRIVATE
10+
sparrow-ipc
11+
doctest::doctest
12+
)
13+
target_include_directories(${test_target}
14+
PRIVATE
15+
${CMAKE_BINARY_DIR}/generated
16+
)
17+
add_dependencies(${test_target} generate_flatbuffers_headers)
18+
add_test(NAME sparrow-ipc-tests COMMAND ${test_target})

‎tests/test.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
2+
3+
#include "sparrow/sparrow.hpp"
4+
#include "doctest/doctest.h"
5+
6+
#include "../generated/Schema_generated.h"
7+
8+
// NOTE this is just testing sparrow internals usability,
9+
// for now we are not testing anything with serialization/deserialization
10+
TEST_CASE("Use sparrow primitive_array")
11+
{
12+
namespace sp = sparrow;
13+
14+
sp::primitive_array<int> ar = { 1, 3, 5, 7, 9 };
15+
CHECK_EQ(ar.size(), 5);
16+
17+
auto [arrow_array, arrow_schema] = sp::extract_arrow_structures(std::move(ar));
18+
CHECK_EQ(arrow_array.length, 5);
19+
20+
// Serialize
21+
// Deserialize
22+
23+
arrow_array.release(&arrow_array);
24+
arrow_schema.release(&arrow_schema);
25+
}

0 commit comments

Comments
 (0)