From e5133a79ff7f1b91e2ee1cc7340c36100bbcc1f8 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 16:31:14 -0400 Subject: [PATCH 01/19] feat: Meson build support for IPC --- dev/release/rat_exclude_files.txt | 1 + meson.build | 11 +++++ meson.options | 2 + src/apps/dump_stream.c | 2 - src/nanoarrow/meson.build | 53 +++++++++++++++++++++ src/nanoarrow/nanoarrow.hpp | 2 +- src/nanoarrow/nanoarrow_ipc_decoder_test.cc | 11 ++--- src/nanoarrow/nanoarrow_testing_test.cc | 2 +- subprojects/flatcc.wrap | 23 +++++++++ subprojects/zlib.wrap | 13 +++++ 10 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 subprojects/flatcc.wrap create mode 100644 subprojects/zlib.wrap diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 26a8b0f61..128077939 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -18,3 +18,4 @@ python/src/nanoarrow/dlpack_abi.h subprojects/google-benchmark.wrap subprojects/gtest.wrap subprojects/nlohmann_json.wrap +subprojects/zlib.wrap diff --git a/meson.build b/meson.build index c56bf4bb1..bd918d011 100644 --- a/meson.build +++ b/meson.build @@ -34,3 +34,14 @@ subdir('src/nanoarrow') if get_option('benchmarks') subdir('dev/benchmarks') endif + + +if get_option('apps') + if get_option('ipc') + executable( + 'dump_stream', + 'src/apps/dump_stream.c', + dependencies: [nanoarrow_dep, nanoarrow_ipc_dep] + ) + endif +endif diff --git a/meson.options b/meson.options index 73cbccd02..b91509820 100644 --- a/meson.options +++ b/meson.options @@ -17,6 +17,8 @@ option('tests', type: 'boolean', description: 'Build tests', value: false) option('benchmarks', type: 'boolean', description: 'Build benchmarks', value: false) +option('apps', type: 'boolean', description: 'Build utility applications', value: false) +option('ipc', type: 'boolean', description: 'Build IPC libraries', value: false) option('integration_tests', type: 'boolean', description: 'Build cross-implementation Arrow integration tests', value: false) diff --git a/src/apps/dump_stream.c b/src/apps/dump_stream.c index f26e566fe..1c331d729 100644 --- a/src/apps/dump_stream.c +++ b/src/apps/dump_stream.c @@ -23,8 +23,6 @@ void dump_schema_to_stdout(struct ArrowSchema* schema, int level, char* buf, int buf_size) { - int n_chars = ArrowSchemaToString(schema, buf, buf_size, 0); - for (int i = 0; i < level; i++) { fprintf(stdout, " "); } diff --git a/src/nanoarrow/meson.build b/src/nanoarrow/meson.build index f6f257161..547f63bef 100644 --- a/src/nanoarrow/meson.build +++ b/src/nanoarrow/meson.build @@ -64,6 +64,25 @@ incdir = include_directories('..') nanoarrow_dep = declare_dependency(include_directories: [curdir, incdir], link_with: nanoarrow_lib) +if get_option('ipc') + cmake = import('cmake') + cmake_opts = cmake.subproject_options() + cmake_opts.add_cmake_defines({'CMAKE_POSITION_INDEPENDENT_CODE': true}) + flatcc_subproj = cmake.subproject('flatcc', options: cmake_opts) + flatcc_dep = flatcc_subproj.dependency('flatccrt') + + nanoarrow_ipc_lib = build_target( + 'nanoarrow_ipc', + 'nanoarrow_ipc_decoder.c', + 'nanoarrow_ipc_reader.c', + dependencies: [nanoarrow_dep, flatcc_dep], + install: true, + target_type: libtype, + ) + nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir], + link_with: nanoarrow_ipc_lib) +endif + if get_option('tests') or get_option('integration_tests') nlohmann_json_dep = dependency('nlohmann_json') @@ -147,4 +166,38 @@ if get_option('tests') include_directories: incdir) test('c_data_integration test', c_data_integration_test) + if get_option('ipc') + zlib_dep = dependency('zlib') + ipc_test_files = { + 'nanoarrow-ipc-decoder': { + 'deps': [nanoarrow_dep, flatcc_dep, arrow_dep, gtest_dep], + }, + 'nanoarrow-ipc-reader': { + 'deps': [nanoarrow_dep, flatcc_dep, arrow_dep, gtest_dep], + }, + 'nanoarrow-ipc-files': { + 'deps': [ + nanoarrow_dep, + flatcc_dep, + zlib_dep, + arrow_dep, + gtest_dep, + nlohmann_json_dep + ], + }, + 'nanoarrow-ipc-hpp': { + 'deps': [nanoarrow_dep, flatcc_dep, gtest_dep], + }, + } + + foreach name, config : ipc_test_files + exc = executable( + name + '-test', + name.replace('-', '_') + '_test.cc', + link_with: nanoarrow_ipc_lib, + dependencies: config['deps'] + ) + test(name, exc) + endforeach + endif endif diff --git a/src/nanoarrow/nanoarrow.hpp b/src/nanoarrow/nanoarrow.hpp index 0de2371b4..5f8aabbac 100644 --- a/src/nanoarrow/nanoarrow.hpp +++ b/src/nanoarrow/nanoarrow.hpp @@ -864,8 +864,8 @@ class ViewArrayStream { }; internal::InputRange range_; - ArrowError* error_; ArrowErrorCode* code_; + ArrowError* error_; ArrowError internal_error_ = {}; ArrowErrorCode internal_code_; bool code_was_accessed_ = false; diff --git a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc index db098dc76..37edab2d8 100644 --- a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc +++ b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc @@ -54,7 +54,6 @@ struct ArrowIpcDecoderPrivate { static enum ArrowIpcEndianness ArrowIpcSystemEndianness(void) { uint32_t check = 1; char first_byte; - enum ArrowIpcEndianness system_endianness; memcpy(&first_byte, &check, sizeof(char)); if (first_byte) { return NANOARROW_IPC_ENDIANNESS_LITTLE; @@ -501,8 +500,6 @@ TEST(NanoarrowIpcTest, NanoarrowIpcDecodeSimpleRecordBatchFromShared) { data.size_bytes = sizeof(kSimpleRecordBatch); ArrowIpcDecoderInit(&decoder); - auto decoder_private = - reinterpret_cast(decoder.private_data); ASSERT_EQ(ArrowIpcDecoderSetSchema(&decoder, &schema, nullptr), NANOARROW_OK); EXPECT_EQ(ArrowIpcDecoderDecodeHeader(&decoder, data, &error), NANOARROW_OK); @@ -602,8 +599,10 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSharedBufferThreadSafeDecode) { std::thread threads[10]; for (int i = 0; i < 10; i++) { threads[i] = std::thread([&arrays, i, &one_two_three_le] { - memcmp(arrays[i].children[0]->buffers[1], one_two_three_le, - sizeof(one_two_three_le)); + auto result = memcmp(arrays[i].children[0]->buffers[1], one_two_three_le, + sizeof(one_two_three_le)); + // discard result to silence -Wunused-value + (void)result; ArrowArrayRelease(arrays + i); }); } @@ -791,7 +790,7 @@ TEST_P(ArrowTypeIdParameterizedTestFixture, NanoarrowIpcDecodeSwapEndian) { // Make a data buffer long enough for 10 Decimal256s with a pattern // where an endian swap isn't silently the same value (e.g., 0s) uint8_t data_buffer[32 * 10]; - for (int64_t i = 0; i < sizeof(data_buffer); i++) { + for (size_t i = 0; i < sizeof(data_buffer); i++) { data_buffer[i] = i % 256; } diff --git a/src/nanoarrow/nanoarrow_testing_test.cc b/src/nanoarrow/nanoarrow_testing_test.cc index 884210a67..ebd1853a5 100644 --- a/src/nanoarrow/nanoarrow_testing_test.cc +++ b/src/nanoarrow/nanoarrow_testing_test.cc @@ -470,7 +470,7 @@ TEST(NanoarrowTestingTest, NanoarrowTestingTestFieldMetadata) { NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema, "\0\0\0\0")); return NANOARROW_OK; }, - [](ArrowArray* array) { return NANOARROW_OK; }, &WriteFieldJSON, + [](ArrowArray*) { return NANOARROW_OK; }, &WriteFieldJSON, R"({"name": null, "nullable": true, "type": {"name": "null"}, "children": []})", [](TestingJSONWriter& writer) { writer.set_include_metadata(false); }); } diff --git a/subprojects/flatcc.wrap b/subprojects/flatcc.wrap new file mode 100644 index 000000000..cd1354cf5 --- /dev/null +++ b/subprojects/flatcc.wrap @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-git] +method = cmake +url = https://github.com/dvidelabs/flatcc +revision = v0.6.1 +depth = 1 +clone-recursive = true diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap new file mode 100644 index 000000000..aa14de177 --- /dev/null +++ b/subprojects/zlib.wrap @@ -0,0 +1,13 @@ +[wrap-file] +directory = zlib-1.3.1 +source_url = http://zlib.net/fossils/zlib-1.3.1.tar.gz +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zlib_1.3.1-1/zlib-1.3.1.tar.gz +source_filename = zlib-1.3.1.tar.gz +source_hash = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 +patch_filename = zlib_1.3.1-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/zlib_1.3.1-1/get_patch +patch_hash = e79b98eb24a75392009cec6f99ca5cdca9881ff20bfa174e8b8926d5c7a47095 +wrapdb_version = 1.3.1-1 + +[provide] +zlib = zlib_dep From deb2f21c6a70f5878868dcb6122568bdb7412ce3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 17:40:55 -0400 Subject: [PATCH 02/19] print-errorlogs --- ci/scripts/build-with-meson.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 41592121a..bd7e59c40 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -68,12 +68,12 @@ function main() { show_header "Run test suite" meson configure -Dtests=true -Db_coverage=true meson compile - meson test --wrap valgrind + meson test --wrap valgrind --print-errorlogs show_header "Run benchmarks" meson configure -Dbenchmarks=true meson compile - meson test --benchmark + meson test --benchmark --print-errorlogs show_header "Generate coverage reports" ninja coverage From 025098976fb7939569b5eb700b7294cd9efb03b0 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 19:58:24 -0400 Subject: [PATCH 03/19] Try track-origins=yes --- ci/scripts/build-with-meson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index bd7e59c40..487ecc5bd 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -68,7 +68,7 @@ function main() { show_header "Run test suite" meson configure -Dtests=true -Db_coverage=true meson compile - meson test --wrap valgrind --print-errorlogs + meson test --wrap='valgrind --track-origins=yes' --print-errorlogs show_header "Run benchmarks" meson configure -Dbenchmarks=true From 668e110220322f31968b5932362d46e6de4e7a4a Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:05:25 -0400 Subject: [PATCH 04/19] debug build --- ci/scripts/build-with-meson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 487ecc5bd..45995203a 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -66,7 +66,7 @@ function main() { pushd "${SANDBOX_DIR}" show_header "Run test suite" - meson configure -Dtests=true -Db_coverage=true + meson configure -Dtests=true -Db_coverage=true -Dbuildtype=debug meson compile meson test --wrap='valgrind --track-origins=yes' --print-errorlogs From a6a7ea9b44534bba5c5d0bffa135259c56c09cd3 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:09:35 -0400 Subject: [PATCH 05/19] remove debug --- ci/scripts/build-with-meson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 45995203a..487ecc5bd 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -66,7 +66,7 @@ function main() { pushd "${SANDBOX_DIR}" show_header "Run test suite" - meson configure -Dtests=true -Db_coverage=true -Dbuildtype=debug + meson configure -Dtests=true -Db_coverage=true meson compile meson test --wrap='valgrind --track-origins=yes' --print-errorlogs From a3d1812671514519cdb1830b1f57bef51c533f95 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:10:07 -0400 Subject: [PATCH 06/19] debugoptimized --- ci/scripts/build-with-meson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 487ecc5bd..3e9e67399 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -66,7 +66,7 @@ function main() { pushd "${SANDBOX_DIR}" show_header "Run test suite" - meson configure -Dtests=true -Db_coverage=true + meson configure -Dtests=true -Db_coverage=true -Dbuildtype=debugoptimized meson compile meson test --wrap='valgrind --track-origins=yes' --print-errorlogs From b5a93707b7e5adfacbc019d1af20f403d6b18882 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:15:39 -0400 Subject: [PATCH 07/19] remove debugoptimized --- ci/scripts/build-with-meson.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 3e9e67399..487ecc5bd 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -66,7 +66,7 @@ function main() { pushd "${SANDBOX_DIR}" show_header "Run test suite" - meson configure -Dtests=true -Db_coverage=true -Dbuildtype=debugoptimized + meson configure -Dtests=true -Db_coverage=true meson compile meson test --wrap='valgrind --track-origins=yes' --print-errorlogs From 61516ccbc886f3eb1b705bbf8547bfbd10790200 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:31:26 -0400 Subject: [PATCH 08/19] test code removal --- dist/nanoarrow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/nanoarrow.c b/dist/nanoarrow.c index 37d143057..461b35356 100644 --- a/dist/nanoarrow.c +++ b/dist/nanoarrow.c @@ -2552,7 +2552,7 @@ ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, // was added and C# as later discovered). Only do this fix if we can assume // CPU data access. if (validation_level >= NANOARROW_VALIDATION_LEVEL_DEFAULT) { - NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinalizeBuffers(array), error); + //NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinalizeBuffers(array), error); } // Make sure the value we get with array->buffers[i] is set to the actual From e6f72df37d7cb47049ecdc80f8c9fe7567b81c0e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:35:14 -0400 Subject: [PATCH 09/19] Comment out flush --- dist/nanoarrow.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/nanoarrow.c b/dist/nanoarrow.c index 461b35356..fb153f690 100644 --- a/dist/nanoarrow.c +++ b/dist/nanoarrow.c @@ -2552,12 +2552,12 @@ ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, // was added and C# as later discovered). Only do this fix if we can assume // CPU data access. if (validation_level >= NANOARROW_VALIDATION_LEVEL_DEFAULT) { - //NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinalizeBuffers(array), error); + NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowArrayFinalizeBuffers(array), error); } // Make sure the value we get with array->buffers[i] is set to the actual // pointer (which may have changed from the original due to reallocation) - ArrowArrayFlushInternalPointers(array); + //ArrowArrayFlushInternalPointers(array); if (validation_level == NANOARROW_VALIDATION_LEVEL_NONE) { return NANOARROW_OK; From 702da7b6fca6bff1a802bbe026f1bd8839efc66f Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:44:16 -0400 Subject: [PATCH 10/19] try something else --- dist/nanoarrow.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dist/nanoarrow.c b/dist/nanoarrow.c index fb153f690..084c28a07 100644 --- a/dist/nanoarrow.c +++ b/dist/nanoarrow.c @@ -2557,7 +2557,7 @@ ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, // Make sure the value we get with array->buffers[i] is set to the actual // pointer (which may have changed from the original due to reallocation) - //ArrowArrayFlushInternalPointers(array); + ArrowArrayFlushInternalPointers(array); if (validation_level == NANOARROW_VALIDATION_LEVEL_NONE) { return NANOARROW_OK; @@ -3359,10 +3359,10 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - ArrowErrorSet(error, - "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", - (long)last_run_end, (long)(array_view->offset + array_view->length), - (long)array_view->offset, (long)array_view->length); + //ArrowErrorSet(error, + // "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", + // (long)last_run_end, (long)(array_view->offset + array_view->length), + // (long)array_view->offset, (long)array_view->length); return EINVAL; } } From 262a9733349710f5f8f38a089e03d65a30153239 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:49:44 -0400 Subject: [PATCH 11/19] Try other error --- dist/nanoarrow.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dist/nanoarrow.c b/dist/nanoarrow.c index 084c28a07..8908557d1 100644 --- a/dist/nanoarrow.c +++ b/dist/nanoarrow.c @@ -3349,20 +3349,20 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, for (int64_t i = 1; i < run_ends_view->length; i++) { const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i); if (run_end <= last_run_end) { - ArrowErrorSet(error, - "Every run end must be strictly greater than the previous run end, " - "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", - (long)i, (long)run_end, (long)i - 1, (long)last_run_end); + //ArrowErrorSet(error, + // "Every run end must be strictly greater than the previous run end, " + // "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", + // (long)i, (long)run_end, (long)i - 1, (long)last_run_end); return EINVAL; } last_run_end = run_end; } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - //ArrowErrorSet(error, - // "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", - // (long)last_run_end, (long)(array_view->offset + array_view->length), - // (long)array_view->offset, (long)array_view->length); + ArrowErrorSet(error, + "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", + (long)last_run_end, (long)(array_view->offset + array_view->length), + (long)array_view->offset, (long)array_view->length); return EINVAL; } } From 1de36d2ea5a53a366faff8cae45600f5e310fb8e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:53:18 -0400 Subject: [PATCH 12/19] Uncomment other error --- dist/nanoarrow.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dist/nanoarrow.c b/dist/nanoarrow.c index 8908557d1..354de4232 100644 --- a/dist/nanoarrow.c +++ b/dist/nanoarrow.c @@ -3359,10 +3359,10 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - ArrowErrorSet(error, - "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", - (long)last_run_end, (long)(array_view->offset + array_view->length), - (long)array_view->offset, (long)array_view->length); + //ArrowErrorSet(error, + // "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", + // (long)last_run_end, (long)(array_view->offset + array_view->length), + // (long)array_view->offset, (long)array_view->length); return EINVAL; } } From aeea2c28886cbc89685f5915ae86c45bfc7328a8 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 20:55:17 -0400 Subject: [PATCH 13/19] try to src --- dist/nanoarrow.c | 16 ++++++++-------- src/nanoarrow/array.c | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dist/nanoarrow.c b/dist/nanoarrow.c index 354de4232..37d143057 100644 --- a/dist/nanoarrow.c +++ b/dist/nanoarrow.c @@ -3349,20 +3349,20 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, for (int64_t i = 1; i < run_ends_view->length; i++) { const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i); if (run_end <= last_run_end) { - //ArrowErrorSet(error, - // "Every run end must be strictly greater than the previous run end, " - // "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", - // (long)i, (long)run_end, (long)i - 1, (long)last_run_end); + ArrowErrorSet(error, + "Every run end must be strictly greater than the previous run end, " + "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", + (long)i, (long)run_end, (long)i - 1, (long)last_run_end); return EINVAL; } last_run_end = run_end; } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - //ArrowErrorSet(error, - // "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", - // (long)last_run_end, (long)(array_view->offset + array_view->length), - // (long)array_view->offset, (long)array_view->length); + ArrowErrorSet(error, + "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", + (long)last_run_end, (long)(array_view->offset + array_view->length), + (long)array_view->offset, (long)array_view->length); return EINVAL; } } diff --git a/src/nanoarrow/array.c b/src/nanoarrow/array.c index 36df998f2..01212f3e3 100644 --- a/src/nanoarrow/array.c +++ b/src/nanoarrow/array.c @@ -1250,10 +1250,10 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, for (int64_t i = 1; i < run_ends_view->length; i++) { const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i); if (run_end <= last_run_end) { - ArrowErrorSet(error, - "Every run end must be strictly greater than the previous run end, " - "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", - (long)i, (long)run_end, (long)i - 1, (long)last_run_end); + // ArrowErrorSet(error, + // "Every run end must be strictly greater than the previous run + // end, " "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", + // (long)i, (long)run_end, (long)i - 1, (long)last_run_end); return EINVAL; } last_run_end = run_end; From f5f8001e8c84967fa5b0cefc5005d556930b5a56 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 21:07:40 -0400 Subject: [PATCH 14/19] more comments --- src/nanoarrow/array.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/nanoarrow/array.c b/src/nanoarrow/array.c index 01212f3e3..42f30ec40 100644 --- a/src/nanoarrow/array.c +++ b/src/nanoarrow/array.c @@ -1260,10 +1260,11 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - ArrowErrorSet(error, - "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", - (long)last_run_end, (long)(array_view->offset + array_view->length), - (long)array_view->offset, (long)array_view->length); + // ArrowErrorSet(error, + // "Last run end is %ld but it should >= %ld (offset: %ld, length: + // %ld)", (long)last_run_end, (long)(array_view->offset + + // array_view->length), (long)array_view->offset, + // (long)array_view->length); return EINVAL; } } From df35957307c03b9d54c0131afea50e33008c7f05 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 21:12:31 -0400 Subject: [PATCH 15/19] different message --- src/nanoarrow/array.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/nanoarrow/array.c b/src/nanoarrow/array.c index 42f30ec40..7bbd350d1 100644 --- a/src/nanoarrow/array.c +++ b/src/nanoarrow/array.c @@ -1250,21 +1250,17 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, for (int64_t i = 1; i < run_ends_view->length; i++) { const int64_t run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, i); if (run_end <= last_run_end) { - // ArrowErrorSet(error, - // "Every run end must be strictly greater than the previous run - // end, " "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", - // (long)i, (long)run_end, (long)i - 1, (long)last_run_end); + ArrowErrorSet(error, + "Every run end must be strictly greater than the previous run end, " + "but run_ends[%ld] is %ld and run_ends[%ld] is %ld", + (long)i, (long)run_end, (long)i - 1, (long)last_run_end); return EINVAL; } last_run_end = run_end; } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - // ArrowErrorSet(error, - // "Last run end is %ld but it should >= %ld (offset: %ld, length: - // %ld)", (long)last_run_end, (long)(array_view->offset + - // array_view->length), (long)array_view->offset, - // (long)array_view->length); + ArrowErrorSet(error, "this isn't working"); return EINVAL; } } From 574bea9e86bc3b80b9c2a331a5a72f5ed8a3dfc5 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 10 Jun 2024 21:18:10 -0400 Subject: [PATCH 16/19] revert --- src/nanoarrow/array.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nanoarrow/array.c b/src/nanoarrow/array.c index 7bbd350d1..36df998f2 100644 --- a/src/nanoarrow/array.c +++ b/src/nanoarrow/array.c @@ -1260,7 +1260,10 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, } last_run_end = ArrowArrayViewGetIntUnsafe(run_ends_view, run_ends_view->length - 1); if (last_run_end < (array_view->offset + array_view->length)) { - ArrowErrorSet(error, "this isn't working"); + ArrowErrorSet(error, + "Last run end is %ld but it should >= %ld (offset: %ld, length: %ld)", + (long)last_run_end, (long)(array_view->offset + array_view->length), + (long)array_view->offset, (long)array_view->length); return EINVAL; } } From 859a3f2c551d2ef549e9fddf3a17cf601ff5a527 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 11 Jun 2024 11:24:44 -0400 Subject: [PATCH 17/19] feedback --- ci/scripts/build-with-meson.sh | 2 +- src/apps/dump_stream.c | 2 ++ src/nanoarrow/meson.build | 13 ++++++------- src/nanoarrow/nanoarrow_ipc_decoder_test.cc | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ci/scripts/build-with-meson.sh b/ci/scripts/build-with-meson.sh index 487ecc5bd..cd76c6db8 100755 --- a/ci/scripts/build-with-meson.sh +++ b/ci/scripts/build-with-meson.sh @@ -68,7 +68,7 @@ function main() { show_header "Run test suite" meson configure -Dtests=true -Db_coverage=true meson compile - meson test --wrap='valgrind --track-origins=yes' --print-errorlogs + meson test --wrap='valgrind --track-origins=yes --leak-check=full' --print-errorlogs show_header "Run benchmarks" meson configure -Dbenchmarks=true diff --git a/src/apps/dump_stream.c b/src/apps/dump_stream.c index 1c331d729..ba1baec4d 100644 --- a/src/apps/dump_stream.c +++ b/src/apps/dump_stream.c @@ -23,6 +23,8 @@ void dump_schema_to_stdout(struct ArrowSchema* schema, int level, char* buf, int buf_size) { + ArrowSchemaToString(schema, buf, buf_size, 0); + for (int i = 0; i < level; i++) { fprintf(stdout, " "); } diff --git a/src/nanoarrow/meson.build b/src/nanoarrow/meson.build index 547f63bef..ccf59c990 100644 --- a/src/nanoarrow/meson.build +++ b/src/nanoarrow/meson.build @@ -80,7 +80,8 @@ if get_option('ipc') target_type: libtype, ) nanoarrow_ipc_dep = declare_dependency(include_directories: [incdir], - link_with: nanoarrow_ipc_lib) + link_with: nanoarrow_ipc_lib, + dependencies: [nanoarrow_dep, flatcc_dep]) endif if get_option('tests') or get_option('integration_tests') @@ -170,15 +171,14 @@ if get_option('tests') zlib_dep = dependency('zlib') ipc_test_files = { 'nanoarrow-ipc-decoder': { - 'deps': [nanoarrow_dep, flatcc_dep, arrow_dep, gtest_dep], + 'deps': [nanoarrow_ipc_dep, arrow_dep, gtest_dep], }, 'nanoarrow-ipc-reader': { - 'deps': [nanoarrow_dep, flatcc_dep, arrow_dep, gtest_dep], + 'deps': [nanoarrow_ipc_dep, arrow_dep, gtest_dep], }, 'nanoarrow-ipc-files': { 'deps': [ - nanoarrow_dep, - flatcc_dep, + nanoarrow_ipc_dep, zlib_dep, arrow_dep, gtest_dep, @@ -186,7 +186,7 @@ if get_option('tests') ], }, 'nanoarrow-ipc-hpp': { - 'deps': [nanoarrow_dep, flatcc_dep, gtest_dep], + 'deps': [nanoarrow_ipc_dep, gtest_dep], }, } @@ -194,7 +194,6 @@ if get_option('tests') exc = executable( name + '-test', name.replace('-', '_') + '_test.cc', - link_with: nanoarrow_ipc_lib, dependencies: config['deps'] ) test(name, exc) diff --git a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc index 37edab2d8..4141b9b08 100644 --- a/src/nanoarrow/nanoarrow_ipc_decoder_test.cc +++ b/src/nanoarrow/nanoarrow_ipc_decoder_test.cc @@ -602,7 +602,7 @@ TEST(NanoarrowIpcTest, NanoarrowIpcSharedBufferThreadSafeDecode) { auto result = memcmp(arrays[i].children[0]->buffers[1], one_two_three_le, sizeof(one_two_three_le)); // discard result to silence -Wunused-value - (void)result; + NANOARROW_UNUSED(result); ArrowArrayRelease(arrays + i); }); } From 82a6b48e4218f24b320cfb9e05f972c59d2cdd37 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 11 Jun 2024 11:47:25 -0400 Subject: [PATCH 18/19] License in subprojects --- dev/release/rat_exclude_files.txt | 4 ---- subprojects/google-benchmark.wrap | 17 +++++++++++++++++ subprojects/gtest.wrap | 17 +++++++++++++++++ subprojects/nlohmann_json.wrap | 17 +++++++++++++++++ subprojects/zlib.wrap | 17 +++++++++++++++++ 5 files changed, 68 insertions(+), 4 deletions(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 128077939..6372a56ac 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -15,7 +15,3 @@ dist/flatcc.c src/nanoarrow/nanoarrow_ipc_flatcc_generated.h thirdparty/* python/src/nanoarrow/dlpack_abi.h -subprojects/google-benchmark.wrap -subprojects/gtest.wrap -subprojects/nlohmann_json.wrap -subprojects/zlib.wrap diff --git a/subprojects/google-benchmark.wrap b/subprojects/google-benchmark.wrap index 91ff9528d..3b6704639 100644 --- a/subprojects/google-benchmark.wrap +++ b/subprojects/google-benchmark.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = benchmark-1.8.4 source_url = https://github.com/google/benchmark/archive/refs/tags/v1.8.4.tar.gz diff --git a/subprojects/gtest.wrap b/subprojects/gtest.wrap index adb8a9a6d..ede25fc0b 100644 --- a/subprojects/gtest.wrap +++ b/subprojects/gtest.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = googletest-1.14.0 source_url = https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz diff --git a/subprojects/nlohmann_json.wrap b/subprojects/nlohmann_json.wrap index bf5d7000b..28d4cbf53 100644 --- a/subprojects/nlohmann_json.wrap +++ b/subprojects/nlohmann_json.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = nlohmann_json-3.11.2 lead_directory_missing = true diff --git a/subprojects/zlib.wrap b/subprojects/zlib.wrap index aa14de177..f96fcef03 100644 --- a/subprojects/zlib.wrap +++ b/subprojects/zlib.wrap @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [wrap-file] directory = zlib-1.3.1 source_url = http://zlib.net/fossils/zlib-1.3.1.tar.gz From b878e23bb721f07a058e9f63a4db3aa6588f0faa Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 11 Jun 2024 14:29:39 -0400 Subject: [PATCH 19/19] flatcc using meson --- .gitignore | 1 + subprojects/flatcc.wrap | 12 +++---- subprojects/packagefiles/flatcc/meson.build | 40 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 subprojects/packagefiles/flatcc/meson.build diff --git a/.gitignore b/.gitignore index 797098608..89fb9fe2f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ __pycache__ # meson subprojects - wrap files need to be kept to let meson download # dependencies as needed, but dependencies themselves should not be versioned subprojects/* +!subprojects/packagefiles !subprojects/*.wrap compile_commands.json diff --git a/subprojects/flatcc.wrap b/subprojects/flatcc.wrap index cd1354cf5..311425ae2 100644 --- a/subprojects/flatcc.wrap +++ b/subprojects/flatcc.wrap @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -[wrap-git] -method = cmake -url = https://github.com/dvidelabs/flatcc -revision = v0.6.1 -depth = 1 -clone-recursive = true +[wrap-file] +directory = flatcc-0.6.1 +source_url = https://github.com/dvidelabs/flatcc/archive/refs/tags/v0.6.1.tar.gz +source_filename = flatcc-0.6.1.tar.gz +source_hash = 2533c2f1061498499f15acc7e0937dcf35bc68e685d237325124ae0d6c600c2b +patch_directory = flatcc diff --git a/subprojects/packagefiles/flatcc/meson.build b/subprojects/packagefiles/flatcc/meson.build new file mode 100644 index 000000000..9e12be21f --- /dev/null +++ b/subprojects/packagefiles/flatcc/meson.build @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('flatcc', + 'c', + version : '0.6.1', + license : 'Apache-2.0', +) + +incdir = include_directories('include') + +flatcc_lib = library( + 'flatcc', + sources: [ + 'src/runtime/builder.c', + 'src/runtime/emitter.c', + 'src/runtime/verifier.c', + 'src/runtime/refmap.c', + ], + include_directories: [incdir], +) + +flatcc_dep = declare_dependency( + include_directories: [incdir], + link_with: [flatcc_lib], +)