Skip to content

Commit ac7d85d

Browse files
dalingmengdalingmeng
authored
Merge branch 'main' into feature-load_table_schema
2 parents b53e194 + a1a3ff3 commit ac7d85d

File tree

11 files changed

+292
-21
lines changed

11 files changed

+292
-21
lines changed

‎.github/.licenserc.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Copyright 2024-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
header:
16+
license:
17+
spdx-id: Apache-2.0
18+
copyright-owner: Alibaba Inc.
19+
20+
paths:
21+
- '**'
22+
23+
paths-ignore:
24+
- '**/.gitignore'
25+
- '.github/**'
26+
- 'test/test_data/**'
27+
- 'LICENSE'
28+
- 'NOTICE'
29+
- 'third_party/**'
30+
- '**/*.svg'
31+
- '**/*.diff'
32+
33+
comment: on-failure

‎.github/workflows/clang_test.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright 2024-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Clang Test
16+
17+
on:
18+
push:
19+
branches:
20+
- '**'
21+
tags:
22+
- '**'
23+
pull_request:
24+
25+
concurrency:
26+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
27+
cancel-in-progress: true
28+
29+
permissions:
30+
contents: read
31+
32+
env:
33+
PAIMON_HOME: /tmp/paimon
34+
35+
jobs:
36+
ubuntu:
37+
name: AMD64 Ubuntu 24.04
38+
runs-on: ubuntu-24.04
39+
timeout-minutes: 120
40+
strategy:
41+
fail-fast: false
42+
steps:
43+
- name: Checkout paimon-cpp
44+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
45+
with:
46+
lfs: true
47+
- name: Build Paimon
48+
shell: bash
49+
env:
50+
CC: clang
51+
CXX: clang++
52+
run: ci/scripts/build_paimon.sh $(pwd)

‎.github/workflows/gcc_test.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright 2024-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Gcc Test
16+
17+
on:
18+
push:
19+
branches:
20+
- '**'
21+
tags:
22+
- '**'
23+
pull_request:
24+
25+
concurrency:
26+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
27+
cancel-in-progress: true
28+
29+
permissions:
30+
contents: read
31+
32+
env:
33+
PAIMON_HOME: /tmp/paimon
34+
35+
jobs:
36+
ubuntu:
37+
name: AMD64 Ubuntu 24.04
38+
runs-on: ubuntu-24.04
39+
timeout-minutes: 120
40+
strategy:
41+
fail-fast: false
42+
steps:
43+
- name: Checkout paimon-cpp
44+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
45+
with:
46+
lfs: true
47+
- name: Build Paimon
48+
shell: bash
49+
env:
50+
CC: gcc-14
51+
CXX: g++-14
52+
run: ci/scripts/build_paimon.sh $(pwd)

‎.github/workflows/test_with_sanitizer.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright 2024-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Test with sanitizer
16+
17+
on:
18+
push:
19+
branches:
20+
- '**'
21+
tags:
22+
- '**'
23+
pull_request:
24+
25+
concurrency:
26+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
27+
cancel-in-progress: true
28+
29+
permissions:
30+
contents: read
31+
32+
env:
33+
PAIMON_HOME: /tmp/paimon
34+
35+
jobs:
36+
ubuntu:
37+
name: AMD64 Ubuntu 24.04
38+
runs-on: ubuntu-24.04
39+
timeout-minutes: 120
40+
strategy:
41+
fail-fast: false
42+
steps:
43+
- name: Checkout paimon-cpp
44+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
45+
with:
46+
lfs: true
47+
- name: Build Paimon
48+
shell: bash
49+
env:
50+
CC: clang
51+
CXX: clang++
52+
run: ci/scripts/build_paimon.sh $(pwd) true

‎CMakeLists.txt‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ option(PAIMON_BUILD_STATIC "Build static library" ON)
4646
option(PAIMON_BUILD_SHARED "Build shared library" ON)
4747
option(PAIMON_BUILD_TESTS "Build tests" OFF)
4848
option(PAIMON_USE_ASAN "Use Address Sanitizer" OFF)
49-
option(PAIMON_USE_TSAN "Use Thread Sanitizer" OFF)
5049
option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
5150
option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
5251
option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
@@ -331,23 +330,35 @@ if(PAIMON_BUILD_TESTS)
331330
paimon_global_index_static
332331
paimon_local_file_system_static
333332
paimon_mock_file_format_static
333+
-Wl,--no-as-needed
334334
paimon_parquet_file_format_shared
335-
paimon_blob_file_format_shared)
335+
paimon_blob_file_format_shared
336+
-Wl,--as-needed)
336337

337338
if(PAIMON_ENABLE_LANCE)
339+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
338340
list(APPEND TEST_STATIC_LINK_LIBS paimon_lance_file_format_shared)
341+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
339342
endif()
340343
if(PAIMON_ENABLE_ORC)
344+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
341345
list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared)
346+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
342347
endif()
343348
if(PAIMON_ENABLE_AVRO)
349+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
344350
list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared)
351+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
345352
endif()
346353
if(PAIMON_ENABLE_JINDO)
354+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
347355
list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
356+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
348357
endif()
349358
if(PAIMON_ENABLE_LUMINA)
359+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
350360
list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
361+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
351362
endif()
352363

353364
endif()

‎ci/scripts/build_paimon.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright 2024-present Alibaba Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -eux
18+
19+
source_dir=${1}
20+
enable_sanitizer=${2:-false}
21+
build_dir=${1}/build
22+
23+
mkdir ${build_dir}
24+
pushd ${build_dir}
25+
26+
CMAKE_ARGS=(
27+
"-G Ninja"
28+
"-DCMAKE_BUILD_TYPE=Debug"
29+
"-DPAIMON_BUILD_TESTS=ON"
30+
"-DPAIMON_ENABLE_LANCE=ON"
31+
"-DPAIMON_ENABLE_JINDO=ON"
32+
)
33+
34+
if [[ "${enable_sanitizer}" == "true" ]]; then
35+
CMAKE_ARGS+=(
36+
"-DPAIMON_USE_ASAN=ON"
37+
"-DPAIMON_USE_UBSAN=ON"
38+
)
39+
fi
40+
41+
cmake "${CMAKE_ARGS[@]}" ${source_dir}
42+
cmake --build . -- -j$(nproc)
43+
ctest --output-on-failure -j $(nproc)
44+
45+
popd
46+
47+
rm -rf ${build_dir}

‎cmake_modules/san-config.cmake‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,6 @@
1010
# See the License for the specific language governing permissions and
1111
# limitations under the License. See accompanying LICENSE file.
1212

13-
# Clang does not support using ASAN and TSAN simultaneously.
14-
if("${PAIMON_USE_ASAN}" AND "${PAIMON_USE_TSAN}")
15-
message(SEND_ERROR "Can only enable one of ASAN or TSAN at a time")
16-
endif()
17-
1813
add_library(paimon_sanitizer_flags INTERFACE)
1914

2015
if(PAIMON_USE_ASAN)
@@ -28,17 +23,6 @@ if(PAIMON_USE_ASAN)
2823
endif()
2924
endif()
3025

31-
if(PAIMON_USE_TSAN)
32-
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
33-
target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread
34-
-fno-omit-frame-pointer)
35-
target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread)
36-
message(STATUS "Thread Sanitizer enabled")
37-
else()
38-
message(WARNING "Thread Sanitizer is only supported for GCC and Clang compilers")
39-
endif()
40-
endif()
41-
4226
if(PAIMON_USE_UBSAN)
4327
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
4428
target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=undefined

‎src/paimon/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,7 @@ if(PAIMON_BUILD_TESTS)
335335
common/io/buffered_input_stream_test.cpp
336336
common/io/memory_segment_output_stream_test.cpp
337337
common/io/offset_input_stream_test.cpp
338+
common/logging/logging_test.cpp
338339
common/metrics/metrics_impl_test.cpp
339340
common/options/memory_size_test.cpp
340341
common/options/time_duration_test.cpp

‎src/paimon/common/logging/logging.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ std::unique_ptr<Logger> Logger::GetLogger(const std::string& path) {
8181
std::shared_lock<std::shared_mutex> lock(getRegistryLock());
8282
return creator.value()(path);
8383
}
84-
84+
std::unique_lock<std::shared_mutex> ulock(getRegistryLock());
8585
if (!google::IsGoogleLoggingInitialized()) {
8686
google::InitGoogleLogging(program_invocation_name);
8787
}

‎src/paimon/common/logging/logging_test.cpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright 2025-present Alibaba Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
#include "paimon/logging.h"
17+
18+
#include <future>
19+
#include <random>
20+
#include <thread>
21+
22+
#include "paimon/common/executor/future.h"
23+
#include "paimon/executor.h"
24+
#include "paimon/testing/utils/testharness.h"
25+
namespace paimon::test {
26+
TEST(LoggerTest, TestMultiThreadGetLogger) {
27+
auto executor = CreateDefaultExecutor(/*thread_count=*/4);
28+
auto get_logger = []() {
29+
auto logger = Logger::GetLogger("my_log");
30+
ASSERT_TRUE(logger);
31+
};
32+
33+
std::vector<std::future<void>> futures;
34+
for (int i = 0; i < 1000; ++i) {
35+
futures.push_back(Via(executor.get(), get_logger));
36+
}
37+
Wait(futures);
38+
}
39+
} // namespace paimon::test

0 commit comments

Comments
 (0)