Skip to content

Commit e8b0382

Browse files
zjw1111zjw1111
authored
Merge branch 'main' into fix_multithread_mkdir
2 parents 58215ef + ae18f95 commit e8b0382

File tree

154 files changed

+2671
-1157
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

154 files changed

+2671
-1157
lines changed

‎.github/workflows/build_docs.yaml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Copyright 2025-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: "Publish Docs"
16+
17+
on:
18+
push:
19+
branches:
20+
- main
21+
paths:
22+
- 'apidoc/**'
23+
- 'docs/**'
24+
- 'src/**'
25+
- 'include/**'
26+
27+
concurrency:
28+
group: ${{ github.workflow }}-${{ github.ref }}
29+
cancel-in-progress: false
30+
31+
permissions:
32+
contents: write
33+
34+
jobs:
35+
docs:
36+
runs-on: ubuntu-24.04
37+
38+
steps:
39+
- uses: actions/[email protected]
40+
with:
41+
fetch-depth: 1
42+
43+
- uses: actions/setup-python@v6
44+
with:
45+
python-version: '3.x'
46+
47+
- name: Install dependencies
48+
run: |
49+
sudo apt-get update
50+
sudo apt-get install -y doxygen
51+
pip install -r docs/requirements.txt
52+
53+
- name: Build API
54+
run: |
55+
cd ./apidoc
56+
doxygen
57+
cd -
58+
59+
- name: Build Docs
60+
run: |
61+
cd docs
62+
make html
63+
mkdir -p /tmp/site
64+
cp -r ./_build/html/* /tmp/site/
65+
cd -
66+
touch .nojekyll
67+
68+
- name: Deploy to gh-pages
69+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
70+
run: |
71+
git config --global user.name 'GitHub Actions'
72+
git config --global user.email '[email protected]'
73+
74+
git checkout --orphan gh-pages-tmp
75+
git rm --quiet -rf .
76+
cp -r /tmp/site/* .
77+
git add --all
78+
git commit -m "Publish docs from commit ${{ github.sha }}"
79+
git push -f origin gh-pages-tmp:gh-pages

.github/workflows/test.yaml renamed to .github/workflows/clang_test.yaml

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2024-present Alibaba Inc.
1+
# Copyright 2025-present Alibaba Inc.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
name: Test
15+
name: Clang Test
1616

1717
on:
1818
push:
@@ -29,9 +29,6 @@ concurrency:
2929
permissions:
3030
contents: read
3131

32-
env:
33-
PAIMON_HOME: /tmp/paimon
34-
3532
jobs:
3633
ubuntu:
3734
name: AMD64 Ubuntu 24.04
@@ -49,4 +46,4 @@ jobs:
4946
env:
5047
CC: clang
5148
CXX: clang++
52-
run: ci/scripts/build_paimon.sh $(pwd) ON
49+
run: ci/scripts/build_paimon.sh $(pwd)

‎.github/workflows/gcc_test.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2024-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Gcc Test
16+
17+
on:
18+
push:
19+
branches:
20+
- '**'
21+
tags:
22+
- '**'
23+
pull_request:
24+
25+
concurrency:
26+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
27+
cancel-in-progress: true
28+
29+
permissions:
30+
contents: read
31+
32+
jobs:
33+
ubuntu:
34+
name: AMD64 Ubuntu 24.04
35+
runs-on: ubuntu-24.04
36+
timeout-minutes: 120
37+
strategy:
38+
fail-fast: false
39+
steps:
40+
- name: Checkout paimon-cpp
41+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
42+
with:
43+
lfs: true
44+
- name: Build Paimon
45+
shell: bash
46+
env:
47+
CC: gcc-14
48+
CXX: g++-14
49+
run: ci/scripts/build_paimon.sh $(pwd)

‎.github/workflows/pre-commit.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2024-present Alibaba Inc.
1+
# Copyright 2025-present Alibaba Inc.
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.

‎.github/workflows/test_with_sanitizer.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright 2025-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Test with sanitizer
16+
17+
on:
18+
push:
19+
branches:
20+
- '**'
21+
tags:
22+
- '**'
23+
pull_request:
24+
25+
concurrency:
26+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
27+
cancel-in-progress: true
28+
29+
permissions:
30+
contents: read
31+
32+
jobs:
33+
ubuntu:
34+
name: AMD64 Ubuntu 24.04
35+
runs-on: ubuntu-24.04
36+
timeout-minutes: 120
37+
strategy:
38+
fail-fast: false
39+
steps:
40+
- name: Checkout paimon-cpp
41+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
42+
with:
43+
lfs: true
44+
- name: Build Paimon
45+
shell: bash
46+
env:
47+
CC: clang
48+
CXX: clang++
49+
run: ci/scripts/build_paimon.sh $(pwd) true

‎.pre-commit-config.yaml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ repos:
4343
exclude: (^test/test_data/.*|^third_party/.*)
4444

4545
- repo: https://github.com/cheshirekow/cmake-format-precommit
46-
rev: v0.6.10
46+
rev: v0.6.13
4747
hooks:
4848
- id: cmake-format
4949
exclude: (^test/test_data/.*|^third_party/.*)
@@ -67,3 +67,23 @@ repos:
6767
'--disable',
6868
'dangling-hyphen,line-too-long',
6969
]
70+
71+
- repo: https://github.com/cpplint/cpplint
72+
rev: 1.6.1
73+
hooks:
74+
- id: cpplint
75+
alias: cpp
76+
name: C++ Lint
77+
args:
78+
- "--quiet"
79+
- "--verbose=2"
80+
- "--filter=-whitespace/line_length,-whitespace/parens,-build/c++11,-readability/nolint,-runtime/references"
81+
82+
types_or:
83+
- c++
84+
exclude: |
85+
(?x)^(
86+
(third_party)/.*|
87+
(test)/.*|
88+
(src)/.*
89+
)$

‎CMakeLists.txt‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ option(PAIMON_BUILD_STATIC "Build static library" ON)
4646
option(PAIMON_BUILD_SHARED "Build shared library" ON)
4747
option(PAIMON_BUILD_TESTS "Build tests" OFF)
4848
option(PAIMON_USE_ASAN "Use Address Sanitizer" OFF)
49-
option(PAIMON_USE_TSAN "Use Thread Sanitizer" OFF)
5049
option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
5150
option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
5251
option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
@@ -331,23 +330,35 @@ if(PAIMON_BUILD_TESTS)
331330
paimon_global_index_static
332331
paimon_local_file_system_static
333332
paimon_mock_file_format_static
333+
-Wl,--no-as-needed
334334
paimon_parquet_file_format_shared
335-
paimon_blob_file_format_shared)
335+
paimon_blob_file_format_shared
336+
-Wl,--as-needed)
336337

337338
if(PAIMON_ENABLE_LANCE)
339+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
338340
list(APPEND TEST_STATIC_LINK_LIBS paimon_lance_file_format_shared)
341+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
339342
endif()
340343
if(PAIMON_ENABLE_ORC)
344+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
341345
list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared)
346+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
342347
endif()
343348
if(PAIMON_ENABLE_AVRO)
349+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
344350
list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared)
351+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
345352
endif()
346353
if(PAIMON_ENABLE_JINDO)
354+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
347355
list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
356+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
348357
endif()
349358
if(PAIMON_ENABLE_LUMINA)
359+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--no-as-needed)
350360
list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
361+
list(APPEND TEST_STATIC_LINK_LIBS -Wl,--as-needed)
351362
endif()
352363

353364
endif()

‎ci/scripts/build_paimon.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
set -eux
1818

1919
source_dir=${1}
20+
enable_sanitizer=${2:-false}
2021
build_dir=${1}/build
2122

2223
mkdir ${build_dir}
@@ -30,6 +31,13 @@ CMAKE_ARGS=(
3031
"-DPAIMON_ENABLE_JINDO=ON"
3132
)
3233

34+
if [[ "${enable_sanitizer}" == "true" ]]; then
35+
CMAKE_ARGS+=(
36+
"-DPAIMON_USE_ASAN=ON"
37+
"-DPAIMON_USE_UBSAN=ON"
38+
)
39+
fi
40+
3341
cmake "${CMAKE_ARGS[@]}" ${source_dir}
3442
cmake --build . -- -j$(nproc)
3543
ctest --output-on-failure -j $(nproc)

‎cmake_modules/BuildUtils.cmake‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,9 @@ function(add_test_case REL_TEST_NAME)
291291
add_dependencies(${LABEL_TEST_NAME} ${TEST_NAME})
292292
endforeach()
293293

294-
set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${LABELS})
294+
set_property(TEST ${TEST_NAME}
295+
APPEND
296+
PROPERTY LABELS ${LABELS})
295297
endfunction()
296298

297299
# Adding unit tests part of the "paimon" portion of the test suite

‎cmake_modules/SetupCxxFlags.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,8 @@ if(NOT WIN32 AND NOT APPLE)
297297
# won't find it when invoked directly from make/ninja (which is typically
298298
# done outside devtoolset).
299299
execute_process(COMMAND which ld.gold
300-
OUTPUT_VARIABLE GOLD_LOCATION OUTPUT_STRIP_TRAILING_WHITESPACE
301-
ERROR_QUIET)
300+
OUTPUT_VARIABLE GOLD_LOCATION
301+
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
302302
if("${GOLD_LOCATION}" MATCHES "^/opt/rh/devtoolset")
303303
message("Skipping optional gold linker (version ${GOLD_VERSION}) because "
304304
"it's in devtoolset")

0 commit comments

Comments
 (0)