Skip to content

Commit ac8d2c7

Browse files
yhmoyhmo
committed
Migrate to Conan 2.x
Signed-off-by: yhmo <yihua.mo@zilliz.com>
1 parent b08080f commit ac8d2c7

File tree

11 files changed

+193
-152
lines changed

11 files changed

+193
-152
lines changed

‎.github/workflows/cpp-ci.yml‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,15 @@ jobs:
2323

2424
- name: Install dependencies
2525
uses: aminya/setup-cpp@v1
26-
with:
27-
conan: 1.64.0
26+
with:
27+
conan: 2.25.1
2828
cmake: true
2929

30+
# Conan 2.x CMakeDeps doesn't propagate system_libs through shared library targets,
31+
# so libaio (required by folly) must be explicitly installed and linked.
32+
- name: Install system libraries
33+
run: sudo apt-get update && sudo apt-get install -y libaio-dev
34+
3035
- name: Setup Rust
3136
id: rust-toolchain
3237
uses: dtolnay/rust-toolchain@stable
@@ -40,18 +45,20 @@ jobs:
4045
restore-keys: |
4146
storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-
4247
43-
- name: setup conan
44-
run:
45-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
46-
&& conan remote list
47-
4848
- name: conan package cache
4949
uses: actions/cache@v4
5050
with:
51-
path: ~/.conan
51+
path: ~/.conan2
5252
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
5353
restore-keys: conan-cpp-
5454

55+
- name: setup conan
56+
run: |
57+
conan profile detect --force
58+
# || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
59+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
60+
conan remote list
61+
5562
- name: Build
5663
working-directory: ./cpp
5764
run: |
@@ -77,9 +84,14 @@ jobs:
7784
- name: Install dependencies
7885
uses: aminya/setup-cpp@v1
7986
with:
80-
conan: 1.64.0
87+
conan: 2.25.1
8188
cmake: true
8289

90+
# Conan 2.x CMakeDeps doesn't propagate system_libs through shared library targets,
91+
# so libaio (required by folly) must be explicitly installed and linked.
92+
- name: Install system libraries
93+
run: sudo apt-get update && sudo apt-get install -y libaio-dev
94+
8395
- name: Setup Rust
8496
id: rust-toolchain
8597
uses: dtolnay/rust-toolchain@stable
@@ -93,18 +105,20 @@ jobs:
93105
restore-keys: |
94106
storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-
95107
96-
- name: setup conan
97-
run:
98-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
99-
&& conan remote list
100-
101108
- name: conan package cache
102109
uses: actions/cache@v4
103110
with:
104-
path: ~/.conan
111+
path: ~/.conan2
105112
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
106113
restore-keys: conan-cpp-
107114

115+
- name: setup conan
116+
run: |
117+
conan profile detect --force
118+
# || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
119+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
120+
conan remote list
121+
108122
- name: Build
109123
working-directory: ./cpp
110124
run: |

‎.github/workflows/java-ci.yml‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- '.github/workflows/java-ci.yml'
1414

1515
env:
16-
CONAN_VERSION: "1.64.0"
16+
CONAN_VERSION: "2.25.1"
1717

1818
jobs:
1919
build:
@@ -22,10 +22,12 @@ jobs:
2222
steps:
2323
- uses: actions/checkout@v4
2424

25+
# libaio-dev: Conan 2.x CMakeDeps doesn't propagate system_libs through shared
26+
# library targets, so libaio (required by folly) must be explicitly installed.
2527
- name: Install C++ dependencies
2628
run: |
2729
sudo apt-get update
28-
sudo apt-get install -y cmake
30+
sudo apt-get install -y cmake libaio-dev
2931
pip install conan==${{ env.CONAN_VERSION }}
3032
3133
- name: Setup JDK
@@ -50,20 +52,20 @@ jobs:
5052
restore-keys: |
5153
storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-
5254
53-
- name: Setup Conan remote
54-
run: |
55-
if ! conan remote list | grep -q "default-conan-local"; then
56-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
57-
fi
58-
conan remote list
59-
6055
- name: Cache Conan packages
6156
uses: actions/cache@v4
6257
with:
63-
path: ~/.conan
58+
path: ~/.conan2
6459
key: conan-java-${{ hashFiles('./cpp/conanfile.py') }}
6560
restore-keys: conan-java-
6661

62+
- name: Setup Conan remote
63+
run: |
64+
conan profile detect --force
65+
# || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
66+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
67+
conan remote list
68+
6769
- name: Build JNI library
6870
working-directory: ./cpp
6971
run: |

‎.github/workflows/python-ci.yml‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- '.github/workflows/python-ci.yml'
1414

1515
env:
16-
CONAN_VERSION: "1.64.0"
16+
CONAN_VERSION: "2.25.1"
1717

1818
jobs:
1919
lint:
@@ -55,27 +55,29 @@ jobs:
5555
- name: Set up uv
5656
uses: astral-sh/setup-uv@v4
5757

58+
# libaio-dev: Conan 2.x CMakeDeps doesn't propagate system_libs through shared
59+
# library targets, so libaio (required by folly) must be explicitly installed.
5860
- name: Install C++ dependencies
5961
run: |
6062
sudo apt-get update
61-
sudo apt-get install -y cmake
63+
sudo apt-get install -y cmake libaio-dev
6264
pip install conan==${{ env.CONAN_VERSION }}
6365
64-
- name: Setup Conan remote
65-
run: |
66-
if ! conan remote list | grep -q "default-conan-local"; then
67-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
68-
fi
69-
conan remote list
70-
7166
- name: Cache Conan packages
7267
uses: actions/cache@v4
7368
with:
74-
path: ~/.conan
69+
path: ~/.conan2
7570
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
7671
restore-keys: |
7772
conan-cpp-
7873
74+
- name: Setup Conan remote
75+
run: |
76+
conan profile detect --force
77+
# || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
78+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
79+
conan remote list
80+
7981
- name: Setup Rust
8082
id: rust-toolchain
8183
uses: dtolnay/rust-toolchain@stable
@@ -97,29 +99,43 @@ jobs:
9799
- name: Set library path
98100
run: |
99101
# Find all conan library directories and add to LD_LIBRARY_PATH
100-
CONAN_LIB_PATHS=$(find ~/.conan/data -name "lib" -type d 2>/dev/null | grep "/package/" | tr '\n' ':')
102+
CONAN_LIB_PATHS=$(find ~/.conan2/p -name "lib" -type d 2>/dev/null | grep "/p/" | tr '\n' ':')
101103
# Remove trailing colon from CONAN_LIB_PATHS if present
102104
CONAN_LIB_PATHS="${CONAN_LIB_PATHS%:}"
103105
# Put CONAN_LIB_PATHS first to ensure conan-provided libraries (like newer liblzma) are preferred over system ones
104106
echo "LD_LIBRARY_PATH=${CONAN_LIB_PATHS}:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
105107
106108
# Find Conan's liblzma and preload it to override system liblzma
107-
# We search for the shared library file directly across all xz_utils builds
108-
CONAN_LZMA=$(find ~/.conan/data -path "*/xz_utils/*/package/*/lib/liblzma.so*" | head -n 1)
109+
CONAN_LZMA=$(find ~/.conan2/p -name "liblzma.so*" -path "*/lib/*" 2>/dev/null | head -n 1)
110+
111+
# Build LD_PRELOAD list for libraries that need to be forced at runtime
112+
PRELOAD_LIBS=""
109113
110114
if [ -n "$CONAN_LZMA" ]; then
111115
echo "Found Conan liblzma at: $CONAN_LZMA"
112-
echo "LD_PRELOAD=$CONAN_LZMA" >> $GITHUB_ENV
113-
114116
echo "Checking symbol in $CONAN_LZMA:"
115117
nm -D "$CONAN_LZMA" | grep lzma_index_uncompressed_size || echo "Symbol NOT FOUND in Conan lib"
118+
PRELOAD_LIBS="$CONAN_LZMA"
116119
else
117120
echo "WARNING: Conan liblzma.so* not found! Verify xz_utils:shared=True"
118121
fi
119122
120123
echo "Checking system liblzma symbol:"
121124
nm -D /usr/lib/x86_64-linux-gnu/liblzma.so.5 2>/dev/null | grep lzma_index_uncompressed_size || echo "Symbol NOT FOUND in System lib"
122125
126+
# Preload libaio: folly (shared) uses io_submit but doesn't have libaio in DT_NEEDED
127+
LIBAIO=$(find /usr/lib -name "libaio.so*" 2>/dev/null | head -n 1)
128+
if [ -n "$LIBAIO" ]; then
129+
echo "Found libaio at: $LIBAIO"
130+
PRELOAD_LIBS="${PRELOAD_LIBS:+$PRELOAD_LIBS:}$LIBAIO"
131+
else
132+
echo "WARNING: libaio.so not found!"
133+
fi
134+
135+
if [ -n "$PRELOAD_LIBS" ]; then
136+
echo "LD_PRELOAD=$PRELOAD_LIBS" >> $GITHUB_ENV
137+
fi
138+
123139
echo "Configured LD_LIBRARY_PATH and LD_PRELOAD for runtime"
124140
125141
- name: Install Python package
@@ -133,7 +149,7 @@ jobs:
133149
ldconfig -p | grep lzma || echo "liblzma not in ldconfig, checking file system..."
134150
ls -la /usr/lib/x86_64-linux-gnu/liblzma* || true
135151
# Check glog dependencies
136-
ldd ~/.conan/data/glog/0.7.1/_/_/package/*/lib/libglog.so.2 2>/dev/null | head -20 || true
152+
find ~/.conan2/p -name "libglog.so*" -exec ldd {} \; 2>/dev/null | head -20 || true
137153
138154
- name: Run tests
139155
working-directory: ./python

‎.github/workflows/release.yml‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ on:
3838
- 'v[0-9]+.[0-9]+.[0-9]+-*'
3939

4040
env:
41-
CONAN_VERSION: "1.64.0"
41+
CONAN_VERSION: "2.25.1"
4242

4343
jobs:
4444
# =================================================================================================
@@ -55,25 +55,27 @@ jobs:
5555
with:
5656
python-version: '3.11'
5757

58+
# libaio-dev: Conan 2.x CMakeDeps doesn't propagate system_libs through shared
59+
# library targets, so libaio (required by folly) must be explicitly installed.
5860
- name: Install dependencies
5961
run: |
6062
sudo apt-get update
61-
sudo apt-get install -y cmake patchelf
63+
sudo apt-get install -y cmake patchelf libaio-dev
6264
pip install conan==${{ env.CONAN_VERSION }}
6365
64-
- name: Setup Conan remote
65-
run: |
66-
if ! conan remote list | grep -q "default-conan-local"; then
67-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
68-
fi
69-
7066
- name: Cache Conan packages
7167
uses: actions/cache@v4
7268
with:
73-
path: ~/.conan
69+
path: ~/.conan2
7470
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
7571
restore-keys: conan-cpp-
7672

73+
- name: Setup Conan remote
74+
run: |
75+
conan profile detect --force
76+
# || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
77+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
78+
7779
- name: Build C++ library for Python
7880
working-directory: ./cpp
7981
run: make python-lib
@@ -158,10 +160,12 @@ jobs:
158160
steps:
159161
- uses: actions/checkout@v4
160162

163+
# libaio-dev: Conan 2.x CMakeDeps doesn't propagate system_libs through shared
164+
# library targets, so libaio (required by folly) must be explicitly installed.
161165
- name: Install dependencies
162166
run: |
163167
sudo apt-get update
164-
sudo apt-get install -y cmake
168+
sudo apt-get install -y cmake libaio-dev
165169
pip install conan==${{ env.CONAN_VERSION }}
166170
167171
- name: Setup JDK
@@ -172,27 +176,23 @@ jobs:
172176

173177
- uses: sbt/setup-sbt@v1
174178

175-
- name: Setup Conan remote
176-
run: |
177-
if ! conan remote list | grep -q "default-conan-local"; then
178-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
179-
fi
180-
181179
- name: Cache Conan packages
182180
uses: actions/cache@v4
183181
with:
184-
path: ~/.conan
182+
path: ~/.conan2
185183
key: conan-java-${{ hashFiles('./cpp/conanfile.py') }}
186184
restore-keys: conan-java-
187185

186+
- name: Setup Conan remote
187+
run: |
188+
conan profile detect --force
189+
# || true: Conan 2.x errors if the remote already exists (e.g. restored from cache)
190+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
191+
188192
- name: Build JNI Library
189193
working-directory: ./cpp
190194
run: make java-lib
191195

192-
- name: Import Conan Dependencies
193-
working-directory: ./cpp
194-
run: conan imports . -if build
195-
196196
- name: Copy Native Libs
197197
run: |
198198
mkdir -p java/native/linux-x86_64

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ A high-performance columnar storage engine built on Apache Arrow, designed for v
100100

101101
- CMake >= 3.20.0
102102
- C++17 compiler (GCC 8+, Clang 6+)
103-
- Conan >= 1.60.0 and <= 2.0.0
103+
- Conan >= 2.0
104104

105105

106106
### Build from Source (C++)
@@ -109,8 +109,8 @@ A high-performance columnar storage engine built on Apache Arrow, designed for v
109109
git clone https://github.com/milvus-io/milvus-storage.git
110110
cd milvus-storage/cpp
111111

112-
# Setup Conan remote
113-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert 0
112+
# Setup Conan remote artifactory
113+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2
114114

115115
# Build
116116
make build

‎cpp/CMakeLists.txt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ if(NOT Arrow_INCLUDE_DIRS AND arrow_INCLUDE_DIRS)
3838
endif()
3939
include_directories(${Arrow_INCLUDE_DIRS})
4040

41-
find_package(Protobuf REQUIRED)
41+
find_package(Protobuf REQUIRED CONFIG)
4242
find_package(google-cloud-cpp REQUIRED)
4343
find_package(libavrocpp QUIET)
4444
if(NOT libavrocpp_FOUND)
@@ -70,6 +70,10 @@ list(FILTER ALL_SRC_FILES EXCLUDE REGEX ".*/src/jni/.*\\.cpp$")
7070
add_library(milvus-storage SHARED ${ALL_SRC_FILES})
7171

7272
list(APPEND LINK_LIBS arrow::arrow Boost::boost protobuf::protobuf AWS::aws-sdk-cpp-identity-management google-cloud-cpp::storage ${AVRO_TARGET} Folly::folly fmt::fmt)
73+
# folly (shared) uses Linux AIO but the conan recipe doesn't propagate libaio
74+
if(NOT APPLE)
75+
list(APPEND LINK_LIBS aio)
76+
endif()
7377
list(APPEND INCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src)
7478

7579
# Fault injection support using libfiu

0 commit comments

Comments
 (0)