Skip to content

Commit 261a263

Browse files
yhmoyhmo
committed
Migrate to Conan 2.x
Signed-off-by: yhmo <yihua.mo@zilliz.com>
1 parent 3c48ece commit 261a263

File tree

11 files changed

+134
-145
lines changed

11 files changed

+134
-145
lines changed

‎.github/workflows/cpp-ci.yml‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
- name: Install dependencies
2525
uses: aminya/setup-cpp@v1
2626
with:
27-
conan: 1.64.0
27+
conan: 2.25.1
2828
cmake: true
2929

3030
- name: Setup Rust
@@ -40,18 +40,19 @@ jobs:
4040
restore-keys: |
4141
storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-
4242
43-
- name: setup conan
44-
run:
45-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
46-
&& conan remote list
47-
4843
- name: conan package cache
4944
uses: actions/cache@v4
5045
with:
51-
path: ~/.conan
46+
path: ~/.conan2
5247
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
5348
restore-keys: conan-cpp-
5449

50+
- name: setup conan
51+
run: |
52+
conan profile detect --force
53+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
54+
conan remote list
55+
5556
- name: Build
5657
working-directory: ./cpp
5758
run: |
@@ -77,7 +78,7 @@ jobs:
7778
- name: Install dependencies
7879
uses: aminya/setup-cpp@v1
7980
with:
80-
conan: 1.64.0
81+
conan: 2.25.1
8182
cmake: true
8283

8384
- name: Setup Rust
@@ -93,18 +94,19 @@ jobs:
9394
restore-keys: |
9495
storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-
9596
96-
- name: setup conan
97-
run:
98-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
99-
&& conan remote list
100-
10197
- name: conan package cache
10298
uses: actions/cache@v4
10399
with:
104-
path: ~/.conan
100+
path: ~/.conan2
105101
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
106102
restore-keys: conan-cpp-
107103

104+
- name: setup conan
105+
run: |
106+
conan profile detect --force
107+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
108+
conan remote list
109+
108110
- name: Build
109111
working-directory: ./cpp
110112
run: |

‎.github/workflows/java-ci.yml‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- '.github/workflows/java-ci.yml'
1414

1515
env:
16-
CONAN_VERSION: "1.64.0"
16+
CONAN_VERSION: "2.25.1"
1717

1818
jobs:
1919
build:
@@ -50,20 +50,19 @@ jobs:
5050
restore-keys: |
5151
storage-bridge-rust-${{ runner.os }}-${{ steps.rust-toolchain.outputs.cachekey }}-
5252
53-
- name: Setup Conan remote
54-
run: |
55-
if ! conan remote list | grep -q "default-conan-local"; then
56-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
57-
fi
58-
conan remote list
59-
6053
- name: Cache Conan packages
6154
uses: actions/cache@v4
6255
with:
63-
path: ~/.conan
56+
path: ~/.conan2
6457
key: conan-java-${{ hashFiles('./cpp/conanfile.py') }}
6558
restore-keys: conan-java-
6659

60+
- name: Setup Conan remote
61+
run: |
62+
conan profile detect --force
63+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
64+
conan remote list
65+
6766
- name: Build JNI library
6867
working-directory: ./cpp
6968
run: |

‎.github/workflows/python-ci.yml‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- '.github/workflows/python-ci.yml'
1414

1515
env:
16-
CONAN_VERSION: "1.64.0"
16+
CONAN_VERSION: "2.25.1"
1717

1818
jobs:
1919
lint:
@@ -61,21 +61,20 @@ jobs:
6161
sudo apt-get install -y cmake
6262
pip install conan==${{ env.CONAN_VERSION }}
6363
64-
- name: Setup Conan remote
65-
run: |
66-
if ! conan remote list | grep -q "default-conan-local"; then
67-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
68-
fi
69-
conan remote list
70-
7164
- name: Cache Conan packages
7265
uses: actions/cache@v4
7366
with:
74-
path: ~/.conan
67+
path: ~/.conan2
7568
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
7669
restore-keys: |
7770
conan-cpp-
7871
72+
- name: Setup Conan remote
73+
run: |
74+
conan profile detect --force
75+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
76+
conan remote list
77+
7978
- name: Setup Rust
8079
id: rust-toolchain
8180
uses: dtolnay/rust-toolchain@stable
@@ -97,15 +96,14 @@ jobs:
9796
- name: Set library path
9897
run: |
9998
# Find all conan library directories and add to LD_LIBRARY_PATH
100-
CONAN_LIB_PATHS=$(find ~/.conan/data -name "lib" -type d 2>/dev/null | grep "/package/" | tr '\n' ':')
99+
CONAN_LIB_PATHS=$(find ~/.conan2/p -name "lib" -type d 2>/dev/null | grep "/p/" | tr '\n' ':')
101100
# Remove trailing colon from CONAN_LIB_PATHS if present
102101
CONAN_LIB_PATHS="${CONAN_LIB_PATHS%:}"
103102
# Put CONAN_LIB_PATHS first to ensure conan-provided libraries (like newer liblzma) are preferred over system ones
104103
echo "LD_LIBRARY_PATH=${CONAN_LIB_PATHS}:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
105104
106105
# Find Conan's liblzma and preload it to override system liblzma
107-
# We search for the shared library file directly across all xz_utils builds
108-
CONAN_LZMA=$(find ~/.conan/data -path "*/xz_utils/*/package/*/lib/liblzma.so*" | head -n 1)
106+
CONAN_LZMA=$(find ~/.conan2/p -name "liblzma.so*" -path "*/lib/*" 2>/dev/null | head -n 1)
109107
110108
if [ -n "$CONAN_LZMA" ]; then
111109
echo "Found Conan liblzma at: $CONAN_LZMA"
@@ -133,7 +131,7 @@ jobs:
133131
ldconfig -p | grep lzma || echo "liblzma not in ldconfig, checking file system..."
134132
ls -la /usr/lib/x86_64-linux-gnu/liblzma* || true
135133
# Check glog dependencies
136-
ldd ~/.conan/data/glog/0.7.1/_/_/package/*/lib/libglog.so.2 2>/dev/null | head -20 || true
134+
find ~/.conan2/p -name "libglog.so*" -exec ldd {} \; 2>/dev/null | head -20 || true
137135
138136
- name: Run tests
139137
working-directory: ./python

‎.github/workflows/release.yml‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ on:
3838
- 'v[0-9]+.[0-9]+.[0-9]+-*'
3939

4040
env:
41-
CONAN_VERSION: "1.64.0"
41+
CONAN_VERSION: "2.25.1"
4242

4343
jobs:
4444
# =================================================================================================
@@ -61,19 +61,18 @@ jobs:
6161
sudo apt-get install -y cmake patchelf
6262
pip install conan==${{ env.CONAN_VERSION }}
6363
64-
- name: Setup Conan remote
65-
run: |
66-
if ! conan remote list | grep -q "default-conan-local"; then
67-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
68-
fi
69-
7064
- name: Cache Conan packages
7165
uses: actions/cache@v4
7266
with:
73-
path: ~/.conan
67+
path: ~/.conan2
7468
key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
7569
restore-keys: conan-cpp-
7670

71+
- name: Setup Conan remote
72+
run: |
73+
conan profile detect --force
74+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
75+
7776
- name: Build C++ library for Python
7877
working-directory: ./cpp
7978
run: make python-lib
@@ -172,27 +171,22 @@ jobs:
172171

173172
- uses: sbt/setup-sbt@v1
174173

175-
- name: Setup Conan remote
176-
run: |
177-
if ! conan remote list | grep -q "default-conan-local"; then
178-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
179-
fi
180-
181174
- name: Cache Conan packages
182175
uses: actions/cache@v4
183176
with:
184-
path: ~/.conan
177+
path: ~/.conan2
185178
key: conan-java-${{ hashFiles('./cpp/conanfile.py') }}
186179
restore-keys: conan-java-
187180

181+
- name: Setup Conan remote
182+
run: |
183+
conan profile detect --force
184+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2 || true
185+
188186
- name: Build JNI Library
189187
working-directory: ./cpp
190188
run: make java-lib
191189

192-
- name: Import Conan Dependencies
193-
working-directory: ./cpp
194-
run: conan imports . -if build
195-
196190
- name: Copy Native Libs
197191
run: |
198192
mkdir -p java/native/linux-x86_64

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ A high-performance columnar storage engine built on Apache Arrow, designed for v
100100

101101
- CMake >= 3.20.0
102102
- C++17 compiler (GCC 8+, Clang 6+)
103-
- Conan >= 1.60.0 and <= 2.0.0
103+
- Conan >= 2.0
104104

105105

106106
### Build from Source (C++)
@@ -109,8 +109,8 @@ A high-performance columnar storage engine built on Apache Arrow, designed for v
109109
git clone https://github.com/milvus-io/milvus-storage.git
110110
cd milvus-storage/cpp
111111

112-
# Setup Conan remote
113-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert 0
112+
# Setup Conan remote artifactory
113+
conan remote add default-conan-local2 https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local2
114114

115115
# Build
116116
make build

‎cpp/CMakeLists.txt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ if(NOT Arrow_INCLUDE_DIRS AND arrow_INCLUDE_DIRS)
3838
endif()
3939
include_directories(${Arrow_INCLUDE_DIRS})
4040

41-
find_package(Protobuf REQUIRED)
41+
find_package(Protobuf REQUIRED CONFIG)
4242
find_package(google-cloud-cpp REQUIRED)
4343
find_package(libavrocpp QUIET)
4444
if(NOT libavrocpp_FOUND)
@@ -70,6 +70,10 @@ list(FILTER ALL_SRC_FILES EXCLUDE REGEX ".*/src/jni/.*\\.cpp$")
7070
add_library(milvus-storage SHARED ${ALL_SRC_FILES})
7171

7272
list(APPEND LINK_LIBS arrow::arrow Boost::boost protobuf::protobuf AWS::aws-sdk-cpp-identity-management google-cloud-cpp::storage ${AVRO_TARGET} Folly::folly fmt::fmt)
73+
# folly (shared) uses Linux AIO but the conan recipe doesn't propagate libaio
74+
if(NOT APPLE)
75+
list(APPEND LINK_LIBS aio)
76+
endif()
7377
list(APPEND INCLUDE_PATHS ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src)
7478

7579
# Fault injection support using libfiu

‎cpp/Makefile‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ ifeq ($(uname_s),Darwin)
2424
endif
2525

2626
# Common conan flags
27-
CONAN_BASE = conan install .. --build=missing ${libcxx_setting} -s build_type=${build_type} --update
27+
CONAN_SETTINGS = ${libcxx_setting} -s build_type=${build_type}
28+
CONAN_BASE = conan install .. --build=missing ${CONAN_SETTINGS} --update
2829

2930
build:
3031
# Example building debug mode: BUILD_TYPE=Debug WITH_UT=False make
@@ -39,29 +40,36 @@ build:
3940

4041
mkdir -p build && cd build && \
4142
${CONAN_BASE} \
42-
-o with_ut=${with_ut} -o with_benchmark=${with_benchmark} -o with_asan=${use_asan} \
43-
-o with_jni=${use_jni} -o with_python_binding=${use_python_binding} -o with_fiu=${with_fiu} && conan build ..
43+
-o "&:with_ut=${with_ut}" -o "&:with_benchmark=${with_benchmark}" -o "&:with_asan=${use_asan}" \
44+
-o "&:with_jni=${use_jni}" -o "&:with_python_binding=${use_python_binding}" -o "&:with_fiu=${with_fiu}" && \
45+
conan build .. ${CONAN_SETTINGS} \
46+
-o "&:with_ut=${with_ut}" -o "&:with_benchmark=${with_benchmark}" -o "&:with_asan=${use_asan}" \
47+
-o "&:with_jni=${use_jni}" -o "&:with_python_binding=${use_python_binding}" -o "&:with_fiu=${with_fiu}"
4448

4549
package: build
4650
mkdir -p build && cd build && \
47-
conan export .. milvus-storage/0.1.0@milvus/dev
51+
conan export .. --user=milvus --channel=dev
4852

4953
python-lib:
5054
@echo "Building library for Python bindings with hidden symbols..."
5155
@echo "with_fiu: ${with_fiu}"
5256
mkdir -p build && cd build && \
5357
${CONAN_BASE} \
54-
-o with_ut=False -o with_benchmark=False -o with_asan=False -o with_jni=False -o with_jemalloc=False \
55-
-o with_python_binding=True -o with_azure=False -o with_fiu=${with_fiu} && \
56-
conan build ..
58+
-o "&:with_ut=False" -o "&:with_benchmark=False" -o "&:with_asan=False" -o "&:with_jni=False" -o "&:with_jemalloc=False" \
59+
-o "&:with_python_binding=True" -o "&:with_azure=False" -o "&:with_fiu=${with_fiu}" && \
60+
conan build .. ${CONAN_SETTINGS} \
61+
-o "&:with_ut=False" -o "&:with_benchmark=False" -o "&:with_asan=False" -o "&:with_jni=False" -o "&:with_jemalloc=False" \
62+
-o "&:with_python_binding=True" -o "&:with_azure=False" -o "&:with_fiu=${with_fiu}"
5763

5864
java-lib:
5965
@echo "Building library for Java/Scala dependencies..."
6066
mkdir -p build && cd build && \
6167
${CONAN_BASE} \
62-
-o with_ut=False -o with_benchmark=False -o with_asan=False -o with_jni=True -o with_jemalloc=False \
63-
-o with_python_binding=False -o with_azure=True -o with_fiu=${with_fiu} && \
64-
conan build ..
68+
-o "&:with_ut=False" -o "&:with_benchmark=False" -o "&:with_asan=False" -o "&:with_jni=True" -o "&:with_jemalloc=False" \
69+
-o "&:with_python_binding=False" -o "&:with_azure=True" -o "&:with_fiu=${with_fiu}" && \
70+
conan build .. ${CONAN_SETTINGS} \
71+
-o "&:with_ut=False" -o "&:with_benchmark=False" -o "&:with_asan=False" -o "&:with_jni=True" -o "&:with_jemalloc=False" \
72+
-o "&:with_python_binding=False" -o "&:with_azure=True" -o "&:with_fiu=${with_fiu}"
6573

6674
clean:
6775
rm -rf build

0 commit comments

Comments
 (0)