Skip to content

Commit 699ad52

Browse files
Subham-KRLXSubham-KRLX
committed
Add China mirror support for dependencies
Fixes #239
1 parent 6fe80fe commit 699ad52

File tree

3 files changed

+156
-16
lines changed

3 files changed

+156
-16
lines changed

‎BUILD_CHINA.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Building in China
2+
3+
This guide helps developers in China build iceberg-cpp when network access to GitHub and other international sites is limited.
4+
5+
## Mirror Support
6+
7+
The build system automatically tries alternative download mirrors when the primary URL fails. All third-party dependencies have been configured with China-based mirrors.
8+
9+
### Available Mirrors
10+
11+
Dependencies are automatically downloaded from these mirror sites:
12+
13+
**Apache Projects (Arrow, Nanoarrow):**
14+
- Tsinghua University: https://mirrors.tuna.tsinghua.edu.cn/apache/
15+
- USTC: https://mirrors.ustc.edu.cn/apache/
16+
17+
**GitHub Projects (CRoaring, nlohmann-json, spdlog, cpr):**
18+
- Gitee: https://gitee.com/mirrors/
19+
- FastGit: https://hub.fastgit.xyz/
20+
21+
**Note**: Avro requires a git repository (unreleased version). Automatic mirror fallback is not available for git repositories, but you can specify a custom git mirror using the `ICEBERG_AVRO_GIT_URL` environment variable.
22+
23+
### Custom Mirror URLs
24+
25+
To override the default mirrors, set environment variables before running CMake:
26+
27+
```bash
28+
export ICEBERG_ARROW_URL="https://mirrors.tuna.tsinghua.edu.cn/apache/arrow/arrow-22.0.0/apache-arrow-22.0.0.tar.gz"
29+
export ICEBERG_NANOARROW_URL="https://mirrors.tuna.tsinghua.edu.cn/apache/arrow/apache-arrow-nanoarrow-0.7.0/apache-arrow-nanoarrow-0.7.0.tar.gz"
30+
export ICEBERG_CROARING_URL="https://gitee.com/mirrors/CRoaring/repository/archive/v4.3.11.tar.gz"
31+
export ICEBERG_NLOHMANN_JSON_URL="https://gitee.com/mirrors/JSON-for-Modern-CPP/releases/download/v3.11.3/json.tar.xz"
32+
export ICEBERG_SPDLOG_URL="https://gitee.com/mirrors/spdlog/repository/archive/v1.15.3.tar.gz"
33+
export ICEBERG_CPR_URL="https://gitee.com/mirrors/cpr/repository/archive/1.12.0.tar.gz"
34+
35+
# For Avro, you can use either a tarball URL or a git repository URL:
36+
export ICEBERG_AVRO_URL="https://example.com/avro.tar.gz" # if you have a tarball
37+
# OR
38+
export ICEBERG_AVRO_GIT_URL="https://gitee.com/mirrors/avro.git" # for git mirror
39+
```
40+
41+
Then build as usual:
42+
43+
```bash
44+
cmake -S . -B build
45+
cmake --build build
46+
```
47+
48+
## Troubleshooting
49+
50+
**Download failures:**
51+
- Try setting a specific mirror using environment variables
52+
- Use a VPN or proxy: `export https_proxy=http://proxy:port`
53+
- Pre-download tarballs to `~/.cmake/Downloads/`
54+
55+
**Slow downloads:**
56+
- The build will automatically retry with different mirrors
57+
- Consider using Meson build system as an alternative
58+
59+
**Still having issues?**
60+
Open an issue at https://github.com/apache/iceberg-cpp/issues with details about which dependency failed and the error message.

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ C++ implementation of [Apache Iceberg™](https://iceberg.apache.org/).
3131
- CMake 3.25 or higher
3232
- C++23 compliant compiler
3333

34+
> **Note**: For developers in China experiencing network issues when downloading dependencies, see [BUILD_CHINA.md](BUILD_CHINA.md) for mirror configuration.
35+
3436
## Build
3537

3638
### Build, Run Test and Install Core Libraries

‎cmake_modules/IcebergThirdpartyToolchain.cmake‎

Lines changed: 94 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ else()
3333
set(ARROW_SOURCE_URL
3434
"https://www.apache.org/dyn/closer.lua?action=download&filename=/arrow/arrow-${ICEBERG_ARROW_BUILD_VERSION}/apache-arrow-${ICEBERG_ARROW_BUILD_VERSION}.tar.gz"
3535
"https://downloads.apache.org/arrow/arrow-${ICEBERG_ARROW_BUILD_VERSION}/apache-arrow-${ICEBERG_ARROW_BUILD_VERSION}.tar.gz"
36+
"https://mirrors.tuna.tsinghua.edu.cn/apache/arrow/arrow-${ICEBERG_ARROW_BUILD_VERSION}/apache-arrow-${ICEBERG_ARROW_BUILD_VERSION}.tar.gz"
37+
"https://mirrors.ustc.edu.cn/apache/arrow/arrow-${ICEBERG_ARROW_BUILD_VERSION}/apache-arrow-${ICEBERG_ARROW_BUILD_VERSION}.tar.gz"
3638
)
3739
endif()
3840

@@ -164,17 +166,42 @@ function(resolve_avro_dependency)
164166
OFF
165167
CACHE BOOL "" FORCE)
166168

167-
fetchcontent_declare(avro-cpp
168-
${FC_DECLARE_COMMON_OPTIONS}
169-
# TODO: switch to Apache Avro 1.13.0 once released.
170-
GIT_REPOSITORY https://github.com/apache/avro.git
171-
GIT_TAG e6c308780e876b4c11a470b9900995947f7b0fb5
172-
SOURCE_SUBDIR
173-
lang/c++
174-
FIND_PACKAGE_ARGS
175-
NAMES
176-
avro-cpp
177-
CONFIG)
169+
if(DEFINED ENV{ICEBERG_AVRO_URL})
170+
# Support custom tarball URL
171+
fetchcontent_declare(avro-cpp
172+
${FC_DECLARE_COMMON_OPTIONS}
173+
URL $ENV{ICEBERG_AVRO_URL}
174+
SOURCE_SUBDIR
175+
lang/c++
176+
FIND_PACKAGE_ARGS
177+
NAMES
178+
avro-cpp
179+
CONFIG)
180+
elseif(DEFINED ENV{ICEBERG_AVRO_GIT_URL})
181+
# Support custom git URL for mirrors
182+
fetchcontent_declare(avro-cpp
183+
${FC_DECLARE_COMMON_OPTIONS}
184+
GIT_REPOSITORY $ENV{ICEBERG_AVRO_GIT_URL}
185+
GIT_TAG e6c308780e876b4c11a470b9900995947f7b0fb5
186+
SOURCE_SUBDIR
187+
lang/c++
188+
FIND_PACKAGE_ARGS
189+
NAMES
190+
avro-cpp
191+
CONFIG)
192+
else()
193+
# Default to GitHub - uses unreleased version
194+
fetchcontent_declare(avro-cpp
195+
${FC_DECLARE_COMMON_OPTIONS}
196+
GIT_REPOSITORY https://github.com/apache/avro.git
197+
GIT_TAG e6c308780e876b4c11a470b9900995947f7b0fb5
198+
SOURCE_SUBDIR
199+
lang/c++
200+
FIND_PACKAGE_ARGS
201+
NAMES
202+
avro-cpp
203+
CONFIG)
204+
endif()
178205

179206
fetchcontent_makeavailable(avro-cpp)
180207

@@ -221,9 +248,20 @@ endfunction()
221248
function(resolve_nanoarrow_dependency)
222249
prepare_fetchcontent()
223250

251+
if(DEFINED ENV{ICEBERG_NANOARROW_URL})
252+
set(NANOARROW_URL "$ENV{ICEBERG_NANOARROW_URL}")
253+
else()
254+
set(NANOARROW_URL
255+
"https://dlcdn.apache.org/arrow/apache-arrow-nanoarrow-0.7.0/apache-arrow-nanoarrow-0.7.0.tar.gz"
256+
"https://downloads.apache.org/arrow/apache-arrow-nanoarrow-0.7.0/apache-arrow-nanoarrow-0.7.0.tar.gz"
257+
"https://mirrors.tuna.tsinghua.edu.cn/apache/arrow/apache-arrow-nanoarrow-0.7.0/apache-arrow-nanoarrow-0.7.0.tar.gz"
258+
"https://mirrors.ustc.edu.cn/apache/arrow/apache-arrow-nanoarrow-0.7.0/apache-arrow-nanoarrow-0.7.0.tar.gz"
259+
)
260+
endif()
261+
224262
fetchcontent_declare(nanoarrow
225263
${FC_DECLARE_COMMON_OPTIONS}
226-
URL "https://dlcdn.apache.org/arrow/apache-arrow-nanoarrow-0.7.0/apache-arrow-nanoarrow-0.7.0.tar.gz"
264+
URL ${NANOARROW_URL}
227265
FIND_PACKAGE_ARGS
228266
NAMES
229267
nanoarrow
@@ -270,9 +308,19 @@ function(resolve_croaring_dependency)
270308
set(ENABLE_ROARING_TESTS OFF)
271309
set(ENABLE_ROARING_MICROBENCHMARKS OFF)
272310

311+
if(DEFINED ENV{ICEBERG_CROARING_URL})
312+
set(CROARING_URL "$ENV{ICEBERG_CROARING_URL}")
313+
else()
314+
set(CROARING_URL
315+
"https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v4.3.11.tar.gz"
316+
"https://gitee.com/mirrors/CRoaring/repository/archive/v4.3.11.tar.gz"
317+
"https://hub.fastgit.xyz/RoaringBitmap/CRoaring/archive/refs/tags/v4.3.11.tar.gz"
318+
)
319+
endif()
320+
273321
fetchcontent_declare(croaring
274322
${FC_DECLARE_COMMON_OPTIONS}
275-
URL "https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v4.3.11.tar.gz"
323+
URL ${CROARING_URL}
276324
FIND_PACKAGE_ARGS
277325
NAMES
278326
roaring
@@ -318,9 +366,19 @@ function(resolve_nlohmann_json_dependency)
318366
OFF
319367
CACHE BOOL "" FORCE)
320368

369+
if(DEFINED ENV{ICEBERG_NLOHMANN_JSON_URL})
370+
set(NLOHMANN_JSON_URL "$ENV{ICEBERG_NLOHMANN_JSON_URL}")
371+
else()
372+
set(NLOHMANN_JSON_URL
373+
"https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz"
374+
"https://gitee.com/mirrors/JSON-for-Modern-CPP/releases/download/v3.11.3/json.tar.xz"
375+
"https://hub.fastgit.xyz/nlohmann/json/releases/download/v3.11.3/json.tar.xz"
376+
)
377+
endif()
378+
321379
fetchcontent_declare(nlohmann_json
322380
${FC_DECLARE_COMMON_OPTIONS}
323-
URL "https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz"
381+
URL ${NLOHMANN_JSON_URL}
324382
FIND_PACKAGE_ARGS
325383
NAMES
326384
nlohmann_json
@@ -378,9 +436,19 @@ function(resolve_spdlog_dependency)
378436
ON
379437
CACHE BOOL "" FORCE)
380438

439+
if(DEFINED ENV{ICEBERG_SPDLOG_URL})
440+
set(SPDLOG_URL "$ENV{ICEBERG_SPDLOG_URL}")
441+
else()
442+
set(SPDLOG_URL
443+
"https://github.com/gabime/spdlog/archive/refs/tags/v1.15.3.tar.gz"
444+
"https://gitee.com/mirrors/spdlog/repository/archive/v1.15.3.tar.gz"
445+
"https://hub.fastgit.xyz/gabime/spdlog/archive/refs/tags/v1.15.3.tar.gz"
446+
)
447+
endif()
448+
381449
fetchcontent_declare(spdlog
382450
${FC_DECLARE_COMMON_OPTIONS}
383-
URL "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.3.tar.gz"
451+
URL ${SPDLOG_URL}
384452
FIND_PACKAGE_ARGS
385453
NAMES
386454
spdlog
@@ -440,9 +508,19 @@ function(resolve_cpr_dependency)
440508
set(CPR_ENABLE_SSL ON)
441509
set(CPR_USE_SYSTEM_CURL ON)
442510

511+
if(DEFINED ENV{ICEBERG_CPR_URL})
512+
set(CPR_URL "$ENV{ICEBERG_CPR_URL}")
513+
else()
514+
set(CPR_URL
515+
"https://github.com/libcpr/cpr/archive/refs/tags/1.12.0.tar.gz"
516+
"https://gitee.com/mirrors/cpr/repository/archive/1.12.0.tar.gz"
517+
"https://hub.fastgit.xyz/libcpr/cpr/archive/refs/tags/1.12.0.tar.gz"
518+
)
519+
endif()
520+
443521
fetchcontent_declare(cpr
444522
${FC_DECLARE_COMMON_OPTIONS}
445-
URL https://github.com/libcpr/cpr/archive/refs/tags/1.12.0.tar.gz
523+
URL ${CPR_URL}
446524
FIND_PACKAGE_ARGS
447525
NAMES
448526
cpr

0 commit comments

Comments
 (0)