Merge pull request ClickHouse#62297 from Algunenano/rust_vendor

Vendor rust dependencies

Author: Algunenano

.gitmodules

@@ -363,6 +363,9 @@
 [submodule "contrib/idna"]
 	path = contrib/idna
 	url = https://github.com/ada-url/idna.git
+[submodule "contrib/rust_vendor"]
+	path = contrib/rust_vendor
+	url = https://github.com/ClickHouse/rust_vendor.git
 [submodule "contrib/openssl"]
 	path = contrib/openssl
 	url = https://github.com/ClickHouse/openssl.git

contrib/corrosion

@@ -13,8 +13,8 @@ ENV CARGO_HOME=/rust/cargo
 ENV PATH="/rust/cargo/bin:${PATH}"
 RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
     chmod 777 -R /rust && \
-    rustup toolchain install nightly-2023-07-04 && \
-    rustup default nightly-2023-07-04 && \
+    rustup toolchain install nightly-2024-04-01 && \
+    rustup default nightly-2024-04-01 && \
     rustup toolchain remove stable && \
     rustup component add rust-src && \
     rustup target add x86_64-unknown-linux-gnu && \

contrib/rust_vendor

@@ -58,6 +58,23 @@ export CC=clang-18
 export CXX=clang++-18
 ```
 
+### Install Rust compiler
+
+First follow the steps in the official [rust documentation](https://www.rust-lang.org/tools/install) to install `rustup`.
+
+As with C++ dependencies, ClickHouse uses vendoring to control exactly what's installed and avoid depending on third
+party services (like the `crates.io` registry).
+
+Although in release mode any rust modern rustup toolchain version should work with this dependencies, if you plan to
+enable sanitizers you must use a version that matches the exact same `std` as the one used in CI (for which we vendor
+the crates):
+
+```bash
+rustup toolchain install nightly-2024-04-01
+rustup default nightly-2024-04-01
+rustup component add rust-src
+```
+
 ### Checkout ClickHouse Sources {#checkout-clickhouse-sources}
 
 ``` bash
@@ -97,6 +114,7 @@ The build requires the following components:
 - Ninja
 - Yasm
 - Gawk
+- rustc
 
 If all the components are installed, you may build it in the same way as the steps above.
 

contrib/sysroot

@@ -1,3 +1,21 @@
+if (OS_FREEBSD)
+    # Right nix/libc requires fspacectl and it had been added only since FreeBSD14.
+    # And since sysroot has older libraries you will got undefined reference for clickhouse binary.
+    #
+    # But likely everything should work without this syscall, however it is not
+    # possible right now to gently override libraries versions for dependencies,
+    # and forking rust modules is a little bit too much for this thing.
+    #
+    # You can take a look at the details in the following issue [1].
+    #
+    #   [1]: https://github.com/rust-lang/cargo/issues/5640
+    #
+    # Update 2024-04: Now prql also requires getrandom() via std::sys::pal::unix::rand::imp::getrandom_fill_bytes
+    message(STATUS "Rust build is disabled for FreeBSD because we use old sysroot files")
+    return()
+endif()
+
+
 # NOTE: should be macro to export RUST_CXXFLAGS/RUST_CFLAGS for subfolders
 macro(configure_rustc)
     # NOTE: this can also be done by overriding rustc, but it not trivial with rustup.
@@ -52,66 +70,19 @@ macro(configure_rustc)
     message(STATUS "RUSTFLAGS: ${RUSTFLAGS}")
     message(STATUS "RUST_CARGO_BUILD_STD: ${RUST_CARGO_BUILD_STD}")
 
-    # NOTE: requires RW access for the source dir
-    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml.in" "${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml" @ONLY)
+    set(RUST_VENDOR_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../contrib/rust_vendor")
 endmacro()
 configure_rustc()
 
-function(clickhouse_import_crate)
-    # This is a workaround for Corrosion case sensitive build type matching in
-    # _generator_add_cargo_targets(), that leads to different paths in
-    # IMPORTED_LOCATION and real path of the library.
-    #
-    # It uses CMAKE_CONFIGURATION_TYPES and $<CONFIG>, so here we preserve the
-    # case of ${CMAKE_BUILD_TYPE} in ${CMAKE_CONFIGURATION_TYPES}.
-    if ("${CMAKE_BUILD_TYPE_UC}" STREQUAL "DEBUG")
-        set(CMAKE_CONFIGURATION_TYPES "${CMAKE_BUILD_TYPE};release")
-    else()
-        set(CMAKE_CONFIGURATION_TYPES "${CMAKE_BUILD_TYPE};debug")
-    endif()
-
-    if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-        set(profile "")
-    else()
-        if (ENABLE_THINLTO)
-            set(profile "release-thinlto")
-        else()
-            set(profile "release")
-        endif()
-    endif()
-
-    # Note, here --offline is not used, since on CI vendor archive is used, and
-    # passing --offline here will be inconvenient for local development.
-    corrosion_import_crate(NO_STD ${ARGN} PROFILE ${profile})
-endfunction()
-
 # Add crate from the build directory.
-#
-# Our crates has configuration files:
-# - config for cargo (see config.toml.in)
-# - and possibly config for build (build.rs.in)
-#
-# And to avoid overlaps different builds for one source directory, crate will
+# To avoid overlaps different builds for one source directory, crate will
 # be copied from source directory to the binary directory.
-file(COPY ".cargo" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
 function(add_rust_subdirectory src)
     set(dst "${CMAKE_CURRENT_BINARY_DIR}/${src}")
     message(STATUS "Copy ${src} to ${dst}")
     file(COPY "${src}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}"
         PATTERN target EXCLUDE)
 
-    # Check is Rust available or not.
-    #
-    # `cargo update --dry-run` will not update anything, but will check the internet connectivity.
-    execute_process(COMMAND ${Rust_CARGO_CACHED} update --dry-run
-        WORKING_DIRECTORY "${dst}"
-        RESULT_VARIABLE CARGO_UPDATE_RESULT
-        OUTPUT_VARIABLE CARGO_UPDATE_STDOUT
-        ERROR_VARIABLE CARGO_UPDATE_STDERR)
-    if (CARGO_UPDATE_RESULT)
-        message(FATAL_ERROR "Rust (${Rust_CARGO_CACHED}) support is not available (likely there is no internet connectivity):\n${CARGO_UPDATE_STDERR}\nYou can disable Rust support with -DENABLE_RUST=OFF")
-    endif()
-
     add_subdirectory("${dst}" "${dst}")
 
     # cmake -E copy* do now know how to exclude files
@@ -126,5 +97,4 @@ function(add_rust_subdirectory src)
         VERBATIM)
 endfunction()
 
-add_rust_subdirectory (skim)
-add_rust_subdirectory (prql)
+add_rust_subdirectory (workspace)

docker/packager/binary-builder/Dockerfile

@@ -0,0 +1,51 @@
+As we have multiple projects we use a workspace to manage them (it's way simpler and leads to less issues). In order
+to vendor all the dependencies we need to store both the registry and the packages themselves.
+
+Note that this includes the exact `std` dependencies for the rustc version used in CI (currently nightly-2024-04-01),
+so you need to install `rustup component add rust-src` for the specific version.
+
+* First step: (Re)-generate the Cargo.lock file (run under `workspace/`).
+
+```bash
+cargo generate-lockfile
+```
+
+* Generate the local registry:
+
+Note that we use both commands to vendor both registry and crates. No idea why both are necessary.
+
+  * First we need to install the tool if you don't already have it:
+```bash
+cargo install --version 0.2.6 cargo-local-registry
+```
+
+  * Now add the local packages:
+
+```bash
+export CH_TOP_DIR=$(git rev-parse --show-toplevel)
+export RUSTC_ROOT=$(rustc --print=sysroot)
+
+cd "$CH_TOP_DIR"/rust/workspace
+
+cargo local-registry --git --sync Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor
+cp "$RUSTC_ROOT"/lib/rustlib/src/rust/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/
+cargo local-registry --no-delete --git --sync "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor
+cp "$RUSTC_ROOT"/lib/rustlib/src/rust/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/
+cargo local-registry --no-delete --git --sync "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/Cargo.lock "$CH_TOP_DIR"/contrib/rust_vendor
+
+cargo vendor --no-delete --locked "$CH_TOP_DIR"/contrib/rust_vendor
+cd "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/
+cargo vendor --no-delete "$CH_TOP_DIR"/contrib/rust_vendor
+cd "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/
+cargo vendor --no-delete "$CH_TOP_DIR"/contrib/rust_vendor
+
+# Remove windows only dependencies (which are really heavy and we don't want in the repo)
+rm -rf "$CH_TOP_DIR"/contrib/rust_vendor/winapi* "$CH_TOP_DIR"/contrib/rust_vendor/windows*
+
+# Cleanup the lock files we copied
+rm "$RUSTC_ROOT"/lib/rustlib/src/rust/library/std/Cargo.lock "$RUSTC_ROOT"/lib/rustlib/src/rust/library/test/Cargo.lock
+cd "$CH_TOP_DIR"/rust/workspace
+```
+
+The `rustc --print=sysroot` part includes `std` dependencies, required to build with sanitizer flags. It must be kept
+in sync with the rustc version used in CI.