WIP tsan support for rust

Author: graydon

configure.ac

@@ -94,6 +94,11 @@ AX_APPEND_COMPILE_FLAGS($WFLAGS)
 AX_APPEND_COMPILE_FLAGS([-pthread])
 AC_LANG_POP(C)
 
+AC_ARG_ENABLE(unified-rust-unsafe-for-production,
+    AS_HELP_STRING([--enable-unified-rust-unsafe-for-production],
+        [Build rust crates as a single cargo library, risking version drift]))
+AM_CONDITIONAL(UNIFIED_RUST, [test "x$enable_unified_rust_unsafe_for_production" = "xyes"])
+
 unset sanitizeopts
 
 AC_ARG_ENABLE([asan],

src/Makefile.am

@@ -117,13 +117,15 @@ BUILT_SOURCES += rust/RustBridge.h rust/RustBridge.cpp
 stellar_core_SOURCES += rust/RustBridge.h rust/RustBridge.cpp
 
 RUST_TOOLCHAIN_FILE=$(top_srcdir)/rust-toolchain.toml
-RUST_TOOLCHAIN_CHANNEL=$(shell sed -n 's/channel *= *"\([^"]*\)"/\1/p' $(RUST_TOOLCHAIN_FILE))
+RUST_TOOLCHAIN_FILE_CHANNEL=$(shell sed -n 's/channel *= *"\([^"]*\)"/\1/p' $(RUST_TOOLCHAIN_FILE))
+RUST_TOOLCHAIN_CHANNEL=$(if $(findstring sanitize,$(CXXFLAGS)),nightly,$(RUST_TOOLCHAIN_FILE_CHANNEL))
 CARGO=cargo +$(RUST_TOOLCHAIN_CHANNEL)
 
 # we pass RUST_TOOLCHAIN_CHANNEL by environment variable
 # to tests since they can't take command-line arguments.
 export RUST_TOOLCHAIN_CHANNEL
 
+RUST_TARGET=$(shell rustc -vV | sed -n 's/host: //p')
 RUST_BUILD_DIR=$(top_builddir)/src/rust
 RUST_BIN_DIR=$(RUST_BUILD_DIR)/bin
 RUST_TARGET_DIR=$(top_builddir)/target
@@ -132,7 +134,7 @@ RUST_PROFILE=release
 RUST_DEP_TREE_STAMP=$(RUST_BUILD_DIR)/src/dep-trees/equal-trees.stamp
 SOROBAN_LIBS_STAMP=$(RUST_BUILD_DIR)/soroban/soroban-libs.stamp
 RUST_HOST_DEPFILES=rust/Cargo.toml $(top_srcdir)/Cargo.toml $(top_srcdir)/Cargo.lock $(RUST_DEP_TREE_STAMP)
-LIBRUST_STELLAR_CORE=$(RUST_TARGET_DIR)/$(RUST_PROFILE)/librust_stellar_core.a
+LIBRUST_STELLAR_CORE=$(RUST_TARGET_DIR)/$(RUST_TARGET)/$(RUST_PROFILE)/librust_stellar_core.a
 stellar_core_LDADD += $(LIBRUST_STELLAR_CORE) -ldl
 
 SOROBAN_BUILD_DIR=$(abspath $(RUST_BUILD_DIR))/soroban
@@ -178,7 +180,7 @@ endif
 SOROBAN_MAX_PROTOCOL=$(lastword $(sort $(ALL_SOROBAN_PROTOCOLS)))
 
 define soroban_lib_dir
-$(shell printf '$(SOROBAN_BUILD_DIR)/%s/target/$(RUST_PROFILE)' $(1))
+$(shell printf '$(SOROBAN_BUILD_DIR)/%s/target/$(RUST_TARGET)/$(RUST_PROFILE)' $(1))
 endef
 
 define soroban_rlib
@@ -232,6 +234,32 @@ $(RUST_DEP_TREE_STAMP): $(wildcard rust/soroban/*/Cargo.*) Makefile $(RUST_TOOLC
 # So we have to derive a new variable here.
 RUST_PROFILE_ARG := $(if $(findstring release,$(RUST_PROFILE)),--release,)
 
+RUSTFLAGS_ASAN := $(if $(findstring -fsanitize=address,$(CXXFLAGS)),-Zsanitizer=address,)
+RUSTFLAGS_TSAN := $(if $(findstring -fsanitize=thread,$(CXXFLAGS)),-Zsanitizer=thread,)
+RUSTFLAGS_SANI := $(RUSTFLAGS_ASAN) $(RUSTFLAGS_TSAN)
+CARGOFLAGS_BUILDSTD := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),-Zbuild-std,)
+RUSTFLAGS_CFGS := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),--cfg curve25519_dalek_backend=\"serial\",)
+
+if UNIFIED_RUST
+
+$(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(RUST_TOOLCHAIN_FILE)
+	rm -rf $(abspath $(RUST_TARGET_DIR))
+	CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \
+	RUSTFLAGS="$(RUSTFLAGS_SANI) $(RUSTFLAGS_CFGS)" \
+	CARGO_NET_GIT_FETCH_WITH_CLI=true \
+	$(CARGO) rustc \
+		$(CARGOFLAGS_BUILDSTD) \
+		--package stellar-core \
+		--target $(RUST_TARGET) \
+		--features unified \
+		$(RUST_PROFILE_ARG) \
+		--locked \
+		--target-dir $(abspath $(RUST_TARGET_DIR)) \
+		$(CARGO_FEATURE_TRACY) $(CARGO_FEATURE_NEXT) $(CARGO_FEATURE_TESTUTILS)
+	ranlib $@
+
+else # !UNIFIED_RUST
+
 # This next build command looks a little weird but it's necessary. We have to
 # provide an auxiliary metadata string (using RUSTFLAGS=-Cmetadata=$*)
 # essentially manually marking-as-different the separate dependency trees
@@ -292,10 +320,11 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) Makefile $(RUST_DE
 		esac ; \
 		cd $(abspath $(RUST_BUILD_DIR))/soroban/$$proto && \
 		CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \
-		RUSTFLAGS="-Cmetadata=$$proto $(RUSTFLAGS_ASAN)" \
+		RUSTFLAGS="-Cmetadata=$$proto" \
 		CARGO_NET_GIT_FETCH_WITH_CLI=true \
 		$(CARGO) build \
 			--package soroban-env-host \
+			--target $(RUST_TARGET) \
 			$(RUST_PROFILE_ARG) \
 			--locked \
 			$$FEATURE_FLAGS \
@@ -319,10 +348,10 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) Makefile $(RUST_DE
 $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(SOROBAN_LIBS_STAMP) $(RUST_TOOLCHAIN_FILE)
 	rm -rf $(abspath $(RUST_TARGET_DIR))
 	CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \
-	RUSTFLAGS="$(RUSTFLAGS_ASAN)" \
 	CARGO_NET_GIT_FETCH_WITH_CLI=true \
 	$(CARGO) rustc \
 		--package stellar-core \
+		--target $(RUST_TARGET) \
 		$(RUST_PROFILE_ARG) \
 		--locked \
 		--target-dir $(abspath $(RUST_TARGET_DIR)) \
@@ -332,6 +361,8 @@ $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(SORO
 		$(ALL_SOROBAN_DEPEND_ARGS)
 	ranlib $@
 
+endif # UNIFIED_RUST
+
 $(srcdir)/src.mk: $(top_srcdir)/make-mks
 	cd $(top_srcdir) && ./make-mks
 

src/rust/Cargo.toml

@@ -93,36 +93,39 @@ tracy-client = { version = "=0.17.0", features = [
 # the host dependencies. Make sure they are commented back out before committing
 # (and be careful to reset any changes the IDE makes to Cargo.lock)
 
-# [dependencies.soroban-env-host-p23]
-# version = "=23.0.0"
-# git = "https://github.com/stellar/rs-soroban-env"
-# package = "soroban-env-host"
-# rev = "31cb455b87a25a1c049360b5422c411deae63ba2"
-
-# [dependencies.soroban-env-host-p22]
-# version = "=22.0.0"
-# git = "https://github.com/stellar/rs-soroban-env"
-# package = "soroban-env-host"
-# rev = "1cd8b8dca9aeeca9ce45b129cd923992b32dc258"
-
-# [dependencies.soroban-env-host-p21]
-# version = "=21.2.2"
-# git = "https://github.com/stellar/rs-soroban-env"
-# package = "soroban-env-host"
-# rev = "7eeddd897cfb0f700f938b0c8d6f0541150d1fcb"
+[dependencies.soroban-env-host-p23]
+version = "=23.0.0-rc.2"
+git = "https://github.com/stellar/rs-soroban-env"
+package = "soroban-env-host"
+rev = "339d5957d2a258ca888d2f7eb50e92e425555256"
+optional = true
+
+[dependencies.soroban-env-host-p22]
+version = "=22.0.0"
+git = "https://github.com/stellar/rs-soroban-env"
+package = "soroban-env-host"
+rev = "80ebd78a53d608f76f4a5955fd9b03248efcb7c1"
+optional = true
+
+[dependencies.soroban-env-host-p21]
+version = "=21.2.2"
+git = "https://github.com/stellar/rs-soroban-env"
+package = "soroban-env-host"
+rev = "7eeddd897cfb0f700f938b0c8d6f0541150d1fcb"
+optional = true
 
 # The test wasms and synth-wasm crate should usually be taken from the highest
 # supported host, since test material usually just grows over time.
 
 [dependencies.soroban-test-wasms]
 version = "=22.0.0"
 git = "https://github.com/stellar/rs-soroban-env"
-rev = "a3f7fca9c2ad89796c7525a648da086543502dd5"
+rev = "80ebd78a53d608f76f4a5955fd9b03248efcb7c1"
 
 [dependencies.soroban-synth-wasm]
 version = "=22.0.0"
 git = "https://github.com/stellar/rs-soroban-env"
-rev = "a3f7fca9c2ad89796c7525a648da086543502dd5"
+rev = "80ebd78a53d608f76f4a5955fd9b03248efcb7c1"
 
 [dependencies.stellar-quorum-analyzer]
 version = "0.1.0"
@@ -131,6 +134,8 @@ rev = "678acf18ee635c4270e241030d6e83bb0b98b5d5"
 
 [features]
 
+unified = ["dep:soroban-env-host-p21", "dep:soroban-env-host-p22", "dep:soroban-env-host-p23"]
+
 tracy = ["dep:tracy-client"]
 
 # The "next" feature enables not-yet-released features that (a) break protocol