Merge pull request #1649 from input-output-hk/jpraynaud/1629-fix-memory-fragmentation

Use new memory allocator 'jemallocator'

Author: jpraynaud

CHANGELOG.md

@@ -22,6 +22,8 @@ As a minor extension, we have adopted a slightly different versioning convention
    - `portable` feature now has no effect and should be removed from crate dependencies.
    - Removed it from all other crates (including `mithril-common`).
 
+- Switched memory allocator to `jemallocator` on signer and aggregator to avoid memory fragmentation when signing transactions (which lead to RES memory not being properly returned to the OS).
+
 - Crates versions:
 
 |  Crate  |  Version  |

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.61"
+version = "0.4.62"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -46,6 +46,9 @@ uuid = { version = "1.7.0", features = ["v4", "fast-rng", "macro-diagnostics"] }
 warp = "0.3.6"
 zstd = { version = "0.13.0", features = ["zstdmt"] }
 
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+tikv-jemallocator = { version = "0.5.4", optional = true }
+
 [dev-dependencies]
 httpmock = "0.7.0"
 mithril-common = { path = "../mithril-common", features = [
@@ -57,4 +60,7 @@ slog-term = "2.9.0"
 tempfile = "3.9.0"
 
 [features]
+default = ["jemallocator"]
+
 bundle_openssl = ["dep:openssl", "dep:openssl-probe"]
+jemallocator = ["dep:tikv-jemallocator"]

mithril-aggregator/Cargo.toml

@@ -64,6 +64,14 @@ pub use tools::{
 #[cfg(test)]
 pub use dependency_injection::tests::initialize_dependencies;
 
+// Memory allocator (to handle properly memory fragmentation)
+#[cfg(all(not(target_env = "msvc"), feature = "jemallocator"))]
+use tikv_jemallocator::Jemalloc;
+
+#[cfg(all(not(target_env = "msvc"), feature = "jemallocator"))]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
 #[cfg(test)]
 pub(crate) mod test_tools {
     use slog::Drain;

mithril-aggregator/src/lib.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-signer"
-version = "0.2.125"
+version = "0.2.126"
 description = "A Mithril Signer"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -38,6 +38,9 @@ sqlite = { version = "0.34.0", features = ["bundled"] }
 thiserror = "1.0.56"
 tokio = { version = "1.35.1", features = ["full"] }
 
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+tikv-jemallocator = { version = "0.5.4", optional = true }
+
 [dev-dependencies]
 httpmock = "0.7.0"
 mithril-common = { path = "../mithril-common" }
@@ -46,4 +49,7 @@ prometheus-parse = "0.2.5"
 slog-term = "2.9.0"
 
 [features]
+default = ["jemallocator"]
+
 bundle_openssl = ["dep:openssl", "dep:openssl-probe"]
+jemallocator = ["dep:tikv-jemallocator"]

mithril-signer/Cargo.toml

@@ -36,6 +36,14 @@ const HTTP_REQUEST_TIMEOUT_DURATION: u64 = 30000;
 const SQLITE_FILE: &str = "signer.sqlite3";
 const SQLITE_FILE_CARDANO_TRANSACTION: &str = "cardano-transaction.sqlite3";
 
+// Memory allocator (to handle properly memory fragmentation)
+#[cfg(all(not(target_env = "msvc"), feature = "jemallocator"))]
+use tikv_jemallocator::Jemalloc;
+
+#[cfg(all(not(target_env = "msvc"), feature = "jemallocator"))]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
 #[cfg(test)]
 pub mod test_tools {
     use slog::Drain;