Merge pull request #1344 from input-output-hk/djo/1311/feature_for_snapshot_download

Mithril-client-lib: Gate snapshot download & message computation behind a feature

Author: jpraynaud

.github/workflows/actions/publish-crate-package/action.yml

@@ -11,6 +11,9 @@ inputs:
   api_token: 
     description: crates.io API token.
     required: false
+  publish_args:
+    description: Additional arguments that will be passed to 'cargo publish'
+    required: false
 
 runs:
   using: "composite"
@@ -36,18 +39,18 @@ runs:
   - name: Cargo publish dry run
     shell: bash
     run: |
-      echo "Cargo publish '${{ inputs.package}}' package (dry run)"
-      cargo publish -p ${{ inputs.package}} --dry-run --no-verify
+      echo "Cargo publish '${{ inputs.package }}' package (dry run)"
+      cargo publish -p ${{ inputs.package }} --dry-run --no-verify ${{ inputs.publish_args }} 
 
   - name: Cargo package list
     shell: bash
     run: |
-      echo "Cargo package list '${{ inputs.package}}' package"
-      cargo package -p ${{ inputs.package}} --list
+      echo "Cargo package list '${{ inputs.package }}' package"
+      cargo package -p ${{ inputs.package }} --list
 
   - name: Cargo publish
     if: inputs.dry_run == 'false' && steps.check_version.outputs.should_deploy == 'true'
     shell: bash
     run: |
-      echo "Cargo publish '${{ inputs.package}}' package"
-      cargo publish -p ${{ inputs.package}} --token ${{ inputs.api_token }} --no-verify
+      echo "Cargo publish '${{ inputs.package }}' package"
+      cargo publish -p ${{ inputs.package }} --token ${{ inputs.api_token }} --no-verify ${{ inputs.publish_args }}

.github/workflows/ci.yml

@@ -46,7 +46,7 @@ jobs:
         uses: ./.github/workflows/actions/build-upload-mithril-artifact
         with:
           binaries-build-args: --bin mithril-aggregator --bin mithril-signer --bin mithril-client --bin mithril-relay --features bundle_openssl
-          libraries-build-args: --package mithril-stm --package mithril-client
+          libraries-build-args: --package mithril-stm --package mithril-client --features full
 
       - name: Build Debian packages
         shell: bash
@@ -87,12 +87,12 @@ jobs:
         # Only build client on windows & mac
         - os: macos-12
           binaries-build-args: --bin mithril-client --features bundle_openssl
-          libraries-build-args: --package mithril-stm --package mithril-client
+          libraries-build-args: --package mithril-stm --package mithril-client --features full
         - os: windows-latest
           # Use `--bins --package <package>` instead of `--bin <package>`, otherwise the 'windows' compatibility
           # hack in mithril common cargo.toml doesn't apply (we have no idea why).
           binaries-build-args: --bins --package mithril-client-cli --features bundle_openssl
-          libraries-build-args: --package mithril-stm --package mithril-client --no-default-features --features num-integer-backend
+          libraries-build-args: --package mithril-stm --package mithril-client --no-default-features --features num-integer-backend,full
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -119,12 +119,12 @@ jobs:
 
         include:
           - os: ubuntu-22.04
-            test-args: --features portable --workspace
+            test-args: --features portable,full --workspace
           # Only test client on windows & mac (since its the only binaries supported for those os for now)
           - os: macos-12
-            test-args: --package mithril-client --package mithril-client-cli
+            test-args: --package mithril-client --package mithril-client-cli --features full
           - os: windows-latest
-            test-args: --package mithril-client --package mithril-client-cli
+            test-args: --package mithril-client --package mithril-client-cli --features full
 
     runs-on: ${{ matrix.os }}
 

.github/workflows/docs.yml

@@ -27,7 +27,8 @@ jobs:
       - name: Generate cargo doc
         run: |
           cargo doc --no-deps -p mithril-stm -p mithril-common -p mithril-aggregator \
-                    -p mithril-signer -p mithril-client -p mithril-client-cli --message-format=json \
+                    -p mithril-signer -p mithril-client -p mithril-client-cli \
+                    --all-features --message-format=json \
                     | clippy-sarif | tee rust-cargo-doc-results.sarif | sarif-fmt
           
           # Update tool sarif metadata from "clippy" to "cargo-doc" (since it's set this way by clippy-sarif)

Cargo.lock

@@ -117,7 +117,7 @@ Snapshot download and certificate chain validation can take quite some time even
 An example of implementation with the crate [indicatif](https://crates.io/crates/indicatif) is available in the [Mithril repository](https://github.com/input-output-hk/mithril/tree/main/mithril-client/examples/snapshot_list_get_show_download_verify.rs). To run it, execute the following command:
 
 ```bash
-cargo run --example snapshot_list_get_show_download_verify 
+cargo run --example snapshot_list_get_show_download_verify --features fs
 ```
 
 :::

docs/website/root/manual/developer-docs/nodes/mithril-client-library.md

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client"
-version = "0.5.6"
+version = "0.5.7"
 description = "Mithril client library"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -13,25 +13,35 @@ include = ["**/*.rs", "Cargo.toml", "README.md", ".gitignore"]
 [lib]
 crate-type = ["lib", "cdylib", "staticlib"]
 
+[[example]]
+name = "snapshot_list_get_show_download_verify"
+path = "examples/snapshot_list_get_show_download_verify.rs"
+required-features = ["fs"]
+
+[[test]]
+name = "snapshot_list_get_show_download_verify"
+path = "tests/snapshot_list_get_show_download_verify.rs"
+required-features = ["fs"]
+
 [dependencies]
 anyhow = "1.0.75"
 async-recursion = "1.0.5"
 async-trait = "0.1.73"
 chrono = { version = "0.4.31", features = ["serde"] }
-flate2 = "1.0.27"
-flume = "0.11.0"
+flate2 = { version = "1.0.27", optional = true }
+flume = { version = "0.11.0", optional = true }
 futures = "0.3.28"
 mithril-common = { path = "../mithril-common", version = "0.2" }
 reqwest = { version = "0.11.22", features = ["json", "stream"] }
 semver = "1.0.19"
 serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0.107"
 slog = "2.7.0"
-tar = "0.4.40"
+tar = { version = "0.4.40", optional = true }
 thiserror = "1.0.49"
 tokio = { version = "1.32.0", features = ["sync"] }
 uuid = { version = "1.5.0", features = ["v4"] }
-zstd = "0.13.0"
+zstd = { version = "0.13.0", optional = true }
 
 [dev-dependencies]
 httpmock = "0.6.8"
@@ -47,4 +57,15 @@ tokio = { version = "1.32.0", features = ["macros", "rt"] }
 warp = "0.3"
 
 [features]
+# Include nothing by default
+# TODO: rollback to default = []
+default = ["full"]
+full = ["fs"]
+# Enable file system releated functionnality, right now that mean ony snapshot download
+fs = ["flate2", "flume", "tar", "tokio/rt", "zstd"]
 portable = ["mithril-common/portable"]
+
+[package.metadata.docs.rs]
+all-features = true
+# enable unstable features in the documentation
+rustdoc-args = ["--cfg", "docsrs"]

mithril-client/Cargo.toml

@@ -11,10 +11,10 @@ all: test build
 
 build:
 	# We use 'portable' feature to avoid SIGILL crashes
-	${CARGO} build --release --features portable
+	${CARGO} build --release --features portable,full
 
 test:
-	${CARGO} test
+	${CARGO} test --features full
 
 check:
 	${CARGO} check --release --all-features --all-targets
@@ -25,4 +25,4 @@ clean:
 	${CARGO} clean
 
 doc:
-	${CARGO} doc --no-deps --open
+	${CARGO} doc --no-deps --open --features full

mithril-client/Makefile

@@ -5,6 +5,7 @@ use crate::certificate_client::{
 use crate::feedback::{FeedbackReceiver, FeedbackSender};
 use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient;
 use crate::snapshot_client::SnapshotClient;
+#[cfg(feature = "fs")]
 use crate::snapshot_downloader::{HttpSnapshotDownloader, SnapshotDownloader};
 use crate::MithrilResult;
 use anyhow::{anyhow, Context};
@@ -45,6 +46,7 @@ pub struct ClientBuilder {
     genesis_verification_key: String,
     aggregator_client: Option<Arc<dyn AggregatorClient>>,
     certificate_verifier: Option<Arc<dyn CertificateVerifier>>,
+    #[cfg(feature = "fs")]
     snapshot_downloader: Option<Arc<dyn SnapshotDownloader>>,
     logger: Option<Logger>,
     feedback_receivers: Vec<Arc<dyn FeedbackReceiver>>,
@@ -59,6 +61,7 @@ impl ClientBuilder {
             genesis_verification_key: genesis_verification_key.to_string(),
             aggregator_client: None,
             certificate_verifier: None,
+            #[cfg(feature = "fs")]
             snapshot_downloader: None,
             logger: None,
             feedback_receivers: vec![],
@@ -75,6 +78,7 @@ impl ClientBuilder {
             genesis_verification_key: genesis_verification_key.to_string(),
             aggregator_client: None,
             certificate_verifier: None,
+            #[cfg(feature = "fs")]
             snapshot_downloader: None,
             logger: None,
             feedback_receivers: vec![],
@@ -115,6 +119,7 @@ impl ClientBuilder {
             Some(client) => client,
         };
 
+        #[cfg(feature = "fs")]
         let snapshot_downloader = match self.snapshot_downloader {
             None => Arc::new(
                 HttpSnapshotDownloader::new(feedback_sender.clone(), logger.clone())
@@ -146,8 +151,11 @@ impl ClientBuilder {
         ));
         let snapshot_client = Arc::new(SnapshotClient::new(
             aggregator_client,
+            #[cfg(feature = "fs")]
             snapshot_downloader,
+            #[cfg(feature = "fs")]
             feedback_sender,
+            #[cfg(feature = "fs")]
             logger,
         ));
 
@@ -176,6 +184,7 @@ impl ClientBuilder {
         self
     }
 
+    cfg_fs! {
     /// Set the [SnapshotDownloader] that will be used to download snapshots.
     pub fn with_snapshot_downloader(
         mut self,
@@ -184,6 +193,7 @@ impl ClientBuilder {
         self.snapshot_downloader = Some(snapshot_downloader);
         self
     }
+    }
 
     /// Set the [Logger] to use.
     pub fn with_logger(mut self, logger: Logger) -> Self {

mithril-client/src/client.rs

@@ -1,4 +1,5 @@
 #![warn(missing_docs)]
+#![cfg_attr(docsrs, feature(doc_cfg))]
 
 //! Define all the tooling necessary to manipulate Mithril certified types from a
 //! [Mithril Aggregator](https://mithril.network/rust-doc/mithril_aggregator/index.html).
@@ -19,6 +20,8 @@
 //!
 //! Below is an example describing the usage of most of the library's functions together:
 //!
+//! **Note:** _Snapshot download and the compute snapshot message functions are available using crate feature_ **fs**.
+//!
 //! ```no_run
 //! # async fn run() -> mithril_client::MithrilResult<()> {
 //! use mithril_client::{ClientBuilder, MessageBuilder};
@@ -52,14 +55,26 @@
 //! # }
 //! ```
 
+macro_rules! cfg_fs {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "fs")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
+            $item
+        )*
+    }
+}
+
 pub mod aggregator_client;
 pub mod certificate_client;
 mod client;
 pub mod feedback;
 mod message;
 pub mod mithril_stake_distribution_client;
 pub mod snapshot_client;
-pub mod snapshot_downloader;
+cfg_fs! {
+    pub mod snapshot_downloader;
+}
 mod type_alias;
 mod utils;
 

mithril-client/src/lib.rs

@@ -1,16 +1,22 @@
 use anyhow::Context;
+#[cfg(feature = "fs")]
 use mithril_common::digesters::{CardanoImmutableDigester, ImmutableDigester};
 use mithril_common::entities::{ProtocolMessage, ProtocolMessagePartKey};
 use mithril_common::messages::SignerWithStakeMessagePart;
 use mithril_common::protocol::SignerBuilder;
 use slog::{o, Logger};
+#[cfg(feature = "fs")]
 use std::path::Path;
+#[cfg(feature = "fs")]
 use std::sync::Arc;
 
-use crate::{MithrilCertificate, MithrilResult, MithrilStakeDistribution};
+#[cfg(feature = "fs")]
+use crate::MithrilCertificate;
+use crate::{MithrilResult, MithrilStakeDistribution};
 
 /// A [MessageBuilder] can be used to compute the message of Mithril artifacts.
 pub struct MessageBuilder {
+    #[cfg(feature = "fs")]
     immutable_digester: Option<Arc<dyn ImmutableDigester>>,
     logger: Logger,
 }
@@ -20,35 +26,37 @@ impl MessageBuilder {
     pub fn new() -> MessageBuilder {
         let logger = Logger::root(slog::Discard, o!());
         Self {
+            #[cfg(feature = "fs")]
             immutable_digester: None,
             logger,
         }
     }
 
-    /// Set the [ImmutableDigester] to be used for the message computation for snapshot.
-    ///
-    /// If not set a default implementation will be used.
-    pub fn with_immutable_digester(
-        mut self,
-        immutable_digester: Arc<dyn ImmutableDigester>,
-    ) -> Self {
-        self.immutable_digester = Some(immutable_digester);
-        self
-    }
-
     /// Set the [Logger] to use.
     pub fn with_logger(mut self, logger: Logger) -> Self {
         self.logger = logger;
         self
     }
 
+    cfg_fs! {
     fn get_immutable_digester(&self) -> Arc<dyn ImmutableDigester> {
         match self.immutable_digester.as_ref() {
             None => Arc::new(CardanoImmutableDigester::new(None, self.logger.clone())),
             Some(digester) => digester.clone(),
         }
     }
 
+    /// Set the [ImmutableDigester] to be used for the message computation for snapshot.
+    ///
+    /// If not set a default implementation will be used.
+    pub fn with_immutable_digester(
+        mut self,
+        immutable_digester: Arc<dyn ImmutableDigester>,
+    ) -> Self {
+        self.immutable_digester = Some(immutable_digester);
+        self
+    }
+
     /// Compute message for a snapshot (based on the directory where it was unpacked).
     ///
     /// Warning: this operation can be quite long depending on the snapshot size.
@@ -74,6 +82,7 @@ impl MessageBuilder {
 
         Ok(message)
     }
+    }
 
     /// Compute message for a Mithril stake distribution.
     pub fn compute_mithril_stake_distribution_message(