Merge branch 'develop' of https://github.com/stacks-network/stacks-core into feat/signers-read-stackerdb

Author: jferrant

.cargo/config

@@ -2,12 +2,10 @@
 stacks-node = "run --package stacks-node --"
 fmt-stacks = "fmt -- --config group_imports=StdExternalCrate,imports_granularity=Module"
 
-# Default to `native`
-# This makes it slightly faster for running tests and locally built binaries.
-# This can cause trouble when building "portable" binaries, such as for docker,
-# so disable it with the "portable" feature.
-[target.'cfg(not(feature = "portable"))']
-rustflags = ["-Ctarget-cpu=native"]
+# Uncomment to improve performance slightly, at the cost of portability
+#   * Note that native binaries may not run on CPUs that are different from the build machine
+# [build]
+# rustflags = ["-Ctarget-cpu=native"]
 
 # Needed by perf to generate flamegraphs.
 #[target.x86_64-unknown-linux-gnu]

.github/actions/dockerfiles/Dockerfile.debian-source

@@ -19,7 +19,7 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
     && cd ${BUILD_DIR} \
     && rustup target add ${TARGET} \
     && rustup component add rustfmt \
-    && cargo build --features monitoring_prom,slog_json,portable --release --workspace --target ${TARGET} \
+    && cargo build --features monitoring_prom,slog_json --release --workspace --target ${TARGET} \
     && mkdir -p /out \
     && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
 

CHANGELOG.md

@@ -363,7 +363,7 @@ The changelog for this release is a high-level summary of these SIPs.
 ### Added
 
 - Added prometheus output for "transactions in last block" (#3138).
-- Added envrionement variable STACKS_LOG_FORMAT_TIME to set the time format
+- Added environment variable STACKS_LOG_FORMAT_TIME to set the time format
   stacks-node uses for logging. (#3219)
   Example: STACKS_LOG_FORMAT_TIME="%Y-%m-%d %H:%M:%S" cargo stacks-node
 - Added mock-miner sample config (#3225)

Cargo.lock

@@ -12,7 +12,7 @@ RUN apk add --no-cache musl-dev
 
 RUN mkdir /out
 
-RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json,portable --release
+RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json --release
 
 RUN cp target/release/stacks-node /out
 

Dockerfile

@@ -10,7 +10,7 @@ COPY . .
 
 RUN mkdir /out
 
-RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json,portable --release
+RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json --release
 
 RUN cp target/release/stacks-node /out
 

Dockerfile.debian

@@ -49,6 +49,19 @@ cargo build --release
 cargo build --profile release-lite
 ```
 
+_Note on building_: you may set `RUSTFLAGS` to build binaries for your native cpu:
+
+```
+RUSTFLAGS="-Ctarget-cpu=native"
+```
+
+or uncomment these lines in `./cargo/config`:
+
+```
+# [build]
+# rustflags = ["-Ctarget-cpu=native"]
+```
+
 ## Testing
 
 **Run the tests:**

README.md

@@ -33,6 +33,7 @@ tiny_http = "0.12"
 wsts = { workspace = true }
 
 [dev-dependencies]
+mutants = "0.0.3"
 rand_core = { workspace = true }
 rand = { workspace = true }
 

libsigner/Cargo.toml

@@ -38,6 +38,7 @@ use blockstack_lib::net::api::postblock_proposal::{
     BlockValidateReject, BlockValidateResponse, ValidateRejectCode,
 };
 use blockstack_lib::util_lib::boot::boot_code_id;
+use clarity::util::retry::BoundReader;
 use clarity::vm::types::serialization::SerializationError;
 use clarity::vm::types::QualifiedContractIdentifier;
 use hashbrown::{HashMap, HashSet};
@@ -94,14 +95,17 @@ MessageSlotID {
     /// Transactions list for miners and signers to observe
     Transactions = 11,
     /// DKG Results
-    DkgResults = 12
+    DkgResults = 12,
+    /// Persisted encrypted signer state containing DKG shares
+    EncryptedSignerState = 13
 });
 
 define_u8_enum!(SignerMessageTypePrefix {
     BlockResponse = 0,
     Packet = 1,
     Transactions = 2,
-    DkgResults = 3
+    DkgResults = 3,
+    EncryptedSignerState = 4
 });
 
 impl MessageSlotID {
@@ -136,12 +140,14 @@ impl TryFrom<u8> for SignerMessageTypePrefix {
 }
 
 impl From<&SignerMessage> for SignerMessageTypePrefix {
+    #[cfg_attr(test, mutants::skip)]
     fn from(message: &SignerMessage) -> Self {
         match message {
             SignerMessage::Packet(_) => SignerMessageTypePrefix::Packet,
             SignerMessage::BlockResponse(_) => SignerMessageTypePrefix::BlockResponse,
             SignerMessage::Transactions(_) => SignerMessageTypePrefix::Transactions,
             SignerMessage::DkgResults { .. } => SignerMessageTypePrefix::DkgResults,
+            SignerMessage::EncryptedSignerState(_) => SignerMessageTypePrefix::EncryptedSignerState,
         }
     }
 }
@@ -234,9 +240,12 @@ pub enum SignerMessage {
         /// The polynomial commits used to construct the aggregate key
         party_polynomials: Vec<(u32, PolyCommitment)>,
     },
+    /// The encrypted state of the signer to be persisted
+    EncryptedSignerState(Vec<u8>),
 }
 
 impl Debug for SignerMessage {
+    #[cfg_attr(test, mutants::skip)]
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::BlockResponse(b) => Debug::fmt(b, f),
@@ -255,12 +264,16 @@ impl Debug for SignerMessage {
                     .field("party_polynomials", &party_polynomials)
                     .finish()
             }
+            Self::EncryptedSignerState(s) => {
+                f.debug_tuple("EncryptedSignerState").field(s).finish()
+            }
         }
     }
 }
 
 impl SignerMessage {
     /// Helper function to determine the slot ID for the provided stacker-db writer id
+    #[cfg_attr(test, mutants::skip)]
     pub fn msg_id(&self) -> MessageSlotID {
         match self {
             Self::Packet(packet) => match packet.msg {
@@ -278,6 +291,7 @@ impl SignerMessage {
             Self::BlockResponse(_) => MessageSlotID::BlockResponse,
             Self::Transactions(_) => MessageSlotID::Transactions,
             Self::DkgResults { .. } => MessageSlotID::DkgResults,
+            Self::EncryptedSignerState(_) => MessageSlotID::EncryptedSignerState,
         }
     }
 }
@@ -345,10 +359,14 @@ impl StacksMessageCodec for SignerMessage {
                     party_polynomials.iter().map(|(a, b)| (a, b)),
                 )?;
             }
+            SignerMessage::EncryptedSignerState(encrypted_state) => {
+                write_next(fd, encrypted_state)?;
+            }
         };
         Ok(())
     }
 
+    #[cfg_attr(test, mutants::skip)]
     fn consensus_deserialize<R: Read>(fd: &mut R) -> Result<Self, CodecError> {
         let type_prefix_byte = read_next::<u8, _>(fd)?;
         let type_prefix = SignerMessageTypePrefix::try_from(type_prefix_byte)?;
@@ -383,6 +401,15 @@ impl StacksMessageCodec for SignerMessage {
                     party_polynomials,
                 }
             }
+            SignerMessageTypePrefix::EncryptedSignerState => {
+                // Typically the size of the signer state is much smaller, but in the fully degenerate case the size of the persisted state is
+                // 2800 * 32 * 4 + C for some small constant C.
+                // To have some margin, we're expanding the left term with an additional factor 4
+                let max_encrypted_state_size = 2800 * 32 * 4 * 4;
+                let mut bound_reader = BoundReader::from_reader(fd, max_encrypted_state_size);
+                let encrypted_state = read_next::<_, _>(&mut bound_reader)?;
+                SignerMessage::EncryptedSignerState(encrypted_state)
+            }
         };
         Ok(message)
     }