Merge pull request #788 from input-output-hk/jpraynaud/786-fix-datum-generation-era-markers

jpraynaud · web-flow · commit f3a5e393a259 · 2023-03-08T10:22:47.000+01:00
Fix Datum generation for era markers
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.2.26"
+version = "0.2.27"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
diff --git a/mithril-aggregator/src/tools/era.rs b/mithril-aggregator/src/tools/era.rs
@@ -4,10 +4,7 @@ use mithril_common::{
     chain_observer::{TxDatumBuilder, TxDatumFieldValue},
     crypto_helper::{key_encode_hex, EraMarkersSigner},
     entities::Epoch,
-    era::{
-        adapters::{EraMarkerItemCardanoChain, EraMarkersPayloadCardanoChain},
-        SupportedEra,
-    },
+    era::{adapters::EraMarkersPayloadCardanoChain, EraMarker, SupportedEra},
 };
 
 type EraToolsResult<R> = Result<R, Box<dyn Error>>;
@@ -40,8 +37,8 @@ impl EraTools {
         let mut era_markers = Vec::new();
         for (index, era) in SupportedEra::eras().iter().enumerate() {
             let era_marker = match index {
-                0 => EraMarkerItemCardanoChain::new(&era.to_string(), Some(current_era_epoch)),
-                1 => EraMarkerItemCardanoChain::new(&era.to_string(), maybe_next_era_epoch),
+                0 => EraMarker::new(&era.to_string(), Some(current_era_epoch)),
+                1 => EraMarker::new(&era.to_string(), maybe_next_era_epoch),
                 _ => Err("too many eras retrieved, can't generate tx datum".to_string())?,
             };
             era_markers.push(era_marker);
@@ -54,15 +51,10 @@ impl EraTools {
 
         let tx_datum = TxDatumBuilder::new()
             .add_field(TxDatumFieldValue::Bytes(
-                key_encode_hex(era_markers_payload.markers).map_err(|e| {
-                    format!("era markers payload markers could not be hex encoded: {e}")
-                })?,
-            ))
-            .add_field(TxDatumFieldValue::Bytes(
-                era_markers_payload.signature.unwrap_or_default(),
+                key_encode_hex(era_markers_payload)
+                    .map_err(|e| format!("era markerspayload could not be hex encoded: {e}"))?,
             ))
             .build()?;
-
         Ok(tx_datum.0)
     }
 }
diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mithril-common"
-version = "0.2.21"
+version = "0.2.22"
 authors = { workspace = true }
 edition = { workspace = true }
 documentation = { workspace = true }
diff --git a/mithril-common/src/chain_observer/model.rs b/mithril-common/src/chain_observer/model.rs
@@ -24,12 +24,11 @@ pub enum TxDatumError {
 pub struct TxDatum(pub String);
 
 impl TxDatum {
-    /// Retrieves the nth field of the datum with given type
-    pub fn get_nth_field_by_type(
+    /// Retrieves the fields of the datum with given type
+    pub fn get_fields_by_type(
         &self,
         type_name: &TxDatumFieldTypeName,
-        index: usize,
-    ) -> Result<Value, Box<dyn StdError>> {
+    ) -> Result<Vec<Value>, Box<dyn StdError>> {
         let tx_datum_raw = &self.0;
         // 1- Parse the Utxo raw data to a hashmap
         let v: HashMap<String, Value> = serde_json::from_str(tx_datum_raw).map_err(|e| {
@@ -48,22 +47,28 @@ impl TxDatum {
             )
         })?;
         // 3- Filter the vec (keep the ones that match the given type), and retrieve the nth entry of this filtered vec
-        let field_value = fields
+        Ok(fields
             .iter()
             .filter(|&field| field.get(type_name.to_string()).is_some())
-            .nth(index)
+            .map(|field| field.get(type_name.to_string()).unwrap().to_owned())
+            .collect::<_>())
+    }
+
+    /// Retrieves the nth field of the datum with given type
+    pub fn get_nth_field_by_type(
+        &self,
+        type_name: &TxDatumFieldTypeName,
+        index: usize,
+    ) -> Result<Value, Box<dyn StdError>> {
+        Ok(self
+            .get_fields_by_type(type_name)?
+            .get(index)
             .ok_or_else(|| {
                 TxDatumError::InvalidContent(
-                    format!(
-                        "Error: missing field at index {index}, tx datum was = '{tx_datum_raw}'"
-                    )
-                    .into(),
+                    format!("Error: missing field at index {index}").into(),
                 )
             })?
-            .get(type_name.to_string())
-            .unwrap();
-
-        Ok(field_value.to_owned())
+            .to_owned())
     }
 }
 
@@ -101,9 +106,29 @@ impl TxDatumBuilder {
 
     /// Add a field to the builder
     pub fn add_field(&mut self, field_value: TxDatumFieldValue) -> &mut TxDatumBuilder {
-        let mut field = HashMap::new();
-        field.insert(TxDatumFieldTypeName::from(&field_value), field_value);
-        self.fields.push(field);
+        match &field_value {
+            TxDatumFieldValue::Bytes(datum_str) => {
+                // TODO: Remove this chunking of the bytes fields once the cardano-cli 1.36.0+ is released
+                // The bytes fields are currently limited to 128 bytes and need to be chunked in multiple fields
+                let field_type = TxDatumFieldTypeName::from(&field_value);
+                let field_value_chunks = datum_str.as_bytes().chunks(128);
+                for field_value_chunk in field_value_chunks {
+                    let mut field = HashMap::new();
+                    field.insert(
+                        field_type,
+                        TxDatumFieldValue::Bytes(
+                            std::str::from_utf8(field_value_chunk).unwrap().to_string(),
+                        ),
+                    );
+                    self.fields.push(field);
+                }
+            }
+            _ => {
+                let mut field = HashMap::new();
+                field.insert(TxDatumFieldTypeName::from(&field_value), field_value);
+                self.fields.push(field);
+            }
+        }
 
         self
     }
@@ -135,6 +160,7 @@ mod test {
             .add_field(TxDatumFieldValue::Bytes("bytes1".to_string()))
             .add_field(TxDatumFieldValue::Bytes("bytes2".to_string()))
             .add_field(TxDatumFieldValue::Int(2))
+            .add_field(TxDatumFieldValue::Bytes("012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789".to_string()))
             .build()
             .expect("tx_datum build should not fail");
         tx_datum
@@ -143,7 +169,7 @@ mod test {
     #[test]
     fn test_build_tx_datum() {
         let tx_datum = dummy_tx_datum();
-        let tx_datum_expected = TxDatum(r#"{"constructor":0,"fields":[{"bytes":"bytes0"},{"int":0},{"int":1},{"bytes":"bytes1"},{"bytes":"bytes2"},{"int":2}]}"#.to_string());
+        let tx_datum_expected = TxDatum(r#"{"constructor":0,"fields":[{"bytes":"bytes0"},{"int":0},{"int":1},{"bytes":"bytes1"},{"bytes":"bytes2"},{"int":2},{"bytes":"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567"},{"bytes":"8901234567890123456789"}]}"#.to_string());
         assert_eq!(tx_datum_expected, tx_datum);
     }
 
diff --git a/mithril-common/src/era/adapters/cardano_chain.rs b/mithril-common/src/era/adapters/cardano_chain.rs
@@ -4,7 +4,7 @@ use crate::{
         key_decode_hex, EraMarkersSigner, EraMarkersVerifier, EraMarkersVerifierSignature,
         EraMarkersVerifierVerificationKey,
     },
-    entities::{Epoch, HexEncodedEraMarkersSignature},
+    entities::HexEncodedEraMarkersSignature,
     era::{EraMarker, EraReaderAdapter},
 };
 use async_trait::async_trait;
@@ -40,45 +40,11 @@ pub enum EraMarkersPayloadError {
     CreateSignature(GeneralError),
 }
 
-/// Era marker item
-/// Value object that represents a tag of Era change.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct EraMarkerItem {
-    /// Era name
-    #[serde(rename = "n")]
-    pub name: String,
-
-    /// Eventual information that advertises the Epoch of transition.
-    #[serde(rename = "e")]
-    pub epoch: Option<Epoch>,
-}
-
-impl EraMarkerItem {
-    /// Instantiate a new [EraMarkerItem].
-    pub fn new(name: &str, epoch: Option<Epoch>) -> Self {
-        let name = name.to_string();
-
-        Self { name, epoch }
-    }
-}
-
-impl From<EraMarker> for EraMarkerItem {
-    fn from(other: EraMarker) -> EraMarkerItem {
-        EraMarkerItem::new(&other.name, other.epoch)
-    }
-}
-
-impl From<EraMarkerItem> for EraMarker {
-    fn from(other: EraMarkerItem) -> EraMarker {
-        EraMarker::new(&other.name, other.epoch)
-    }
-}
-
 /// Era markers payload
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct EraMarkersPayload {
     /// List of Era markers
-    pub markers: Vec<EraMarkerItem>,
+    pub markers: Vec<EraMarker>,
 
     /// Era markers signature
     pub signature: Option<HexEncodedEraMarkersSignature>,
@@ -162,27 +128,20 @@ impl EraReaderAdapter for CardanoChainAdapter {
             .await?;
         let markers_list = tx_datums
             .into_iter()
-            .filter_map(|datum| {
-                match (
-                    datum.get_nth_field_by_type(&TxDatumFieldTypeName::Bytes, 0),
-                    datum.get_nth_field_by_type(&TxDatumFieldTypeName::Bytes, 1),
-                ) {
-                    (Ok(markers), Ok(signature)) => {
-                        let markers = markers.as_str().map(|s| s.to_string()).unwrap_or_default();
-                        let signature = signature.as_str().map(|s| s.to_string());
-                        match key_decode_hex::<Vec<_>>(&markers) {
-                            Ok(markers) => EraMarkersPayload {
-                                markers: markers.clone(),
-                                signature,
-                            }
-                            .verify_signature(self.verification_key)
-                            .ok()
-                            .map(|_| markers.into_iter().map(|em| em.into()).collect()),
-                            Err(_) => None,
-                        }
-                    }
-                    _ => None,
-                }
+            .filter_map(|datum| datum.get_fields_by_type(&TxDatumFieldTypeName::Bytes).ok())
+            .map(|fields| {
+                fields
+                    .iter()
+                    .filter_map(|field_value| field_value.as_str().map(|s| s.to_string()))
+                    .collect::<Vec<String>>()
+                    .join("")
+            })
+            .filter_map(|field_value_str| key_decode_hex(&field_value_str).ok())
+            .filter_map(|era_markers_payload: EraMarkersPayload| {
+                era_markers_payload
+                    .verify_signature(self.verification_key)
+                    .ok()
+                    .map(|_| era_markers_payload.markers)
             })
             .collect::<Vec<Vec<EraMarker>>>();
 
@@ -203,12 +162,7 @@ mod test {
             .into_iter()
             .map(|payload| {
                 TxDatumBuilder::new()
-                    .add_field(TxDatumFieldValue::Bytes(
-                        key_encode_hex(payload.markers).unwrap(),
-                    ))
-                    .add_field(TxDatumFieldValue::Bytes(
-                        payload.signature.unwrap_or_default(),
-                    ))
+                    .add_field(TxDatumFieldValue::Bytes(key_encode_hex(payload).unwrap()))
                     .build()
                     .unwrap()
             })
@@ -221,15 +175,15 @@ mod test {
         let fake_address = "addr_test_123456".to_string();
         let era_marker_payload_1 = EraMarkersPayload {
             markers: vec![
-                EraMarkerItem::new("thales", Some(Epoch(1))),
-                EraMarkerItem::new("pythagoras", None),
+                EraMarker::new("thales", Some(Epoch(1))),
+                EraMarker::new("pythagoras", None),
             ],
             signature: None,
         };
         let era_marker_payload_2 = EraMarkersPayload {
             markers: vec![
-                EraMarkerItem::new("thales", Some(Epoch(1))),
-                EraMarkerItem::new("pythagoras", Some(Epoch(2))),
+                EraMarker::new("thales", Some(Epoch(1))),
+                EraMarker::new("pythagoras", Some(Epoch(2))),
             ],
             signature: None,
         };
@@ -252,11 +206,7 @@ mod test {
             .read()
             .await
             .expect("CardanoChainAdapter read should not fail");
-        let expected_markers = era_marker_payload_2
-            .markers
-            .into_iter()
-            .map(|em| em.into())
-            .collect::<Vec<EraMarker>>();
+        let expected_markers = era_marker_payload_2.markers.to_owned();
         assert_eq!(expected_markers, markers);
     }
 }
diff --git a/mithril-common/src/era/adapters/mod.rs b/mithril-common/src/era/adapters/mod.rs
@@ -9,7 +9,7 @@ pub use bootstrap::BootstrapAdapter as EraReaderBootstrapAdapter;
 pub use builder::{AdapterBuilder as EraReaderAdapterBuilder, AdapterType as EraReaderAdapterType};
 pub use cardano_chain::{
     CardanoChainAdapter as EraReaderCardanoChainAdapter,
-    EraMarkerItem as EraMarkerItemCardanoChain, EraMarkersPayload as EraMarkersPayloadCardanoChain,
+    EraMarkersPayload as EraMarkersPayloadCardanoChain,
 };
 pub use dummy::DummyAdapter as EraReaderDummyAdapter;
 pub use file::FileAdapter as EraReaderFileAdapter;
diff --git a/mithril-infra/variables.tf b/mithril-infra/variables.tf
@@ -132,15 +132,18 @@ variable "mithril_era_reader_adapter_type" {
 variable "mithril_era_reader_address_url" {
   type        = string
   description = "The url of the Mithril era reader address used to query the on chain Utxo containing the era markers payload"
+  default     = ""
 }
 
 variable "mithril_era_reader_verification_key_url" {
   type        = string
   description = "The url of the Mithril era reader verification key used by to verify an era markers payload"
+  default     = ""
 }
 variable "mithril_era_reader_secret_key" {
   type        = string
   description = "The Mithril genesis secret key used by the aggregator to generate an era marker payload TxDatum file (test only)"
+  default     = ""
 }
 
 variable "mithril_signers" {
diff --git a/mithril-signer/Cargo.toml b/mithril-signer/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "mithril-signer"
-version = "0.2.18"
+version = "0.2.19"
 description = "A Mithril Signer"
 authors = { workspace = true }
 edition = { workspace = true }

Original file line number	Diff line number	Diff line change
`@@ -132,15 +132,18 @@ variable "mithril_era_reader_adapter_type" {`
`132`	`132`	`variable "mithril_era_reader_address_url" {`
`133`	`133`	`type = string`
`134`	`134`	`description = "The url of the Mithril era reader address used to query the on chain Utxo containing the era markers payload"`
	`135`	`+ default = ""`
`135`	`136`	`}`
`136`	`137`
`137`	`138`	`variable "mithril_era_reader_verification_key_url" {`
`138`	`139`	`type = string`
`139`	`140`	`description = "The url of the Mithril era reader verification key used by to verify an era markers payload"`
	`141`	`+ default = ""`
`140`	`142`	`}`
`141`	`143`	`variable "mithril_era_reader_secret_key" {`
`142`	`144`	`type = string`
`143`	`145`	`description = "The Mithril genesis secret key used by the aggregator to generate an era marker payload TxDatum file (test only)"`
	`146`	`+ default = ""`
`144`	`147`	`}`
`145`	`148`
`146`	`149`	`variable "mithril_signers" {`