Merge pull request #1345 from input-output-hk/ensemble/1343-fix-recompute-hashes-certificates

Fix certificate hashes re-computation in aggregator

Author: jpraynaud

Cargo.lock

@@ -5,12 +5,14 @@ Export the environment variables:
 ```bash
 export MITHRIL_VM=**MITHRIL_VM**
 export CARDANO_NETWORK=**CARDANO_NETWORK**
+export MITHRIL_DISTRIBUTION_LINUX_PKG=**MITHRIL_DISTRIBUTION_LINUX_PKG**
 ```
 
 Here is an exmaple for the `release-mainnet` network:
 ```bash
 export MITHRIL_VM=aggregator.release-mainnet.api.mithril.network
 export CARDANO_NETWORK=mainnet
+export MITHRIL_DISTRIBUTION_LINUX_PKG=https://github.com/input-output-hk/mithril/releases/download/2342.0/mithril-2342.0-linux-x64.tar.gz
 ```
 
 ## Make a backup of the aggregator database
@@ -30,29 +32,64 @@ And copy the SQLite database file `aggregator.sqlite3`:
 cp /home/curry/data/$CARDANO_NETWORK/mithril-aggregator/mithril/stores/aggregator.sqlite3 /home/curry/data/$CARDANO_NETWORK/mithril-aggregator/mithril/stores/aggregator.sqlite3.bak.$(date +%Y-%m-%d)
 ```
 
-And connect to the aggregator container:
+## Create a temp directory
+
+Create a temp directory:
 ```bash
-docker exec -it mithril-aggregator bash
+rm -rf /home/curry/temp
+mkdir -p /home/curry/temp/config
+cd /home/curry/temp
 ```
 
-Once connected to the aggregator container, recompute the certificates hashes:
+And download the configuration file for tools:
 ```bash
-/app/bin/mithril-aggregator -vvvv tools recompute-certificates-hash
+wget https://raw.githubusercontent.com/input-output-hk/mithril/main/docs/runbook/recompute-certificates-hash/config/tools.json -O /home/curry/temp/config/tools.json
 ```
 
-Then disconnect from the aggregator container:
+## Download the pre-compiled aggregator binary
+
+Download the mithril pre-compiled binaries package:
 ```bash
-exit
+wget $MITHRIL_DISTRIBUTION_LINUX_PKG -O mithril-bin.tar.gz
+```
+
+Unpack the mithril aggregator binary:
+```bash
+tar xzf mithril-bin.tar.gz mithril-aggregator
+```
+
+Make mithril aggregator binary executable:
+```bash
+chmod u+x mithril-aggregator
+```
+
+Make sure you are running the expected version of the aggregator:
+```bash
+./mithril-aggregator --version
+```
+
+## Stop the aggregator
+
+Stop the aggregator container:
+```bash
+docker stop mithril-aggregator
+```
+
+## Run the migration
+
+Once connected to the aggregator container, recompute the certificates hashes:
+```bash
+DATA_STORES_DIRECTORY=/home/curry/data/$CARDANO_NETWORK/mithril-aggregator/mithril/stores/ ./mithril-aggregator --run-mode tools -vvv tools recompute-certificates-hash
 ```
 
 ## Restart the aggregator
 
 Restart the aggregator to make sure that the certificate chain is valid:
 ```bash
-docker restart mithril-aggregator
+docker start mithril-aggregator
 ```
 
-Make sure that the certificate chain is valid (wait for the state machiene to go into the state `READY`):
+Make sure that the certificate chain is valid (wait for the state machine to go into the `READY` state):
 ```bash
 docker logs -f --tail 1000 mithril-aggregator
 ```
@@ -62,6 +99,13 @@ Then disconnect from the aggregator VM:
 exit
 ```
 
+## Cleanup the temp directory
+
+Remove the temp directory:
+```bash
+rm -rf /home/curry/temp
+```
+
 ## Rollback procedure
 
 If the recomputation fails, you can rollback the database.

docs/runbook/recompute-certificates-hash/README.md

@@ -0,0 +1,17 @@
+{
+    "cardano_cli_path": "-",
+    "cardano_node_socket_path": "-",
+    "network": "-",
+    "network_magic": 0,
+    "run_interval": 0,
+    "protocol_parameters": {
+        "k": 0,
+        "m": 0,
+        "phi_f": 0
+    },
+    "snapshot_store_type": "local",
+    "snapshot_uploader_type": "local",
+    "genesis_verification_key": "-",
+    "era_adapter_type": "bootstrap",
+    "cardano_node_version": "-"
+}

docs/runbook/recompute-certificates-hash/config/tools.json

@@ -20,6 +20,6 @@ Currently, the following [Mithril networks](https://mithril.network/doc/manual/d
 
 A user with administrator rights can simply remove the lock file:
 - In GCP [**Cloud Storage**](https://console.cloud.google.com/storage/browser)
-- In the terraform administration bucket that you have identified earlier, the file that needs to be removed is at path `**TERRAFORM_BACKEND_BUCKET**/terraform/mithril-**MITHRIL_NETWORK_IDENTIFIER**/.terraform.lock.hcl` (e.g. `mithril-terraform-prod/terraform/mithril-release-mainnet/terraform.lock.hcl`) 
+- In the terraform administration bucket that you have identified earlier, the file that needs to be removed is at path `**TERRAFORM_BACKEND_BUCKET**/terraform/mithril-**MITHRIL_NETWORK_IDENTIFIER**/default.tflock` (e.g. `mithril-terraform-prod/terraform/mithril-release-mainnet/terraform.lock.hcl`) 
 
 :warning: never delete/modify the `**TERRAFORM_BACKEND_BUCKET**/terraform/mithril-**MITHRIL_NETWORK_IDENTIFIER**/default.tfstate` file.

docs/runbook/terraform-lock/README.md

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.8"
+version = "0.4.9"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -48,7 +48,7 @@ zstd = { version = "0.13.0", features = ["zstdmt"] }
 httpmock = "0.6.8"
 mithril-common = { path = "../mithril-common", features = [
     "allow_skip_signer_certification",
-    "test_tools"
+    "test_tools",
 ] }
 mockall = "0.11.4"
 slog-term = "2.9.0"

mithril-aggregator/Cargo.toml

@@ -48,6 +48,7 @@ impl CertificatesHashMigrator {
             // arbitrary high value to get all existing certificates
             .get_latest_certificates(usize::MAX)
             .await?;
+        let mut certificates_to_remove = vec![];
 
         let mut migrated_certificates = vec![];
         let mut old_and_new_hashes: HashMap<String, String> = HashMap::new();
@@ -57,7 +58,7 @@ impl CertificatesHashMigrator {
         // in order to have a strong guarantee that when inserting a certificate in the db its
         // previous_hash exist we have to work in the reverse order.
         debug!("🔧 Certificate Hash Migrator: computing new hash for all certificates");
-        for mut certificate in old_certificates.clone().into_iter().rev() {
+        for mut certificate in old_certificates.into_iter().rev() {
             let old_previous_hash = if certificate.is_genesis() {
                 certificate.previous_hash.clone()
             } else {
@@ -72,40 +73,37 @@ impl CertificatesHashMigrator {
                 old_previous_hash
             };
 
-            let new_hash = {
+            if let Some(new_hash) = {
                 let computed_hash = certificate.compute_hash();
-                // return none if the hash did not change to trigger an error
+                // return none if the hash did not change
                 (computed_hash != certificate.hash).then_some(computed_hash)
-            }
-            .ok_or(anyhow!(
-                "Hash did not change for certificate {:?}, hash: {}",
-                certificate.beacon,
-                certificate.hash
-            ))?
-            .to_owned();
-
-            old_and_new_hashes.insert(certificate.hash.clone(), new_hash.clone());
+            } {
+                old_and_new_hashes.insert(certificate.hash.clone(), new_hash.clone());
+
+                if certificate.is_genesis() {
+                    trace!(
+                        "🔧 Certificate Hash Migrator: new hash computed for genesis certificate {:?}",
+                        certificate.beacon;
+                        "old_hash" => &certificate.hash,
+                        "new_hash" => &new_hash,
+                    );
+                } else {
+                    trace!(
+                        "🔧 Certificate Hash Migrator: new hash computed for certificate {:?}",
+                        certificate.beacon;
+                        "old_hash" => &certificate.hash,
+                        "new_hash" => &new_hash,
+                        "old_previous_hash" => &old_previous_hash,
+                        "new_previous_hash" => &certificate.previous_hash
+                    );
+                }
 
-            if certificate.is_genesis() {
-                trace!(
-                    "🔧 Certificate Hash Migrator: new hash computed for genesis certificate {:?}",
-                    certificate.beacon;
-                    "old_hash" => &certificate.hash,
-                    "new_hash" => &new_hash,
-                );
+                certificates_to_remove.push(certificate.clone());
+                certificate.hash = new_hash;
+                migrated_certificates.push(certificate);
             } else {
-                trace!(
-                    "🔧 Certificate Hash Migrator: new hash computed for certificate {:?}",
-                    certificate.beacon;
-                    "old_hash" => &certificate.hash,
-                    "new_hash" => &new_hash,
-                    "old_previous_hash" => &old_previous_hash,
-                    "new_previous_hash" => &certificate.previous_hash
-                );
+                old_and_new_hashes.insert(certificate.hash.clone(), certificate.hash);
             }
-
-            certificate.hash = new_hash;
-            migrated_certificates.push(certificate);
         }
 
         // 2 - Certificates migrated, we can insert them in the db
@@ -123,7 +121,7 @@ impl CertificatesHashMigrator {
             })?;
         }
 
-        Ok((old_certificates, old_and_new_hashes))
+        Ok((certificates_to_remove, old_and_new_hashes))
     }
 
     async fn update_signed_entities_certificate_hash(
@@ -590,7 +588,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn should_fail_if_any_hash_doesnt_change() {
+    async fn should_not_fail_if_some_hash_dont_change() {
         let connection = Arc::new(connection_without_foreign_key_support());
         let certificate = {
             let mut cert = dummy_genesis("whatever", 1, 2);
@@ -608,6 +606,6 @@ mod test {
         migrator
             .migrate()
             .await
-            .expect_err("Migration should fail if an hash doesnt change");
+            .expect("Migration should not fail if a hash doesn't change");
     }
 }