Merge pull request #2554 from input-output-hk/dlachaume/2492/add-documentation-and-ci-tests-for-utxo-hd-ledger-state-snapshot-conversion

Feat: documentation and CI tests for UTxO-HD ledger state snapshot conversion `mithril-client` CLI command

Author: dlachaume

.github/workflows/test-client.yml

@@ -167,6 +167,14 @@ jobs:
         shell: bash
         run: .github/workflows/scripts/verify-cardano-db-restoration.sh ./bin/cdb-download-output.txt "${{ matrix.extra_args }}"
 
+      - name: Ledger state snapshot conversion from InMemory to LMDB
+        if: matrix.extra_args == '--include-ancillary'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        working-directory: ./bin
+        run: ./mithril-client ${{ steps.prepare.outputs.debug_level }} --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LMDB --cardano-network $NETWORK --commit
+
       - name: Remove downloaded artifacts to free up disk space (Linux, Windows)
         if: runner.os != 'macOS'
         shell: bash
@@ -310,12 +318,29 @@ jobs:
           echo "CDB_SNAPSHOT_DIGEST=$(${{ steps.command.outputs.mithril_client }} --origin-tag CI cardano-db snapshot list --json | jq -r '.[0].digest')" >> $GITHUB_ENV
 
       - name: Cardano Database Snapshot / download & restore latest
+        if: matrix.extra_args != '--include-ancillary'
         shell: bash
         run: ${{ steps.command.outputs.mithril_client }} ${{ steps.prepare.outputs.debug_level }} --origin-tag CI cardano-db download $CDB_SNAPSHOT_DIGEST --download-dir /app ${{ matrix.extra_args }}
 
-      - name: Remove downloaded artifacts to free up disk space
+      - name: Cardano Database Snapshot / download & restore latest and run conversion from InMemory to LMDB
+        if: matrix.extra_args == '--include-ancillary'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
-        run: rm --force /app/db/immutable/*.{chunk,primary,secondary}
+        run: |
+          docker run --rm \
+            --entrypoint bash \
+            -e NETWORK=$NETWORK \
+            -e GENESIS_VERIFICATION_KEY=$GENESIS_VERIFICATION_KEY \
+            -e ANCILLARY_VERIFICATION_KEY=$ANCILLARY_VERIFICATION_KEY \
+            -e AGGREGATOR_ENDPOINT=$AGGREGATOR_ENDPOINT \
+            -e GITHUB_TOKEN=$GITHUB_TOKEN \
+            --name='mithril-client' \
+            ghcr.io/input-output-hk/mithril-client:$MITHRIL_IMAGE_ID \
+            -c "
+              /app/bin/mithril-client --origin-tag CI cardano-db download $CDB_SNAPSHOT_DIGEST --download-dir /app --include-ancillary &&
+              /app/bin/mithril-client --unstable tools utxo-hd snapshot-converter --db-directory /app/db --cardano-node-version latest --utxo-hd-flavor LMDB --cardano-network \$NETWORK --commit
+            "
 
       - name: Mithril Stake Distribution / list and get last hash
         shell: bash
@@ -454,3 +479,13 @@ jobs:
         run: |
           python3 ./.github/workflows/scripts/run-wasm-tests-browser-headless.py firefox
           ./.github/workflows/scripts/parse-wasm-headless-tests-results.sh firefox-results.html
+
+      - name: Upload Results Artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-run_id_${{ matrix.run_id }}
+          path: |
+            chrome-results.html
+            firefox-results.html
+          if-no-files-found: error

Cargo.lock

@@ -223,6 +223,7 @@ Commands:
   cardano-transaction         Cardano transactions management (alias: ctx)
   cardano-stake-distribution  Cardano stake distribution management (alias: csd)
   cardano-db-v2               Deprecated, use `cardano-db` instead
+  tools                       [unstable] Tools commands
   help                        Print this message or the help of the given subcommand(s)
 
 Options:
@@ -523,6 +524,13 @@ Here are the subcommands available:
 | **snapshot list** | Lists available cardano-db v2 snapshots                     |
 | **snapshot show** | Shows information about a cardano-db v2 snapshot            |
 
+### Tools (`unstable`)
+
+| Subcommand  | Performed action                                                                      |
+| ----------- | ------------------------------------------------------------------------------------- |
+| **utxo-hd** | UTxO-HD related commands (e.g., converting a ledger state snapshot to another flavor) |
+| **help**    | Prints this message or the help for the given subcommand(s)                           |
+
 ## Configuration parameters
 
 The configuration parameters can be set in either of the following ways:
@@ -670,3 +678,14 @@ Deprecated, use `cardano-db download` with option `--backend v2` instead
 | `include_ancillary`          | `--include-ancillary`          |          -           | -                            | Include ancillary files in the download, if set the `ancillary_verification_key` is required in order to verify the ancillary files | `false`       | -       |         -          |
 | `ancillary_verification_key` | `--ancillary-verification-key` |          -           | `ANCILLARY_VERIFICATION_KEY` | Ancillary verification key to verify the ancillary files                                                                            | -             | -       |         -          |
 | `allow_override`             | `--allow-override`             |          -           | -                            | Allow existing files in the download directory to be overridden                                                                     | `false`       | -       |         -          |
+
+`tools utxo-hd snapshot-converter` command:
+
+| Parameter              | Command line (long)      | Command line (short) | Environment variable | Description                                                                                                                                             | Default value | Example  |     Mandatory      |
+| ---------------------- | ------------------------ | :------------------: | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -------- | :----------------: |
+| `db_directory`         | `--db-directory`         |          -           | -                    | Path to the Cardano node database directory                                                                                                             | -             | -        | :heavy_check_mark: |
+| `cardano_node_version` | `--cardano-node-version` |          -           | -                    | Cardano node version of the Mithril signed snapshot (`latest` and `pre-release` are also supported to download the latest or pre-release distribution.) | -             | `10.4.1` | :heavy_check_mark: |
+| `cardano_network`      | `--cardano-network`      |          -           | -                    | Cardano network                                                                                                                                         | -             | -        | :heavy_check_mark: |
+| `utxo_hd_flavor`       | `--utxo-hd-flavor`       |          -           | -                    | UTxO-HD flavor to convert the ledger snapshot to (`Legacy` or `LMDB`)                                                                                   | -             | `LMDB`   | :heavy_check_mark: |
+| `commit`               | `--commit`               |          -           | -                    | Replaces the current ledger state in the `db_directory`                                                                                                 | `false`       | -        |         -          |
+| `github_token`         | `--github-token`         |          -           | `GITHUB_TOKEN`       | GitHub token for authenticated API calls                                                                                                                | -             | -        |         -          |

docs/website/root/manual/develop/nodes/mithril-client.md

@@ -482,9 +482,38 @@ Cardano db 'a1b5e6f43521fd9c5f55e3d6bf27dc4a62f43980681cb67e28cc40582a0d1974' ha
     If you are using Cardano Docker image, you can restore a Cardano Node with:
 
     docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source="/home/mithril/data/testnet/a1b5e6f43521fd9c5f55e3d6bf27dc4a62f43980681cb67e28cc40582a0d1974/db",target=/data/db/ -e NETWORK=preview ghcr.io/intersectmbo/cardano-node:10.4.1
+
+
+Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command:
+
+    mithril-client --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version 10.4.1 --utxo-hd-flavor LMDB --cardano-network preview --commit
+
+    Or to 'Legacy' flavor by running the command:
+
+    mithril-client --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version 10.4.1 --utxo-hd-flavor Legacy --cardano-network preview --commit
 ```
 
-### Step 5: Launch a Cardano node from the restored Cardano DB snapshot
+### Step 5 (optional): Convert the ledger state snapshot to another flavor
+
+After restoring a snapshot with the `--include-ancillary` option, the ledger state is in the `InMemory` format. You can convert it to another UTxO-HD flavor (e.g., `LMDB` or `Legacy`) using the Mithril client `tools utxo-hd snapshot-converter` command.
+
+To do so, run the following command with the `--unstable` flag:
+
+```
+mithril-client --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor LMDB --cardano-network $CARDANO_NETWORK
+```
+
+Or, to convert it to the `Legacy` flavor:
+
+```
+mithril-client --unstable tools utxo-hd snapshot-converter --db-directory db --cardano-node-version latest --utxo-hd-flavor Legacy --cardano-network $CARDANO_NETWORK
+```
+
+Use the `--commit` option to replace the current ledger state with the converted snapshot.
+
+You can also replace `latest` with a specific Cardano node version tag which will be used to download the corresponding Cardano node distribution and extract the `snapshot-converter` binary tool.
+
+### Step 6: Launch a Cardano node from the restored Cardano DB snapshot
 
 Launch an empty Cardano node and make it live in minutes!
 

docs/website/root/manual/getting-started/bootstrap-cardano-node.md

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client-cli"
-version = "0.12.9"
+version = "0.12.10"
 description = "A Mithril Client"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client-cli/Cargo.toml

@@ -19,7 +19,7 @@ const GITHUB_ORGANIZATION: &str = "IntersectMBO";
 const GITHUB_REPOSITORY: &str = "cardano-node";
 
 const LATEST_DISTRIBUTION_TAG: &str = "latest";
-const PRERELEASE_DISTRIBUTION_TAG: &str = "prerelease";
+const PRERELEASE_DISTRIBUTION_TAG: &str = "pre-release";
 
 const WORK_DIR: &str = "tmp";
 const CARDANO_DISTRIBUTION_DIR: &str = "cardano-node-distribution";
@@ -76,7 +76,7 @@ pub struct SnapshotConverterCommand {
 
     /// Cardano node version of the Mithril signed snapshot.
     ///
-    /// `latest` and `prerelease` are also supported to download the latest or prerelease distribution.
+    /// `latest` and `pre-release` are also supported to download the latest or pre-release distribution.
     #[clap(long)]
     cardano_node_version: String,
 
@@ -91,6 +91,10 @@ pub struct SnapshotConverterCommand {
     /// If set, the converted snapshot replaces the current ledger state in the `db_directory`.
     #[clap(long)]
     commit: bool,
+
+    /// GitHub token for authenticated API calls.
+    #[clap(long, env = "GITHUB_TOKEN")]
+    github_token: Option<String>,
 }
 
 impl SnapshotConverterCommand {
@@ -113,7 +117,7 @@ impl SnapshotConverterCommand {
                 )
             })?;
             let archive_path = Self::download_cardano_node_distribution(
-                ReqwestGitHubApiClient::new()?,
+                ReqwestGitHubApiClient::new(self.github_token.clone())?,
                 ReqwestHttpDownloader::new()?,
                 &self.cardano_node_version,
                 &distribution_dir,
@@ -185,7 +189,7 @@ impl SnapshotConverterCommand {
             PRERELEASE_DISTRIBUTION_TAG => github_api_client
                 .get_prerelease(GITHUB_ORGANIZATION, GITHUB_REPOSITORY)
                 .await
-                .with_context(|| "Failed to get prerelease")?,
+                .with_context(|| "Failed to get pre-release")?,
             _ => github_api_client
                 .get_release_by_tag(GITHUB_ORGANIZATION, GITHUB_REPOSITORY, tag)
                 .await

mithril-client-cli/src/commands/tools/snapshot_converter.rs

@@ -19,7 +19,7 @@ pub trait GitHubReleaseRetriever {
     /// Retrieves the latest release.
     async fn get_latest_release(&self, owner: &str, repo: &str) -> MithrilResult<GitHubRelease>;
 
-    /// Retrieves the prerelease.
+    /// Retrieves the pre-release.
     async fn get_prerelease(&self, owner: &str, repo: &str) -> MithrilResult<GitHubRelease>;
 
     /// Retrieves all available releases.

mithril-client-cli/src/utils/github_release_retriever/interface.rs

@@ -9,34 +9,41 @@ use super::{GitHubRelease, GitHubReleaseRetriever};
 
 pub struct ReqwestGitHubApiClient {
     client: Client,
+    github_token: Option<String>,
 }
 
 impl ReqwestGitHubApiClient {
-    pub fn new() -> MithrilResult<Self> {
+    pub fn new(github_token: Option<String>) -> MithrilResult<Self> {
         let client = Client::builder()
             .user_agent("mithril-client")
             .build()
             .context("Failed to build Reqwest GitHub API client")?;
 
-        Ok(Self { client })
+        Ok(Self {
+            client,
+            github_token,
+        })
     }
 
     async fn download<U: IntoUrl, T: DeserializeOwned>(&self, source_url: U) -> MithrilResult<T> {
         let url = source_url
             .into_url()
             .with_context(|| "Given `source_url` is not a valid Url")?;
-        let response = self
-            .client
-            .get(url.clone())
+        let mut request = self.client.get(url.clone());
+        if let Some(token) = &self.github_token {
+            request = request.bearer_auth(token);
+        }
+        let response = request
             .send()
             .await
             .with_context(|| format!("Failed to send request to GitHub API: {}", url))?;
         match response.status() {
             reqwest::StatusCode::OK => {}
             status => {
                 return Err(anyhow!(
-                    "GitHub API request failed with status code: {}",
-                    status
+                    "GitHub API request failed with status code '{}': {}",
+                    status,
+                    response.text().await.unwrap()
                 ));
             }
         }
@@ -84,7 +91,7 @@ impl GitHubReleaseRetriever for ReqwestGitHubApiClient {
         let prerelease = releases
             .into_iter()
             .find(|release| release.prerelease)
-            .ok_or_else(|| anyhow!("No prerelease found"))?;
+            .ok_or_else(|| anyhow!("No pre-release found"))?;
 
         Ok(prerelease)
     }
@@ -121,7 +128,7 @@ mod tests {
             when.method(GET).path("/endpoint");
             then.status(200).body(r#"{ "key": "value" }"#);
         });
-        let client = ReqwestGitHubApiClient::new().unwrap();
+        let client = ReqwestGitHubApiClient::new(None).unwrap();
 
         let result: FakeApiResponse = client
             .download(format!("{}/endpoint", server.base_url()))
@@ -143,7 +150,7 @@ mod tests {
             when.method(GET).path("/endpoint");
             then.status(200).body("this is not json");
         });
-        let client = ReqwestGitHubApiClient::new().unwrap();
+        let client = ReqwestGitHubApiClient::new(None).unwrap();
 
         let result: MithrilResult<FakeApiResponse> = client
             .download(format!("{}/endpoint", server.base_url()))
@@ -157,7 +164,7 @@ mod tests {
 
     #[tokio::test]
     async fn download_fails_on_invalid_url() {
-        let client = ReqwestGitHubApiClient::new().unwrap();
+        let client = ReqwestGitHubApiClient::new(None).unwrap();
 
         let result: MithrilResult<FakeApiResponse> = client.download("not a valid url").await;
 
@@ -171,7 +178,7 @@ mod tests {
             when.method(GET).path("/endpoint");
             then.status(StatusCode::INTERNAL_SERVER_ERROR.into());
         });
-        let client = ReqwestGitHubApiClient::new().unwrap();
+        let client = ReqwestGitHubApiClient::new(None).unwrap();
 
         let result: MithrilResult<FakeApiResponse> = client
             .download(format!("{}/endpoint", server.base_url()))