diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index add5467ea..70c155fb1 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "1ad44051fdab3020717ead2412f104d2309e44c90a9a4fea6dd4b6a375a18ba8", + "checksum": "4a2bcbd43338a9a0c33b03fa028db4119525bbfe24bd3194f8ce12b7c3d342d2", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -10734,6 +10734,69 @@ ], "license_file": "LICENSE-APACHE" }, + "dashmap 5.5.3": { + "name": "dashmap", + "version": "5.5.3", + "package_url": "https://github.com/xacrimon/dashmap", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/dashmap/5.5.3/download", + "sha256": "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" + } + }, + "targets": [ + { + "Library": { + "crate_name": "dashmap", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "dashmap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.3", + "target": "cfg_if" + }, + { + "id": "hashbrown 0.14.5", + "target": "hashbrown" + }, + { + "id": "lock_api 0.4.12", + "target": "lock_api" + }, + { + "id": "once_cell 1.21.3", + "target": "once_cell" + }, + { + "id": "parking_lot_core 0.9.10", + "target": "parking_lot_core" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "5.5.3" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE" + }, "data-encoding 2.7.0": { "name": "data-encoding", "version": "2.7.0", @@ -12748,6 +12811,10 @@ "id": "backon 1.5.2", "target": "backon" }, + { + "id": "base64 0.22.1", + "target": "base64" + }, { "id": "candid 0.10.18", "target": "candid" @@ -12816,6 +12883,10 @@ "id": "futures-util 0.3.31", "target": "futures_util" }, + { + "id": "hex 0.4.3", + "target": "hex" + }, { "id": "human_bytes 0.4.3", "target": "human_bytes" @@ -12868,6 +12939,10 @@ "id": "ic-protobuf 0.9.0", "target": "ic_protobuf" }, + { + "id": "ic-registry-common-proto 0.9.0", + "target": "ic_registry_common_proto" + }, { "id": "ic-registry-keys 0.9.0", "target": "ic_registry_keys" @@ -13000,6 +13075,10 @@ { "id": "actix-rt 2.11.0", "target": "actix_rt" + }, + { + "id": "serial_test 2.0.0", + "target": "serial_test" } ], "selects": {} @@ -49232,6 +49311,149 @@ ], "license_file": "LICENSE-APACHE" }, + "serial_test 2.0.0": { + "name": "serial_test", + "version": "2.0.0", + "package_url": "https://github.com/palfrey/serial_test/", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/serial_test/2.0.0/download", + "sha256": "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "serial_test", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "serial_test", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "async", + "default", + "futures", + "log", + "logging" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "dashmap 5.5.3", + "target": "dashmap" + }, + { + "id": "futures 0.3.31", + "target": "futures" + }, + { + "id": "lazy_static 1.5.0", + "target": "lazy_static" + }, + { + "id": "log 0.4.28", + "target": "log" + }, + { + "id": "parking_lot 0.12.3", + "target": "parking_lot" + } + ], + "selects": {} + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "serial_test_derive 2.0.0", + "target": "serial_test_derive" + } + ], + "selects": {} + }, + "version": "2.0.0" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE" + }, + "serial_test_derive 2.0.0": { + "name": "serial_test_derive", + "version": "2.0.0", + "package_url": "https://github.com/palfrey/serial_test/", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/serial_test_derive/2.0.0/download", + "sha256": "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" + } + }, + "targets": [ + { + "ProcMacro": { + "crate_name": "serial_test_derive", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "serial_test_derive", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "async" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "proc-macro2 1.0.101", + "target": "proc_macro2" + }, + { + "id": "quote 1.0.40", + "target": "quote" + }, + { + "id": "syn 2.0.106", + "target": "syn" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "2.0.0" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE" + }, "service-discovery 0.6.7": { "name": "service-discovery", "version": "0.6.7", @@ -62559,13 +62781,8 @@ "actix-rt 2.11.0", "assert_cmd 2.0.17", "maplit 1.0.2", + "serial_test 2.0.0", "wiremock 0.6.4" ], - "unused_patches": [ - { - "name": "ic0", - "version": "0.23.0", - "source": "registry+https://github.com/rust-lang/crates.io-index" - } - ] + "unused_patches": [] } diff --git a/Cargo.lock b/Cargo.lock index 3992877a7..fe40fa03a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1819,6 +1819,19 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.7.0" @@ -2162,6 +2175,7 @@ dependencies = [ "anyhow", "async-recursion", "backon", + "base64 0.22.1", "candid", "chrono", "clap", @@ -2179,6 +2193,7 @@ dependencies = [ "fs-err", "futures", "futures-util", + "hex", "human_bytes", "humantime", "ic-base-types", @@ -2195,6 +2210,7 @@ dependencies = [ "ic-nns-governance", "ic-nns-governance-api", "ic-protobuf", + "ic-registry-common-proto", "ic-registry-keys", "ic-registry-local-registry", "ic-registry-subnet-type", @@ -2217,6 +2233,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "serial_test", "sha2 0.10.9", "shlex", "spinners", @@ -8478,6 +8495,31 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap", + "futures", + "lazy_static", + "log", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "service-discovery" version = "0.6.7" @@ -10438,9 +10480,3 @@ dependencies = [ "cc", "pkg-config", ] - -[[patch.unused]] -name = "ic0" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - diff --git a/Cargo.toml b/Cargo.toml index 061515517..234c86a62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,7 +139,7 @@ cycles-minting-canister = { git = "https://github.com/dfinity/ic.git", rev = "eb ic-icrc1-test-utils = { git = "https://github.com/dfinity/ic.git", rev = "ebed5d4228b71477738316ae3d5274672eae369e" } rosetta-core = { git = "https://github.com/dfinity/ic.git", rev = "ebed5d4228b71477738316ae3d5274672eae369e" } icp-ledger = { git = "https://github.com/dfinity/ic.git", rev = "ebed5d4228b71477738316ae3d5274672eae369e" } -icrc-ledger-types = { git = "https://github.com/dfinity/ic.git",rev = "ebed5d4228b71477738316ae3d5274672eae369e" } +icrc-ledger-types = { git = "https://github.com/dfinity/ic.git", rev = "ebed5d4228b71477738316ae3d5274672eae369e" } ic-metrics-encoder = "1.1.1" ic-transport-types = "0.39.3" ic-utils = "0.39.3" @@ -221,6 +221,3 @@ indexmap = { version = "2.11.1", features = ["serde"] } # Makes flamegraphs and backtraces more readable. # https://doc.rust-lang.org/cargo/reference/manifest.html#the-profile-sections debug = true - -[patch.'https://github.com/dfinity/cdk-rs.git'] -ic0 = "0.23.0" diff --git a/README.md b/README.md index 1dd831567..498635929 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ DRE provides a powerful set of tools and services: - **DRE CLI Tool**: Command-line interface for interacting with IC infrastructure - Available as a pre-built binary on [GitHub Releases](https://github.com/dfinity/dre/releases) - Examples available in [NNS proposals documentation](nns-proposals.md) + - Registry versions dump docs: [registry-versions.md](docs/registry-versions.md) - **DRE Dashboard**: Comprehensive monitoring and management interface - Frontend and backend components diff --git a/docs/nns-proposals.md b/docs/nns-proposals.md index 3a7b81170..2e37e1659 100644 --- a/docs/nns-proposals.md +++ b/docs/nns-proposals.md @@ -109,6 +109,27 @@ dre registry | jq '.nodes[] | select((.dc_id == "bo1") and (.subnet_id != null)) ``` **Explanation**: This command retrieves a dump of the registry, extracts the list of nodes, and selects only those that belong to the `bo1` data center and have a `subnet_id` set (i.e., not null). +#### Dumping raw registry versions (for precise diffs) + +Use the DRE CLI to dump raw registry versions, either a single version or a range, for exact inspection of changes: + +```bash +# One version +dre registry --dump-version 12345 | jq + +# Range: Python-style indexing (end-exclusive). Indexing semantics: +# - Positive indices are 1-based positions (registry is 1-based) +# - 0 means start (same as omitting FROM) +# - Negative indices count from end (-1 is last) +# - Reversed ranges yield empty results +dre registry --dump-versions -5 > last5.json + +# All versions (large output) +dre registry --dump-versions > all.json +``` + +Each output element is a single registry record at a registry version with best-effort decoded value. Unknown byte blobs are shown compactly using base64 as `{ "bytes_base64": "..." }`. For short byte arrays the tool may also add a `principal` string. +
Click here to see the output of the above command ```json diff --git a/docs/registry-versions.md b/docs/registry-versions.md new file mode 100644 index 000000000..893899ba3 --- /dev/null +++ b/docs/registry-versions.md @@ -0,0 +1,65 @@ +# Registry Versions Dump (dre registry) + +This document describes how to use the DRE CLI to inspect Internet Computer Protocol (ICP) registry versions as raw records, in JSON, suitable for precise diffs and troubleshooting. + +## Commands + +- Dump a single version 50000 (flat list of records): + +```bash +dre registry --dump-versions 50000 50001 | jq +``` + +- Dump a range of versions using Python-style indexing (end-exclusive), where -1 is the last index and omitted end means "to the end": + +```bash +dre registry --dump-versions -5 > last5.json +# Indexing semantics (Python slicing, end-exclusive): +# - positive indices are 1-based positions (registry is 1-based) +# - 0 means start (same as omitting FROM) +# - negative indices count from end (-1 is last) +# - reversed ranges yield empty results +# Examples: +# -5 -> last 5 +# -5 -1 -> last 4 (excludes the very last) +# -1 -> last 1 +# 0 -> all (from record 0 to the end) +``` + +- Dump ALL versions (warning: large): + +```bash +dre registry --dump-versions > all.json +``` + +## Output Shape + +Output is a flat JSON array of objects. Each object corresponds to a single registry record at a specific version: + +```json + { + "version": 50000, + "key": "node_record_cekdc-hmzri-3u3or-ei7ip-su7ck-3xt6e-zsse2-tgakq-rolmv-6crkh-hqe", + "value": { + "xnet": { + "ip_addr": "2401:7500:ff1:20:6801:8fff:fe0c:cff4", + "port": 2497 + }, + "http": { + "ip_addr": "2401:7500:ff1:20:6801:8fff:fe0c:cff4", + "port": 8080 + }, + "node_operator_id": { + "bytes_base64": "K0aH3L0TlIkJORV0I4GwVU2GMLDZAv5U8Av1kAI=", + "principal": "ri4lg-drli2-d5zpi-tsseq-soivo-qrydm-cvjwd-dbmgz-al7fj-4al6w-iae" + }, + "chip_id": null, + "hostos_version_id": "68fc31a141b25f842f078c600168d8211339f422", + "public_ipv4_config": null, + "domain": null, + "node_reward_type": 5 + } + }, +[...] +``` + diff --git a/rs/cli/Cargo.toml b/rs/cli/Cargo.toml index e5d4ebf2b..4084ad3f2 100644 --- a/rs/cli/Cargo.toml +++ b/rs/cli/Cargo.toml @@ -82,9 +82,13 @@ tabular = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } url = { workspace = true } +hex = { workspace = true } +ic-registry-common-proto = { workspace = true } +base64 = { version = "0.22" } [dev-dependencies] actix-rt = { workspace = true } +serial_test = "2.0" [build-dependencies] clap = { workspace = true } diff --git a/rs/cli/src/commands/registry.rs b/rs/cli/src/commands/registry.rs index 26a16e4da..1065c516e 100644 --- a/rs/cli/src/commands/registry.rs +++ b/rs/cli/src/commands/registry.rs @@ -1,5 +1,6 @@ use crate::ctx::DreContext; use crate::{auth::AuthRequirement, exe::args::GlobalArgs, exe::ExecutableCommand}; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use clap::Args; use ic_canisters::governance::GovernanceCanisterWrapper; use ic_canisters::IcAgentCanisterClient; @@ -20,6 +21,7 @@ use icp_ledger::AccountIdentifier; use indexmap::IndexMap; use itertools::Itertools; use log::{info, warn}; +use prost::Message; use regex::Regex; use serde::Serialize; use serde_json::Value; @@ -39,7 +41,25 @@ use std::{ dre registry --filter rewards_correct!=true # Entries for which rewardable_nodes != total_up_nodes dre registry --filter "node_type=type1" # Entries where node_type == "type1" dre registry -o registry.json --filter "subnet_id startswith tdb26" # Write to file and filter by subnet_id - dre registry -o registry.json --filter "node_id contains h5zep" # Write to file and filter by node_id"#)] + dre registry -o registry.json --filter "node_id contains h5zep" # Write to file and filter by node_id + + Registry versions/records dump: + dre registry --dump-versions # Dump ALL versions (Python slicing, end-exclusive) + dre registry --dump-versions 0 # Same as above (from start to end) + dre registry --dump-versions -5 # Last 5 versions (from -5 to end) + dre registry --dump-versions -5 -1 # Last 4 versions (excludes last) + + Indexing semantics (important!): + - Python slicing with end-exclusive semantics + - Positive indices are 1-based positions (since registry records are 1-based) + - 0 means start (same as omitting FROM) + - Negative indices count from the end (-1 is last) + - Reversed ranges yield empty results + + Notes: + - Values are best-effort decoded by key; unknown bytes are shown as {"bytes_base64": "..."}. + - For known protobuf records, embedded byte arrays are compacted to base64 strings for readability. +"#)] pub struct Registry { /// Output file (default is stdout) #[clap(short = 'o', long)] @@ -52,6 +72,18 @@ pub struct Registry { /// Specify the height for the registry #[clap(long, visible_aliases = ["registry-height", "version"])] pub height: Option, + + /// Dump raw registry versions in range by index; defaults to 0 -1 if no args + #[clap( + long = "dump-versions", + num_args(0..=2), + value_names = ["FROM", "TO"], + allow_hyphen_values = true, + conflicts_with_all = ["filters"], + visible_aliases = ["dump-version", "dump-versions-json", "dump-json-versions", "json-versions"], + help = "Index-based range over available versions. Negative indexes allowed (Python-style). If omitted, defaults to 0 -1." + )] + pub dump_versions: Option>, } #[derive(Debug, Clone)] @@ -101,11 +133,18 @@ impl ExecutableCommand for Registry { None => Box::new(std::io::stdout()), }; - let registry = self.get_registry(ctx).await?; + // Versions/records mode + if self.dump_versions.is_some() { + let json_value = self.dump_versions_json(ctx).await?; + serde_json::to_writer_pretty(writer, &json_value)?; + return Ok(()); + } + // Friendly aggregated registry view (default) + let registry = self.get_registry(ctx).await?; let mut serde_value = serde_json::to_value(registry)?; self.filters.iter().for_each(|filter| { - filter_json_value(&mut serde_value, &filter.key, &filter.value, &filter.comparison); + let _ = filter_json_value(&mut serde_value, &filter.key, &filter.value, &filter.comparison); }); serde_json::to_writer_pretty(writer, &serde_value)?; @@ -116,6 +155,140 @@ impl ExecutableCommand for Registry { fn validate(&self, _args: &GlobalArgs, _cmd: &mut clap::Command) {} } +impl Registry { + pub(crate) async fn dump_versions_json(&self, ctx: DreContext) -> anyhow::Result { + use ic_registry_common_proto::pb::local_store::v1::ChangelogEntry as PbChangelogEntry; + + // Ensure local registry is initialized/synced to have content available + // (no-op in offline mode) + let _ = ctx.registry_with_version(None).await; + + let base_dirs = local_registry_dirs_for_ctx(&ctx)?; + + // Walk all .pb files recursively across candidates; stop at first non-empty + let entries = load_first_available_entries(&base_dirs)?; + + // Apply version filters + let mut entries_sorted = entries; + entries_sorted.sort_by_key(|(v, _)| *v); + let versions_sorted: Vec = entries_sorted.iter().map(|(v, _)| *v).collect(); + + // Select versions + let selected_versions = select_versions(self.dump_versions.clone(), &versions_sorted)?; + + // Build flat list of records + let entries_map: std::collections::HashMap = entries_sorted.into_iter().collect(); + let out = flatten_version_records(&selected_versions, &entries_map); + Ok(serde_json::to_value(out)?) + } +} + +// Helper: collect candidate base dirs +fn local_registry_dirs_for_ctx(ctx: &DreContext) -> anyhow::Result> { + let mut base_dirs = vec![dirs::cache_dir() + .ok_or_else(|| anyhow::anyhow!("Couldn't find cache dir for dre-store"))? + .join("dre-store") + .join("local_registry") + .join(&ctx.network().name)]; + if let Ok(override_dir) = std::env::var("DRE_LOCAL_REGISTRY_DIR_OVERRIDE") { + base_dirs.insert(0, std::path::PathBuf::from(override_dir)); + } + base_dirs.push(std::path::PathBuf::from("/tmp/dre-test-store/local_registry").join(&ctx.network().name)); + Ok(base_dirs) +} + +fn load_first_available_entries( + base_dirs: &[std::path::PathBuf], +) -> anyhow::Result> { + use ic_registry_common_proto::pb::local_store::v1::ChangelogEntry as PbChangelogEntry; + use std::ffi::OsStr; + + let mut entries: Vec<(u64, PbChangelogEntry)> = Vec::new(); + for base_dir in base_dirs.iter() { + let mut local: Vec<(u64, PbChangelogEntry)> = Vec::new(); + collect_pb_files(base_dir, &mut |path| { + if path.extension() == Some(OsStr::new("pb")) { + if let Some(v) = extract_version_from_registry_path(base_dir, path) { + let bytes = std::fs::read(path).unwrap_or_else(|_| panic!("Failed reading {}", path.display())); + let entry = PbChangelogEntry::decode(bytes.as_slice()).unwrap_or_else(|_| panic!("Failed decoding {}", path.display())); + local.push((v, entry)); + } + } + })?; + if !local.is_empty() { + entries = local; + break; + } + } + if entries.is_empty() { + anyhow::bail!("No registry versions found in local store"); + } + Ok(entries) +} + +// Slicing semantics: +// - Python-like, end-exclusive +// - Positive indices are 1-based positions (since registry records are 1-based) +// - 0 means start (same as omitting FROM) +// - Negative indices are from the end (-1 is last) +// - Reversed ranges yield empty results +fn select_versions(versions: Option>, versions_sorted: &[u64]) -> anyhow::Result> { + let n = versions_sorted.len(); + let args = versions.unwrap_or_default(); + let (from_opt, to_opt): (Option, Option) = match args.as_slice() { + [] => (None, None), + [from] => (Some(*from), None), + [from, to] => (Some(*from), Some(*to)), + _ => unreachable!(), + }; + if n == 0 { + return Ok(vec![]); + } + let norm_index = |idx: i64| -> usize { + match idx { + i if i < 0 => { + let j = (n as i64) + i; // Python-style negative from end + j.clamp(0, n as i64) as usize + } + 0 => 0, + i => { + // Treat positive indices as 1-based positions since registry records are 1-based; convert to zero-based + ((i - 1) as usize).clamp(0, n) + } + } + }; + let a = from_opt.map(norm_index).unwrap_or(0); + let b = to_opt.map(norm_index).unwrap_or(n); + if a >= b { + return Ok(vec![]); + } + Ok(versions_sorted[a..b].to_vec()) +} + +fn flatten_version_records( + selected_versions: &[u64], + entries_map: &std::collections::HashMap, +) -> Vec { + use ic_registry_common_proto::pb::local_store::v1::MutationType; + let mut out: Vec = Vec::new(); + for v in selected_versions { + if let Some(entry) = entries_map.get(v) { + for km in entry.key_mutations.iter() { + let value_json = match km.mutation_type() { + MutationType::Unset => Value::Null, + MutationType::Set => decode_value_to_json(&km.key, &km.value), + _ => Value::Null, + }; + out.push(VersionRecord { + version: *v, + key: km.key.clone(), + value: value_json, + }); + } + } + } + out +} impl Registry { async fn get_registry(&self, ctx: DreContext) -> anyhow::Result { let local_registry = ctx.registry_with_version(self.height).await; @@ -168,6 +341,143 @@ impl Registry { } } +#[derive(Debug, Serialize)] +struct VersionRecord { + version: u64, + key: String, + value: Value, +} + +fn extract_version_from_registry_path(base_dir: &std::path::Path, full_path: &std::path::Path) -> Option { + // Registry path ends with .../<10 hex>/<2 hex>/<2 hex>/<5 hex>.pb + // We reconstruct the hex by concatenating the four segments (without slashes) and parse as hex u64. + let rel = full_path.strip_prefix(base_dir).ok()?; + let parts: Vec<_> = rel.iter().map(|s| s.to_string_lossy()).collect(); + if parts.len() < 4 { + return None; + } + let last = parts[parts.len() - 1].trim_end_matches(".pb"); + let seg3 = &parts[parts.len() - 2]; + let seg2 = &parts[parts.len() - 3]; + let seg1 = &parts[parts.len() - 4]; + let hex = format!("{}{}{}{}", seg1, seg2, seg3, last); + u64::from_str_radix(&hex, 16).ok() +} + +/// Best-effort decode of registry value bytes into JSON. Falls back to hex when unknown. +/// This can be extended to specific types in the future, if needed +fn decode_value_to_json(key: &str, bytes: &[u8]) -> Value { + // Known families where we can decode via protobuf types pulled from workspace crates. + // Use key prefixes to route decoding. Keep minimal and practical. + if key.starts_with(ic_registry_keys::DATA_CENTER_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::dc::v1::DataCenterRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key.starts_with(ic_registry_keys::NODE_OPERATOR_RECORD_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::node_operator::v1::NodeOperatorRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key.starts_with(ic_registry_keys::NODE_RECORD_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::node::v1::NodeRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key.starts_with(ic_registry_keys::SUBNET_RECORD_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::subnet::v1::SubnetRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key.starts_with(ic_registry_keys::REPLICA_VERSION_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::replica_version::v1::ReplicaVersionRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key.starts_with(ic_registry_keys::HOSTOS_VERSION_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::hostos_version::v1::HostosVersionRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key == ic_registry_keys::NODE_REWARDS_TABLE_KEY { + if let Ok(rec) = ic_protobuf::registry::node_rewards::v2::NodeRewardsTable::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key.starts_with(ic_registry_keys::API_BOUNDARY_NODE_RECORD_KEY_PREFIX) { + if let Ok(rec) = ic_protobuf::registry::api_boundary_node::v1::ApiBoundaryNodeRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key == "unassigned_nodes_config" { + if let Ok(rec) = ic_protobuf::registry::unassigned_nodes_config::v1::UnassignedNodesConfigRecord::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } else if key == "blessed_replica_versions" { + if let Ok(rec) = ic_protobuf::registry::replica_version::v1::BlessedReplicaVersions::decode(bytes) { + return normalize_protobuf_json(serde_json::to_value(&rec).unwrap_or(Value::Null)); + } + } + + // Fallback: base64 for compactness + let s = BASE64.encode(bytes); + if bytes.len() <= 29 { + if let Ok(p) = ic_types::PrincipalId::try_from(bytes.to_vec()) { + return serde_json::json!({ "bytes_base64": s, "principal": p.to_string() }); + } + } + serde_json::json!({ "bytes_base64": s }) +} + +/// Recursively convert protobuf-derived JSON so byte arrays become base64 strings +fn normalize_protobuf_json(mut v: Value) -> Value { + match &mut v { + Value::Array(arr) => { + for e in arr.iter_mut() { + *e = normalize_protobuf_json(std::mem::take(e)); + } + } + Value::Object(map) => { + for (_, vv) in map.iter_mut() { + *vv = normalize_protobuf_json(std::mem::take(vv)); + } + } + Value::Number(_) | Value::String(_) | Value::Bool(_) | Value::Null => {} + } + + // Replace array of small integers (likely bytes) with base64 when appropriate + if let Value::Array(arr) = &v { + if !arr.is_empty() + && arr + .iter() + .all(|x| matches!(x, Value::Number(n) if n.as_u64().is_some() && n.as_u64().unwrap() <= 255)) + { + let mut buf = Vec::with_capacity(arr.len()); + for x in arr { + if let Value::Number(n) = x { + buf.push(n.as_u64().unwrap() as u8); + } + } + let s = BASE64.encode(&buf); + if buf.len() <= 29 { + if let Ok(p) = ic_types::PrincipalId::try_from(buf) { + return serde_json::json!({"bytes_base64": s, "principal": p.to_string()}); + } + } + return serde_json::json!({ "bytes_base64": s }); + } + } + v +} + +fn collect_pb_files(base: &std::path::Path, visitor: &mut F) -> anyhow::Result<()> { + if !base.exists() { + return Ok(()); + } + for entry in fs_err::read_dir(base)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_pb_files(&path, visitor)?; + } else { + visitor(&path); + } + } + Ok(()) +} + fn get_elected_guest_os_versions(local_registry: &Arc) -> anyhow::Result> { local_registry.elected_guestos_records() } diff --git a/rs/cli/src/unit_tests/mod.rs b/rs/cli/src/unit_tests/mod.rs index 7fe1946b7..62e4f8c06 100644 --- a/rs/cli/src/unit_tests/mod.rs +++ b/rs/cli/src/unit_tests/mod.rs @@ -6,6 +6,7 @@ mod add_nodes; mod args_parse; mod health_client; mod node_labels; +mod registry_versions; mod replace; mod update_unassigned_nodes; mod version; diff --git a/rs/cli/src/unit_tests/registry_versions.rs b/rs/cli/src/unit_tests/registry_versions.rs new file mode 100644 index 000000000..67cbb4552 --- /dev/null +++ b/rs/cli/src/unit_tests/registry_versions.rs @@ -0,0 +1,269 @@ +use std::path::Path; +use std::sync::Arc; + +use futures::future::ok; +use ic_management_backend::{health::MockHealthStatusQuerier, lazy_git::MockLazyGit, lazy_registry::MockLazyRegistry, proposal::MockProposalAgent}; +use ic_management_types::Network; +use ic_registry_common_proto::pb::local_store::v1::{ChangelogEntry as PbChangelogEntry, KeyMutation as PbKeyMutation, MutationType}; +use prost::Message; +use serial_test::serial; + +use crate::{ + artifact_downloader::MockArtifactDownloader, auth::Neuron, commands::registry::Registry, cordoned_feature_fetcher::MockCordonedFeatureFetcher, + ctx::tests::get_mocked_ctx, ic_admin::MockIcAdmin, +}; + +fn hex_version(v: u64) -> String { + format!("{v:016x}") +} + +fn write_version(base: &Path, version: u64, mutations: Vec) { + let filename = format!("{}.pb", hex_version(version)); + let file_path = base.join(filename); + fs_err::create_dir_all(file_path.parent().unwrap()).unwrap(); + let entry = PbChangelogEntry { key_mutations: mutations }; + fs_err::write(file_path, entry.encode_to_vec()).unwrap(); +} + +#[tokio::test] +#[serial] +async fn dump_versions_outputs_records_sorted() { + // Arrange: write under the test fallback path used by implementation + // Constrain lookup to only our test dir + let base = std::path::PathBuf::from("/tmp/dre-test-store/local_registry/mainnet/t_dump_sorted"); + std::env::set_var("DRE_LOCAL_REGISTRY_DIR_OVERRIDE", &base); + + write_version( + &base, + 2, + vec![PbKeyMutation { + key: "k2".into(), + value: vec![2, 2], + mutation_type: MutationType::Set as i32, + }], + ); + write_version( + &base, + 1, + vec![PbKeyMutation { + key: "k1".into(), + value: vec![], + mutation_type: MutationType::Unset as i32, + }], + ); + + // Mock context + let mut ic_admin = MockIcAdmin::new(); + ic_admin.expect_simulate_proposal().returning(|_, _| Box::pin(async { Ok(()) })); + let mut git = MockLazyGit::new(); + git.expect_guestos_releases().returning(|| { + Box::pin(ok(Arc::new(ic_management_types::ArtifactReleases::new( + ic_management_types::Artifact::GuestOs, + )))) + }); + let mut registry = MockLazyRegistry::new(); + registry.expect_subnets().returning(|| Box::pin(ok(Arc::new(indexmap::IndexMap::new())))); + registry + .expect_unassigned_nodes_replica_version() + .returning(|| Box::pin(ok(Arc::new("some_ver".to_string())))); + let mut proposal_agent = MockProposalAgent::new(); + proposal_agent + .expect_list_open_elect_replica_proposals() + .returning(|| Box::pin(ok(vec![]))); + let mut artifact_downloader = MockArtifactDownloader::new(); + artifact_downloader + .expect_download_images_and_validate_sha256() + .returning(|_, _, _| Box::pin(async { Ok((vec![], String::new())) })); + + let ctx = get_mocked_ctx( + Network::mainnet_unchecked().unwrap(), + Neuron::anonymous_neuron(), + Arc::new(registry), + Arc::new(ic_admin), + Arc::new(git), + Arc::new(proposal_agent), + Arc::new(artifact_downloader), + Arc::new(MockCordonedFeatureFetcher::new()), + Arc::new(MockHealthStatusQuerier::new()), + ); + + // Act & Assert: query versions individually to avoid interference + let cmd_v1 = Registry { + output: None, + filters: vec![], + height: None, + dump_versions: None, + }; + let j1 = cmd_v1.dump_versions_json(ctx.clone()).await.unwrap(); + let a1 = j1.as_array().unwrap(); + assert_eq!(a1.len(), 1); + assert_eq!(a1[0]["version"].as_u64().unwrap(), 1); + assert_eq!(a1[0]["key"], "k1"); + + let cmd_v2 = Registry { + output: None, + filters: vec![], + height: None, + dump_versions: Some(vec![-1]), + }; + let j2 = cmd_v2.dump_versions_json(ctx).await.unwrap(); + let a2 = j2.as_array().unwrap(); + assert_eq!(a2.len(), 1); + assert_eq!(a2[0]["version"].as_u64().unwrap(), 2); + assert_eq!(a2[0]["key"], "k2"); +} + +#[tokio::test] +#[serial] +async fn list_versions_only_outputs_numbers() { + let base = std::path::PathBuf::from("/tmp/dre-test-store/local_registry/mainnet/t_list_only"); + std::env::set_var("DRE_LOCAL_REGISTRY_DIR_OVERRIDE", &base); + write_version( + &base, + 42, + vec![PbKeyMutation { + key: "k".into(), + value: vec![1], + mutation_type: MutationType::Set as i32, + }], + ); + + let mut ic_admin = MockIcAdmin::new(); + ic_admin.expect_simulate_proposal().returning(|_, _| Box::pin(async { Ok(()) })); + let mut git = MockLazyGit::new(); + git.expect_guestos_releases().returning(|| { + Box::pin(ok(Arc::new(ic_management_types::ArtifactReleases::new( + ic_management_types::Artifact::GuestOs, + )))) + }); + let mut registry = MockLazyRegistry::new(); + registry.expect_subnets().returning(|| Box::pin(ok(Arc::new(indexmap::IndexMap::new())))); + registry + .expect_unassigned_nodes_replica_version() + .returning(|| Box::pin(ok(Arc::new("some_ver".to_string())))); + let mut proposal_agent = MockProposalAgent::new(); + proposal_agent + .expect_list_open_elect_replica_proposals() + .returning(|| Box::pin(ok(vec![]))); + let mut artifact_downloader = MockArtifactDownloader::new(); + artifact_downloader + .expect_download_images_and_validate_sha256() + .returning(|_, _, _| Box::pin(async { Ok((vec![], String::new())) })); + + let ctx = get_mocked_ctx( + Network::mainnet_unchecked().unwrap(), + Neuron::anonymous_neuron(), + Arc::new(registry), + Arc::new(ic_admin), + Arc::new(git), + Arc::new(proposal_agent), + Arc::new(artifact_downloader), + Arc::new(MockCordonedFeatureFetcher::new()), + Arc::new(MockHealthStatusQuerier::new()), + ); + + let cmd = Registry { + output: None, + filters: vec![], + height: None, + dump_versions: Some(vec![0, -1]), + }; + let json = cmd.dump_versions_json(ctx).await.unwrap(); + let arr = json.as_array().unwrap(); + assert!(arr.iter().any(|e| e["version"] == 42)); +} + +#[tokio::test] +#[serial] +async fn dump_versions_rejects_reversed_range() { + // Arrange: write under the test fallback path used by implementation + let base = std::path::PathBuf::from("/tmp/dre-test-store/local_registry/mainnet/t_reversed_range"); + std::env::set_var("DRE_LOCAL_REGISTRY_DIR_OVERRIDE", &base); + + // Create a few versions + write_version( + &base, + 10, + vec![PbKeyMutation { + key: "a".into(), + value: vec![1], + mutation_type: MutationType::Set as i32, + }], + ); + write_version( + &base, + 20, + vec![PbKeyMutation { + key: "b".into(), + value: vec![2], + mutation_type: MutationType::Set as i32, + }], + ); + write_version( + &base, + 30, + vec![PbKeyMutation { + key: "c".into(), + value: vec![3], + mutation_type: MutationType::Set as i32, + }], + ); + + let mut ic_admin = MockIcAdmin::new(); + ic_admin.expect_simulate_proposal().returning(|_, _| Box::pin(async { Ok(()) })); + let mut git = MockLazyGit::new(); + git.expect_guestos_releases().returning(|| { + Box::pin(ok(Arc::new(ic_management_types::ArtifactReleases::new( + ic_management_types::Artifact::GuestOs, + )))) + }); + let mut registry = MockLazyRegistry::new(); + registry.expect_subnets().returning(|| Box::pin(ok(Arc::new(indexmap::IndexMap::new())))); + registry + .expect_unassigned_nodes_replica_version() + .returning(|| Box::pin(ok(Arc::new("some_ver".to_string())))); + let mut proposal_agent = MockProposalAgent::new(); + proposal_agent + .expect_list_open_elect_replica_proposals() + .returning(|| Box::pin(ok(vec![]))); + let mut artifact_downloader = MockArtifactDownloader::new(); + artifact_downloader + .expect_download_images_and_validate_sha256() + .returning(|_, _, _| Box::pin(async { Ok((vec![], String::new())) })); + + let ctx = get_mocked_ctx( + Network::mainnet_unchecked().unwrap(), + Neuron::anonymous_neuron(), + Arc::new(registry), + Arc::new(ic_admin), + Arc::new(git), + Arc::new(proposal_agent), + Arc::new(artifact_downloader), + Arc::new(MockCordonedFeatureFetcher::new()), + Arc::new(MockHealthStatusQuerier::new()), + ); + + // Valid negative range: last 2 (end-exclusive) + let ok_cmd = Registry { + output: None, + filters: vec![], + height: None, + dump_versions: Some(vec![-2]), + }; + let ok_json = ok_cmd.dump_versions_json(ctx.clone()).await.unwrap(); + let ok_arr = ok_json.as_array().unwrap(); + assert!(ok_arr + .iter() + .all(|e| e["version"].as_u64().unwrap() == 20 || e["version"].as_u64().unwrap() == 30)); + + // Reversed negative range should yield empty + let bad_cmd = Registry { + output: None, + filters: vec![], + height: None, + dump_versions: Some(vec![-1, -5]), + }; + let empty = bad_cmd.dump_versions_json(ctx).await.unwrap(); + let empty_arr = empty.as_array().unwrap(); + assert!(empty_arr.is_empty(), "expected empty result for reversed range [-1, -5]"); +}