Merge pull request #18032 from MinaProtocol/lyh/dkijania/patch-ldb

Implement an alternative to ldb that statically links to Mina so rocksdb version matches

Author: dkijania

Makefile

@@ -164,6 +164,7 @@ build-daemon-utils: ocaml_checks reformat-diff libp2p_helper ## Build daemon uti
 		src/app/validate_keypair/validate_keypair.exe \
 		src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \
 		src/lib/snark_worker/standalone/run_snark_worker.exe \
+		src/app/rocksdb-scanner/rocksdb_scanner.exe \
 		--profile=$(DUNE_PROFILE) \
 		&& echo "✅ Build complete"
 

scripts/debian/builder-helpers.sh

@@ -169,6 +169,9 @@ copy_common_daemon_configs() {
     "${BUILDDIR}/usr/local/bin/mina-validate-keypair"
   cp ./default/src/lib/snark_worker/standalone/run_snark_worker.exe \
     "${BUILDDIR}/usr/local/bin/mina-standalone-snark-worker"
+  cp ./default/src/app/rocksdb-scanner/rocksdb_scanner.exe \
+    "${BUILDDIR}/usr/local/bin/mina-rocksdb-scanner"
+
   # Copy signature-based Binaries (based on signature type $2 passed into the \
   # function)
   cp ./default/src/app/cli/src/mina_"${2}"_signatures.exe \
@@ -861,4 +864,4 @@ build_create_legacy_genesis_deb() {
 
   build_deb mina-create-legacy-genesis
 }
-## END CREATE LEGACY GENESIS PACKAGE ##
+## END CREATE LEGACY GENESIS PACKAGE ##

scripts/hardfork/mina-verify-packaged-fork-config

@@ -56,6 +56,7 @@ MINA_EXE=${MINA_EXE:-$(source_build_fallback mina ./_build/default/src/app/cli/s
 MINA_GENESIS_EXE=${MINA_GENESIS_EXE:-$(source_build_fallback mina-create-genesis ./_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe)}
 MINA_LEGACY_GENESIS_EXE=${MINA_LEGACY_GENESIS_EXE:-$(source_build_fallback mina-create-legacy-genesis ./runtime_genesis_ledger_of_mainnet.exe)}
 CREATE_RUNTIME_CONFIG=${CREATE_RUNTIME_CONFIG:-$(source_build_fallback mina-hf-create-runtime-config ./scripts/hardfork/create_runtime_config.sh)}
+MINA_ROCKSDB_SCANNER=${MINA_ROCKSDB_SCANNER:-$(source_build_fallback mina-rocksdb-scanner ./_build/default/src/app/rocksdb-scanner/rocksdb_scanner.exe)}
 GSUTIL=${GSUTIL:-$(command -v gsutil || echo "")}
 
 if [[ ! -e "$PRECOMPUTED_FORK_BLOCK" && ( -z "$GSUTIL" || ! -x "$GSUTIL" ) ]]; then
@@ -206,7 +207,6 @@ for file in "$workdir"/packaged-*.json; do
     fi
 done
 
-ldb_cmd=$(command -v rocksdb-ldb || command -v rocksdb_ldb || command -v ldb)
 for file in "$workdir"/ledgers/*.tar.gz; do
     tarname=$(basename "$file")
     tarname=${tarname%.tar.gz}
@@ -218,9 +218,9 @@ for file in "$workdir"/ledgers/*.tar.gz; do
     base_s3_url="${MINA_LEDGER_S3_BUCKET:-https://s3-us-west-2.amazonaws.com/snark-keys-ro.o1test.net}"
     curl "$base_s3_url/$tarname.tar.gz" | tar -xz -C "$tardir/web"
 
-    $ldb_cmd --hex --db="$tardir/packaged" scan > "$workdir/packaged.scan"
-    $ldb_cmd --hex --db="$tardir/web" scan > "$workdir/web.scan"
-    $ldb_cmd --hex --db="$tardir/generated" scan > "$workdir/generated.scan"
+    for kind in {packaged,web,generated}; do
+        $MINA_ROCKSDB_SCANNER --db-path "$tardir/$kind" --output-file "$workdir/$kind.scan"
+    done
 
     if ! cmp "$workdir/generated.scan" "$workdir/packaged.scan" || ! cmp "$workdir/packaged.scan" "$workdir/web.scan"; then
         echo "Error: kvdb contents mismatch for $tarname" >&2

src/app/rocksdb-scanner/dune

@@ -0,0 +1,14 @@
+(executable
+ (package rocksdb_scanner)
+ (name rocksdb_scanner)
+ (public_name mina_rocksdb_scanner)
+ (modes native)
+ (libraries
+  async
+  core
+  logger
+  rocksdb)
+ (instrumentation
+  (backend bisect_ppx))
+ (preprocess
+  (pps ppx_jane ppx_mina ppx_version)))

src/app/rocksdb-scanner/rocksdb_scanner.ml

@@ -0,0 +1,38 @@
+open Core
+open Async
+
+(* RocksDB scanning function that replaces ldb command *)
+let scan_rocksdb_to_and_dump_hex_kvs db_path output_file =
+  let db = Rocksdb.Database.create db_path in
+  let kv_pairs = Rocksdb.Database.to_alist db in
+  let%bind writer = Writer.open_file output_file in
+  List.iter kv_pairs ~f:(fun (key, value) ->
+      let key_hex =
+        Bigstring.to_string key
+        |> String.concat_map ~f:(fun c -> sprintf "%02x" (Char.to_int c))
+      in
+      let value_hex =
+        Bigstring.to_string value
+        |> String.concat_map ~f:(fun c -> sprintf "%02x" (Char.to_int c))
+      in
+      Writer.writef writer "%s : %s\n" key_hex value_hex ) ;
+  let%map () = Writer.close writer in
+  Rocksdb.Database.close db
+
+let root_command =
+  Command.async ~summary:"Scan RocksDB and dump key-value pairs as hex strings"
+    (let%map_open.Command db_path =
+       flag "--db-path" (required string)
+         ~doc:"PATH address of Rocksdb database"
+     and output_file =
+       flag "--output-file" (required string)
+         ~doc:"PATH of file to dump scanning result"
+     in
+     fun () ->
+       let logger = Logger.create () in
+       [%log info] "Scanning RocksDB at %s to output file %s" db_path
+         output_file ;
+       let%map () = scan_rocksdb_to_and_dump_hex_kvs db_path output_file in
+       [%log info] "Successfully dumped data to %s" output_file )
+
+let () = Command.run root_command

src/dune-project

@@ -160,6 +160,7 @@
 (package (name random_oracle))
 (package (name replayer))
 (package (name rocksdb))
+(package (name rocksdb_scanner))
 (package (name rosetta_coding))
 (package (name rosetta_lib))
 (package (name rosetta_models))