Merge pull request #17769 from MinaProtocol/dkijania/single_archive_node_upgrade_script

[HF][Upgrade Script] Upgrade script for single archive node test

Author: glyh

src/test/archive/archive_node_tests/archive_node_tests.ml

@@ -23,4 +23,13 @@ let () =
                         .Make_FixtureWithoutBootstrap
                           (Load_genesis_ledger) ) )
         ] )
+    ; ( "upgrade_archive"
+      , [ test_case
+            "Recreate database from precomputed blocks after upgrade script is \
+             applied"
+            `Quick
+            (Runner.run_blocking
+               ( module Mina_automation_fixture.Archive.Make_FixtureWithBootstrap
+                          (Upgrade_archive) ) )
+        ] )
     ]

src/test/archive/archive_node_tests/archive_precomputed_blocks_test.ml

@@ -2,6 +2,7 @@ open Async
 open Core
 open Mina_automation
 open Mina_automation_fixture.Archive
+open Common
 
 (**
  * Test the basic functionality of the mina archive with mocked deamon
@@ -118,11 +119,8 @@ let test_case (test_data : t) =
   let archive_uri = test_data.archive.config.postgres_uri in
   let output = test_data.temp_dir in
   let%bind precomputed_blocks =
-    Network_data.untar_precomputed_blocks test_data.network_data output
-  in
-  let precomputed_blocks =
-    List.map precomputed_blocks ~f:(fun file -> output ^ "/" ^ file)
-    |> List.filter ~f:(fun file -> String.is_suffix file ~suffix:".json")
+    unpack_precomputed_blocks test_data.network_data
+      ~temp_dir:test_data.temp_dir
   in
   let log_file = output ^ "/precomputed_blocks_test.log" in
   Archive.Process.start_logging test_data.archive ~log_file ;
@@ -136,32 +134,13 @@ let test_case (test_data : t) =
     assert_archived_blocks ~archive_uri
       ~expected:(List.length precomputed_blocks)
   in
-  let connection = Psql.Conn_str archive_uri in
-  let%bind latest_state_hash =
-    Psql.run_command ~connection
-      "SELECT state_hash FROM blocks ORDER BY id DESC LIMIT 1"
-  in
-  let latest_state_hash =
-    match latest_state_hash with
-    | Ok hash ->
-        hash
-    | Error err ->
-        failwith
-          ("Failed to query latest state hash: " ^ Error.to_string_hum err)
-  in
-  let output_ledger = output ^ "/output_ledger.json" in
-  let replayer = Replayer.default in
-  let%bind replayer_output =
-    Replayer.run replayer ~archive_uri
-      ~input_config:
+
+  let%bind () =
+    assert_replayer_run_against_last_block
+      ~replayer_input_file_path:
         (Network_data.replayer_input_file_path test_data.network_data)
-      ~target_state_hash:latest_state_hash ~interval_checkpoint:10
-      ~output_ledger ()
+      archive_uri test_data.temp_dir
   in
-  let () = print_endline replayer_output in
-  let output_ledger = Replayer.Output.of_json_file_exn output_ledger in
-  assert (
-    String.equal output_ledger.target_epoch_ledgers_state_hash latest_state_hash ) ;
 
   let%bind perf_data = extract_perf_metrics log_file in
   perf_metrics_to_yojson perf_data |> Yojson.to_file "archive.perf" ;

src/test/archive/archive_node_tests/common.ml

@@ -0,0 +1,60 @@
+open Async
+open Core
+open Mina_automation
+
+(** Assert that the replayer runs successfully against the latest block in the archive.
+
+  This function queries the archive database to find the latest state hash,
+  then runs the replayer with the specified input configuration file against
+  that target state hash. It verifies that the output ledger's target epoch
+  ledgers state hash matches the latest state hash from the database.
+
+  @param replayer_input_file_path Path to the replayer input configuration file
+  @param archive_uri URI connection string for the archive database
+  @param output Output directory path where the ledger will be written
+  @return Unit deferred that completes when assertion passes *)
+let assert_replayer_run_against_last_block ~replayer_input_file_path archive_uri
+    output =
+  let open Deferred.Let_syntax in
+  let connection = Psql.Conn_str archive_uri in
+  let%bind latest_state_hash =
+    Psql.run_command ~connection
+      "SELECT state_hash FROM blocks ORDER BY id DESC LIMIT 1"
+  in
+  let latest_state_hash =
+    match latest_state_hash with
+    | Ok hash ->
+        hash
+    | Error err ->
+        failwith
+          ("Failed to query latest state hash: " ^ Error.to_string_hum err)
+  in
+  let output_ledger = output ^ "/output_ledger.json" in
+  let replayer = Replayer.default in
+  let%bind replayer_output =
+    Replayer.run replayer ~archive_uri ~input_config:replayer_input_file_path
+      ~target_state_hash:latest_state_hash ~interval_checkpoint:10
+      ~output_ledger ()
+  in
+  let () = print_endline replayer_output in
+  let output_ledger = Replayer.Output.of_json_file_exn output_ledger in
+  assert (
+    String.equal output_ledger.target_epoch_ledgers_state_hash latest_state_hash ) ;
+  Deferred.unit
+
+(** Unpack precomputed blocks from a source archive to a temporary directory.
+
+  This function extracts precomputed blocks from a tar archive into the specified
+  temporary directory and filters the results to only include JSON files.
+
+  @param temp_dir Temporary directory path where blocks will be extracted
+  @param source Source path of the tar archive containing precomputed blocks
+  @return Deferred list of file paths to extracted JSON block files *)
+let unpack_precomputed_blocks ~temp_dir source =
+  let open Deferred.Let_syntax in
+  let%bind precomputed_blocks =
+    Network_data.untar_precomputed_blocks source temp_dir
+  in
+  List.map precomputed_blocks ~f:(fun file -> temp_dir ^ "/" ^ file)
+  |> List.filter ~f:(fun file -> String.is_suffix file ~suffix:".json")
+  |> Deferred.return

src/test/archive/archive_node_tests/upgrade_archive.ml

@@ -0,0 +1,40 @@
+open Async
+open Core
+open Mina_automation
+open Mina_automation_fixture.Archive
+open Common
+
+type t = Mina_automation_fixture.Archive.after_bootstrap
+
+let test_case (test_data : t) =
+  let open Deferred.Let_syntax in
+  let daemon = Daemon.default () in
+  let archive_uri = test_data.archive.config.postgres_uri in
+  let temp_dir = test_data.temp_dir in
+  let%bind precomputed_blocks =
+    unpack_precomputed_blocks ~temp_dir test_data.network_data
+  in
+  let log_file = temp_dir ^ "/upgrade.log" in
+  let upgrade_path =
+    Archive.Scripts.filepath `Upgrade
+    |> Option.value_exn ~message:"Failed to find upgrade script"
+  in
+  let%bind _ =
+    Psql.run_script ~connection:(Psql.Conn_str archive_uri) upgrade_path
+  in
+
+  Archive.Process.start_logging test_data.archive ~log_file ;
+  let%bind () =
+    Daemon.archive_blocks_from_files daemon.executor
+      ~archive_address:test_data.archive.config.server_port ~format:`Precomputed
+      precomputed_blocks
+  in
+
+  let%bind () =
+    assert_replayer_run_against_last_block
+      ~replayer_input_file_path:
+        (Network_data.replayer_input_file_path test_data.network_data)
+      archive_uri temp_dir
+  in
+
+  Deferred.Or_error.return Mina_automation_fixture.Intf.Passed

src/test/mina_automation/archive.ml

@@ -35,6 +35,28 @@ module Paths = struct
   let official_name = "mina-archive"
 end
 
+module Scripts = struct
+  type t = [ `CreateSchema | `DropTables | `Upgrade | `Rollback ]
+
+  let possible_locations = [ "/etc/mina/archive"; "src/app/archive" ]
+
+  let file t =
+    match t with
+    | `CreateSchema ->
+        "create_schema.sql"
+    | `DropTables ->
+        "drop_tables.sql"
+    | `Upgrade ->
+        "upgrade-to-mesa.sql"
+    | `Rollback ->
+        "rollback_to-berkeley.sql"
+
+  let filepath t =
+    let file = file t in
+    let possible_locations = [ "/etc/mina/archive"; "src/app/archive" ] in
+    Utils.possible_locations ~file possible_locations
+end
+
 module Executor = Executor.Make (Paths)
 
 type t = { config : Config.t; executor : Executor.t }

src/test/mina_automation/archive_blocks.ml

@@ -23,8 +23,8 @@ let path () =
   Deferred.map Executor.PathFinder.standalone_path ~f:(fun opt ->
       Option.value_exn opt
         ~message:
-          "Could not find standalone path for archive blocks. App is not \
-           executable outside the dune" )
+          "Could not find standalone path for archive blocks. Archive blocks \
+           is not executable outside the dune" )
 
 let format_to_string format =
   match format with

src/test/mina_automation/daemon.ml

@@ -188,13 +188,13 @@ let archive_blocks t ~archive_address ~format blocks =
         ]
       @ blocks )
 
+type t = { config : Config.t; executor : Executor.t }
+
 let archive_blocks_from_files t ~archive_address ~format ?(sleep = 5) blocks =
   Deferred.List.iter blocks ~f:(fun block ->
       let%bind _ = archive_blocks t ~archive_address ~format [ block ] () in
       after (Time.Span.of_sec (Float.of_int sleep)) )
 
-type t = { config : Config.t; executor : Executor.t }
-
 let of_config config = { config; executor = Executor.AutoDetect }
 
 let default () = { config = Config.default (); executor = Executor.AutoDetect }

src/test/mina_automation/executor.ml

@@ -33,9 +33,6 @@ let logger = Logger.create ()
 module Make_PathFinder (P : AppPaths) = struct
   module Paths = P
 
-  let paths =
-    Option.value_map ~f:(String.split ~on:':') ~default:[] (Sys.getenv "PATH")
-
   let built_name = Printf.sprintf "_build/default/%s" P.dune_name
 
   let exists_at_path path prefix =
@@ -51,7 +48,7 @@ module Make_PathFinder (P : AppPaths) = struct
         Deferred.return (Some built_name)
     | _ -> (
         match%bind
-          Deferred.List.find_map ~f:(exists_at_path P.official_name) paths
+          Deferred.List.find_map ~f:(exists_at_path P.official_name) Utils.paths
         with
         | Some _ ->
             Deferred.return (Some P.official_name)
@@ -133,7 +130,7 @@ module Make (P : AppPaths) = struct
             match%bind
               Deferred.List.find_map
                 ~f:(PathFinder.exists_at_path PathFinder.Paths.official_name)
-                PathFinder.paths
+                Utils.paths
             with
             | Some prefix ->
                 f_debian ~args ~prefix ?env ()

src/test/mina_automation/missing_blocks_auditor.ml

@@ -17,5 +17,5 @@ let path () =
   Deferred.map PathFinder.standalone_path ~f:(fun opt ->
       Option.value_exn opt
         ~message:
-          "Could not find standalone path for missing block auditor. App is \
-           not executable outside the dune" )
+          "Could not find standalone for missing block auditor. Missing block \
+           auditor is not executable outside the dune" )

src/test/mina_automation/psql.ml

@@ -82,9 +82,10 @@ let run_command ~connection command =
   @return A deferred string containing the output of the command.
 *)
 
-let run_script ~connection ~db script =
+let run_script ~connection ?db script =
+  let maybe_db = Option.value_map db ~default:[] ~f:(fun db -> [ "-d"; db ]) in
   let creds = create_credential_arg ~connection in
-  Util.run_cmd_exn "." psql (creds @ [ "-d"; db; "-a"; "-f"; script ])
+  Util.run_cmd_exn "." psql (creds @ maybe_db @ [ "-a"; "-f"; script ])
 
 let create_empty_db ~connection ~db =
   run_command ~connection (sprintf "CREATE DATABASE %s;" db)