Skip to content

Commit 0282259

Browse files
authored
Merge pull request #16329 from MinaProtocol/dkijania/expand_archive_tests
[Archive single node test] Implement archive test with mocked daemon
2 parents b3c64be + 17510fd commit 0282259

File tree

17 files changed

+319
-28
lines changed

17 files changed

+319
-28
lines changed

buildkite/scripts/build-artifact.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,5 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
5252
src/app/zkapp_limits/zkapp_limits.exe \
5353
src/lib/snark_worker/standalone/run_snark_worker.exe \
5454
src/test/command_line_tests/command_line_tests.exe \
55-
src/test/archive/patch_archive_test/patch_archive_test.exe
55+
src/test/archive/patch_archive_test/patch_archive_test.exe \
56+
src/test/archive/archive_node_tests/archive_node_tests.exe
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
let Artifacts = ../Constants/Artifacts.dhall
2+
3+
let Command = ./Base.dhall
4+
5+
let Size = ./Size.dhall
6+
7+
let RunWithPostgres = ./RunWithPostgres.dhall
8+
9+
let BuildFlags = ../Constants/BuildFlags.dhall
10+
11+
let key = "archive-node-test"
12+
13+
in { step =
14+
\(dependsOn : List Command.TaggedKey.Type)
15+
-> Command.build
16+
Command.Config::{
17+
, commands =
18+
[ RunWithPostgres.runInDockerWithPostgresConn
19+
[ "ARCHIVE_TEST_APP=mina-archive-node-test"
20+
, "MINA_TEST_NETWORK_DATA=/etc/mina/test/archive/sample_db"
21+
]
22+
"src/test/archive/sample_db/archive_db.sql"
23+
( Artifacts.fullDockerTag
24+
Artifacts.Tag::{
25+
, artifact = Artifacts.Type.FunctionalTestSuite
26+
, buildFlags = BuildFlags.Type.Instrumented
27+
}
28+
)
29+
"./scripts/tests/archive-node-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}"
30+
]
31+
, label = "Archive: Node Test"
32+
, key = key
33+
, target = Size.Large
34+
, depends_on = dependsOn
35+
}
36+
}

buildkite/src/Command/RunWithPostgres.dhall

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ let runInDockerWithPostgresConn
2626

2727
let dbName = "archive"
2828

29-
let pg_conn =
30-
"postgres://${user}:${password}@localhost:${port}/${dbName}"
29+
let pg_uri = "postgres://${user}:${password}@localhost:${port}"
30+
31+
let pg_conn = "${pg_uri}/${dbName}"
3132

3233
let envVars =
3334
Text/concatMap
@@ -37,6 +38,7 @@ let runInDockerWithPostgresConn
3738
, "POSTGRES_USER=${user}"
3839
, "POSTGRES_PASSWORD=${password}"
3940
, "POSTGRES_DB=${dbName}"
41+
, "POSTGRES_URI=${pg_uri}"
4042
, "PG_CONN=${pg_conn}"
4143
]
4244
# environment
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
let S = ../../Lib/SelectFiles.dhall
2+
3+
let Pipeline = ../../Pipeline/Dsl.dhall
4+
5+
let PipelineTag = ../../Pipeline/Tag.dhall
6+
7+
let JobSpec = ../../Pipeline/JobSpec.dhall
8+
9+
let ArchiveNodeTest = ../../Command/ArchiveNodeTest.dhall
10+
11+
let Artifacts = ../../Constants/Artifacts.dhall
12+
13+
let Dockers = ../../Constants/DockerVersions.dhall
14+
15+
let buildFlags = ../../Constants/BuildFlags.dhall
16+
17+
let dependsOn =
18+
Dockers.dependsOn
19+
Dockers.DepsSpec::{
20+
, artifact = Artifacts.Type.FunctionalTestSuite
21+
, buildFlags = buildFlags.Type.Instrumented
22+
}
23+
24+
in Pipeline.build
25+
Pipeline.Config::{
26+
, spec = JobSpec::{
27+
, dirtyWhen =
28+
[ S.strictlyStart (S.contains "src")
29+
, S.exactly "scripts/patch-archive-test" "sh"
30+
, S.exactly "buildkite/src/Jobs/Test/ArchiveNodeTest" "dhall"
31+
, S.exactly "buildkite/src/Command/ArchiveNodeTest" "dhall"
32+
]
33+
, path = "Test"
34+
, name = "ArchiveNodeTest"
35+
, tags =
36+
[ PipelineTag.Type.Long
37+
, PipelineTag.Type.Test
38+
, PipelineTag.Type.Stable
39+
]
40+
}
41+
, steps = [ ArchiveNodeTest.step dependsOn ]
42+
}

changes/16329.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Implemented component test for archive node. We are sending precomputed blocks from mocked daemon to an archive node and finally running replayer against it.

scripts/debian/builder-helpers.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,11 @@ build_functional_test_suite_deb() {
291291
"${BUILDDIR}/usr/local/bin/mina-zkapp-limits"
292292
cp ./default/src/test/archive/patch_archive_test/patch_archive_test.exe \
293293
"${BUILDDIR}/usr/local/bin/mina-patch-archive-test"
294+
cp ./default/src/test/archive/archive_node_tests/archive_node_tests.exe \
295+
"${BUILDDIR}/usr/local/bin/mina-archive-node-test"
296+
297+
mkdir -p ${BUILDDIR}/etc/mina/test/archive/sample_db
298+
rsync -Huav ../src/test/archive/sample_db* "${BUILDDIR}/etc/mina/test/archive"
294299

295300
build_deb mina-test-suite
296301

scripts/tests/archive-node-test.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
set -x
4+
# test archive node on known archive db
5+
6+
NETWORK_DATA_FOLDER=${NETWORK_DATA_FOLDER:-src/test/archive/sample_db}
7+
ARCHIVE_TEST_APP=${ARCHIVE_TEST_APP:-_build/default/src/test/archive/archive_node_tests/archive_node_tests.exe}
8+
# This env var is used in the test app
9+
# shellcheck disable=SC2034
10+
MINA_TEST_POSTGRES_URI=${POSTGRES_URI:-"postgres://postgres:postgres@localhost:5432"}
11+
12+
echo "Running archive node test"
13+
MINA_TEST_POSTGRES_URI=$MINA_TEST_POSTGRES_URI $ARCHIVE_TEST_APP
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
open Async
2+
open Core
3+
open Mina_automation
4+
open Mina_automation_runner
5+
open Mina_automation_fixture.Archive
6+
7+
(**
8+
* Test the basic functionality of the mina archive with mocked deamon
9+
*)
10+
11+
(* asserts count of archive blocked (we are skipping genesis block) *)
12+
let assert_archived_blocks ~archive_uri ~expected =
13+
let connection = Psql.Conn_str archive_uri in
14+
let%bind actual_blocks_count =
15+
Psql.run_command ~connection
16+
"SELECT COUNT(*) FROM blocks WHERE global_slot_since_genesis > 1"
17+
in
18+
let actual_blocks_count =
19+
match actual_blocks_count with
20+
| Ok count ->
21+
count |> Int.of_string
22+
| Error err ->
23+
failwith ("Failed to query blocks count: " ^ Error.to_string_hum err)
24+
in
25+
if Int.( <> ) actual_blocks_count expected then
26+
failwithf "Invalid number of archive blocks. Actual (%d) vs Expected (%d)"
27+
actual_blocks_count expected ()
28+
else Deferred.unit
29+
30+
module ArchivePrecomputedBlocksFromDaemon = struct
31+
type t = Mina_automation_fixture.Archive.after_bootstrap
32+
33+
let test_case (test_data : t) =
34+
let open Deferred.Let_syntax in
35+
let daemon = Daemon.default in
36+
let archive_uri = test_data.archive.config.postgres_uri in
37+
let output = test_data.temp_dir in
38+
let%bind precomputed_blocks =
39+
Network_data.untar_precomputed_blocks test_data.network_data output
40+
in
41+
let precomputed_blocks =
42+
List.map precomputed_blocks ~f:(fun file -> output ^ "/" ^ file)
43+
|> List.filter ~f:(fun file -> String.is_suffix file ~suffix:".json")
44+
in
45+
Archive.Process.start_logging test_data.archive ;
46+
let%bind () =
47+
Daemon.archive_blocks_from_files daemon
48+
~archive_address:test_data.archive.config.server_port
49+
~format:Archive_blocks.Precomputed precomputed_blocks
50+
in
51+
52+
let%bind () =
53+
assert_archived_blocks ~archive_uri
54+
~expected:(List.length precomputed_blocks)
55+
in
56+
let connection = Psql.Conn_str archive_uri in
57+
let%bind latest_state_hash =
58+
Psql.run_command ~connection
59+
"SELECT state_hash FROM blocks ORDER BY id DESC LIMIT 1"
60+
in
61+
let latest_state_hash =
62+
match latest_state_hash with
63+
| Ok hash ->
64+
hash
65+
| Error err ->
66+
failwith
67+
("Failed to query latest state hash: " ^ Error.to_string_hum err)
68+
in
69+
let output_ledger = output ^ "/output_ledger.json" in
70+
let replayer = Replayer.default in
71+
let%bind replayer_output =
72+
Replayer.run replayer ~archive_uri
73+
~input_config:
74+
(Network_data.replayer_input_file_path test_data.network_data)
75+
~target_state_hash:latest_state_hash ~interval_checkpoint:10
76+
~output_ledger ()
77+
in
78+
let () = print_endline replayer_output in
79+
let output_ledger = Replayer.Output.of_json_file_exn output_ledger in
80+
assert (
81+
String.equal output_ledger.target_epoch_ledgers_state_hash
82+
latest_state_hash ) ;
83+
Deferred.Or_error.return Mina_automation_fixture.Intf.Passed
84+
end
85+
86+
let () =
87+
Backtrace.elide := false ;
88+
Async.Scheduler.set_record_backtraces true
89+
90+
let () =
91+
let open Alcotest in
92+
run "Test archive node."
93+
[ ( "precomputed blocks"
94+
, [ test_case "The mina daemon works in background mode" `Quick
95+
(Runner.run_blocking
96+
( module Mina_automation_fixture.Archive.Make_FixtureWithBootstrap
97+
(ArchivePrecomputedBlocksFromDaemon) ) )
98+
] )
99+
]
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
(tests
2+
(names archive_node_tests)
3+
(libraries
4+
;; opam libraries
5+
async_kernel
6+
core_kernel
7+
ppx_inline_test.config
8+
async
9+
core
10+
async_unix
11+
stdio
12+
alcotest
13+
cmdliner
14+
;; mina libraries
15+
mina_automation
16+
mina_automation.fixture
17+
mina_automation.runner
18+
)
19+
(instrumentation (backend bisect_ppx))
20+
(preprocess (pps ppx_version ppx_jane ppx_compare))
21+
)

src/test/mina_automation/daemon.ml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,11 +148,11 @@ let archive_blocks t ~archive_address ~(format : Archive_blocks.format) blocks =
148148
]
149149
@ blocks )
150150

151-
let dispatch_blocks t ~archive_address ~(format : Archive_blocks.format)
152-
?(sleep = 5) blocks =
151+
let archive_blocks_from_files t ~archive_address
152+
~(format : Archive_blocks.format) ?(sleep = 5) blocks =
153153
Deferred.List.iter blocks ~f:(fun block ->
154-
Core.Unix.sleep sleep ;
155-
archive_blocks t ~archive_address ~format [ block ] () >>| ignore )
154+
let%bind _ = archive_blocks t ~archive_address ~format [ block ] () in
155+
after (Time.Span.of_sec (Float.of_int sleep)) )
156156

157157
let default = Executor.default
158158

0 commit comments

Comments
 (0)