Skip to content

Commit 67b4300

Browse files
riptlripatel-fd
riptl
authored and
ripatel-fd
committed
solfuzz: belt sanding
- Flatten directory structure - Add venv to gitignore - Define and document separation of layers - Centralize memory management of 'solfuzz executor' - Use demand-paged THP workspace memory for solfuzz (Greatly reduces startup time with minimal prod perf impact) - Allow reusing a preallocated workspace for run-test-vectors
1 parent 8f164f4 commit 67b4300

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+1797
-1564
lines changed

‎contrib/test/run_test_vectors.sh

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#!/bin/bash
22

3+
# FIXME This whole file should just really be a firedancer-dev
4+
# invocation with parallelism natively implemented in C.
5+
36
set -ex
47

58
DIR="$( dirname -- "${BASH_SOURCE[0]}"; )"; # Get the directory name
@@ -8,6 +11,8 @@ cd $DIR/../..
811

912
OBJDIR=${OBJDIR:-build/native/gcc}
1013
NUM_PROCESSES=${NUM_PROCESSES:-12}
14+
PAGE_SZ=gigantic
15+
PAGE_CNT=$(( 6 * $NUM_PROCESSES ))
1116

1217
if [ "$LOG_PATH" == "" ]; then
1318
LOG_PATH="`mktemp -d`"
@@ -30,27 +35,39 @@ else
3035
cd dump/test-vectors
3136
fi
3237

33-
git fetch -q --depth=1 origin $GIT_REF
34-
git checkout -q $GIT_REF
38+
if ! git checkout -q $GIT_REF; then
39+
git fetch -q --depth=1 origin $GIT_REF
40+
git checkout -q FETCH_HEAD
41+
fi
3542
cd ../..
3643

37-
LOG=$LOG_PATH/test_exec_block
38-
find dump/test-vectors/block/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
44+
WKSP=run-test-vectors
45+
# If workspace already exists, reset it (and hope that it has the correct size)
46+
if ./$OBJDIR/bin/fd_wksp_ctl query $WKSP --log-path '' >/dev/null 2>/dev/null; then
47+
./$OBJDIR/bin/fd_wksp_ctl reset $WKSP --log-path ''
48+
else
49+
./$OBJDIR/bin/fd_wksp_ctl new run-test-vectors $PAGE_CNT $PAGE_SZ 0 0644 --log-path ''
50+
fi
51+
52+
SOL_COMPAT=( "$OBJDIR/unit-test/test_sol_compat" "--wksp" "$WKSP" )
53+
54+
export FD_LOG_PATH=$LOG_PATH/test_exec_block
55+
find dump/test-vectors/block/fixtures -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ${SOL_COMPAT[@]}
3956

40-
LOG=$LOG_PATH/test_exec_syscall
41-
find dump/test-vectors/syscall/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
57+
export FD_LOG_PATH=$LOG_PATH/test_exec_syscall
58+
find dump/test-vectors/syscall/fixtures -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ${SOL_COMPAT[@]}
4259

43-
LOG=$LOG_PATH/test_exec_interp
44-
find dump/test-vectors/vm_interp/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
60+
export FD_LOG_PATH=$LOG_PATH/test_exec_interp
61+
find dump/test-vectors/vm_interp/fixtures -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ${SOL_COMPAT[@]}
4562

46-
LOG=$LOG_PATH/test_exec_txn
47-
find dump/test-vectors/txn/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
63+
export FD_LOG_PATH=$LOG_PATH/test_exec_txn
64+
find dump/test-vectors/txn/fixtures -type f -name '*.fix' | xargs -P $NUM_PROCESSES ${SOL_COMPAT[@]}
4865

4966
zstd -df dump/test-vectors/elf_loader/fixtures/*.zst
50-
LOG=$LOG_PATH/test_elf_loader
51-
find dump/test-vectors/elf_loader/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
67+
export FD_LOG_PATH=$LOG_PATH/test_elf_loader
68+
find dump/test-vectors/elf_loader/fixtures -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ${SOL_COMPAT[@]}
5269

53-
LOG=$LOG_PATH/test_exec_instr
54-
find dump/test-vectors/instr/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
70+
export FD_LOG_PATH=$LOG_PATH/test_exec_instr
71+
find dump/test-vectors/instr/fixtures -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ${SOL_COMPAT[@]}
5572

5673
echo Test vectors success

‎src/flamenco/capture/fd_solcap_writer.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
#include "fd_solcap_proto.h"
55
#include "fd_solcap.pb.h"
66
#include "../types/fd_types.h"
7-
#include "../runtime/fd_runtime_public.h"
87

98
/* fd_solcap_writer_t is an opaque handle to a capture writer object.
109
Currently, it implements writing SOLCAP_V1_BANK files. See below

‎src/flamenco/fd_flamenco_base.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ fd_base58_enc_64_fmt( char * out,
7575
struct fd_bank;
7676
typedef struct fd_bank fd_bank_t;
7777

78+
struct fd_banks;
79+
typedef struct fd_banks fd_banks_t;
80+
7881
struct fd_exec_slot_ctx;
7982
typedef struct fd_exec_slot_ctx fd_exec_slot_ctx_t;
8083

‎src/flamenco/runtime/context/fd_capture_ctx.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#define HEADER_fd_src_flamenco_runtime_context_fd_capture_ctx_h
33

44
#include "../../capture/fd_solcap_writer.h"
5+
#include "../fd_runtime_public.h" /* FD_RUNTIME_PUBLIC_ACCOUNT_UPDATE_MSG_FOOTPRINT */
56

67
/* Maximum number of accounts that can be updated in a single transaction */
78
#define FD_CAPTURE_CTX_MAX_ACCOUNT_UPDATES (128UL)

‎src/flamenco/runtime/fd_executor.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010
#include "../../ballet/poh/fd_poh.h"
1111
#include "../types/fd_types_yaml.h"
1212
#include "../log_collector/fd_log_collector.h"
13-
#include "tests/harness/generated/invoke.pb.h"
14-
#include "tests/harness/generated/txn.pb.h"
1513
#include "../features/fd_features.h"
1614
#include "fd_runtime.h"
1715

‎src/flamenco/runtime/test_txn_rw_conflicts.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
#include "../../util/fd_util_base.h"
2-
#include "../fd_flamenco_base.h"
31
#include "fd_acc_mgr.h"
42
#include "fd_runtime.h"
53
#include "fd_runtime_err.h"

‎src/flamenco/runtime/tests/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
nanopb_venv/

‎src/flamenco/runtime/tests/Local.mk

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,23 @@
1-
$(call add-hdrs,harness/generated/context.pb.h,harness/generated/elf.pb.h,harness/generated/invoke.pb.h,harness/generated/txn.pb.h,harness/generated/block.pb.h,harness/generated/vm.pb.h,harness/generated/type.pb.h,harness/generated/shred.pb.h harness/generated/metadata.pb.h)
2-
$(call add-objs,harness/generated/context.pb harness/generated/elf.pb harness/generated/invoke.pb harness/generated/txn.pb harness/generated/block.pb harness/generated/vm.pb harness/generated/type.pb harness/generated/shred.pb harness/generated/metadata.pb,fd_flamenco)
3-
41
ifdef FD_HAS_INT128
52
ifdef FD_HAS_SECP256K1
6-
$(call add-hdrs,harness/fd_elf_harness.h harness/fd_instr_harness.h harness/fd_txn_harness.h harness/fd_block_harness.h harness/fd_harness_common.h harness/fd_vm_harness.h harness/fd_types_harness.h)
7-
$(call add-objs,harness/fd_elf_harness harness/fd_instr_harness harness/fd_txn_harness harness/fd_block_harness harness/fd_harness_common harness/fd_vm_harness harness/fd_types_harness,fd_flamenco_test)
8-
$(call add-objs,harness/fd_exec_sol_compat,fd_flamenco_test)
3+
4+
$(call add-hdrs,fd_solfuzz.h)
5+
$(call add-objs,fd_solfuzz fd_solfuzz_exec,fd_flamenco_test)
6+
7+
$(call add-hdrs,fd_instr_harness.h fd_txn_harness.h)
8+
$(call add-objs,fd_elf_harness fd_instr_harness fd_txn_harness fd_block_harness fd_harness_common fd_vm_harness fd_types_harness,fd_flamenco_test)
9+
$(call add-objs,fd_sol_compat,fd_flamenco_test)
10+
11+
$(call add-hdrs,generated/context.pb.h,generated/elf.pb.h,generated/invoke.pb.h,generated/txn.pb.h,generated/block.pb.h,generated/vm.pb.h,generated/type.pb.h,generated/shred.pb.h generated/metadata.pb.h)
12+
$(call add-objs,generated/context.pb generated/elf.pb generated/invoke.pb generated/txn.pb generated/block.pb generated/vm.pb generated/type.pb generated/shred.pb generated/metadata.pb,fd_flamenco)
913

1014
SOL_COMPAT_FLAGS:=-Wl,--undefined=fd_types_vt_by_name
11-
$(call make-unit-test,test_exec_sol_compat,test_exec_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS))
12-
$(call make-shared,libfd_exec_sol_compat.so,harness/fd_exec_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS) $(SOL_COMPAT_FLAGS))
15+
$(call make-unit-test,test_sol_compat,test_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS))
16+
$(call make-shared,libfd_exec_sol_compat.so,fd_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS) $(SOL_COMPAT_FLAGS))
17+
18+
run-runtime-backtest: $(OBJDIR)/bin/fd_ledger
19+
OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_backtest_ci.sh
1320

1421
endif
1522
endif
1623

17-
run-runtime-backtest: $(OBJDIR)/bin/fd_ledger
18-
OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_backtest_ci.sh

‎src/flamenco/runtime/tests/Makefile

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎src/flamenco/runtime/tests/README.md

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# solfuzz APIs
2+
3+
solfuzz wraps the Solana runtime (SVM) in a generic Protobuf format.
4+
This allows users to execute inputs against different SVM
5+
implementations such as Agave, Firedancer, Mithril, or Sig.
6+
7+
This directory provides the Firedancer SVM backend for solfuzz.
8+
9+
## Internal design
10+
11+
This integration is layered as follows top to bottom:
12+
13+
- `sol_compat` (solfuzz public ABI)
14+
- `fd_solfuzz` (internal APIs for solfuzz)
15+
- `fd_runtime` (internal APIs for the Firedancer SVM)
16+
17+
i.e., if a user executes a Solana transaction via `sol_compat`, it is
18+
passed down to `fd_solfuzz`, which in turn executes the transaction in
19+
`fd_runtime`.
20+
21+
### sol_compat layer
22+
23+
`sol_compat` is a C API defined by `solfuzz`.
24+
It is stable-ABI (no breaking changes to symbol names, struct layouts,
25+
and function signatures).
26+
27+
The Firedancer build outputs a `libfd_exec_sol_compat.so` shared library
28+
containing an implementation of the `sol_compat` C API. External users
29+
like `solfuzz` or `solana-conformance` use this API.
30+
31+
See [fd_sol_compat.h](./fd_sol_compat.h).
32+
33+
### fd_solfuzz layer
34+
35+
Like `sol_compat`, `fd_solfuzz` uses Protobuf as its input and output
36+
formats. `fd_solfuzz` is not a stable API but supports a number of
37+
advanced features useful for internal use.
38+
39+
Mainly used by command-line tooling and tests in the Firedancer repo
40+
(e.g. the `fd_exec_sol_compat` executable).
41+
42+
See [fd_solfuzz.h](./fd_solfuzz.h).
43+
44+
### fd_runtime layer
45+
46+
The actual Firedancer SVM. See `src/flamenco/runtime`.

0 commit comments

Comments
 (0)