solfuzz belt sanding

contrib/test/run_test_vectors.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# FIXME This whole file should just really be a firedancer-dev
+#       invocation with parallelism natively implemented in C.
+
 set -ex
 
 DIR="$( dirname -- "${BASH_SOURCE[0]}"; )";   # Get the directory name
@@ -8,6 +11,8 @@ cd $DIR/../..
 
 OBJDIR=${OBJDIR:-build/native/gcc}
 NUM_PROCESSES=${NUM_PROCESSES:-12}
+PAGE_SZ=gigantic
+PAGE_CNT=$(( 6 * $NUM_PROCESSES ))
 
 if [ "$LOG_PATH" == "" ]; then
   LOG_PATH="`mktemp -d`"
@@ -30,27 +35,39 @@ else
   cd dump/test-vectors
 fi
 
-git fetch -q --depth=1 origin $GIT_REF
-git checkout -q $GIT_REF
+if ! git checkout -q $GIT_REF; then
+  git fetch -q --depth=1 origin $GIT_REF
+  git checkout -q FETCH_HEAD
+fi
 cd ../..
 
-LOG=$LOG_PATH/test_exec_block
-find dump/test-vectors/block/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+WKSP=run-test-vectors
+# If workspace already exists, reset it (and hope that it has the correct size)
+if ./$OBJDIR/bin/fd_wksp_ctl query $WKSP --log-path '' >/dev/null 2>/dev/null; then
+  ./$OBJDIR/bin/fd_wksp_ctl reset $WKSP --log-path ''
+else
+  ./$OBJDIR/bin/fd_wksp_ctl new run-test-vectors $PAGE_CNT $PAGE_SZ 0 0644 --log-path ''
+fi
+
+SOL_COMPAT=( "$OBJDIR/unit-test/test_sol_compat" "--wksp" "$WKSP" --tile-cpus "f,0-$(( $NUM_PROCESSES - 1 ))" )
+
+export FD_LOG_PATH=$LOG_PATH/test_exec_block
+${SOL_COMPAT[@]} dump/test-vectors/block/fixtures
 
-LOG=$LOG_PATH/test_exec_syscall
-find dump/test-vectors/syscall/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_syscall
+${SOL_COMPAT[@]} dump/test-vectors/syscall/fixtures
 
-LOG=$LOG_PATH/test_exec_interp
-find dump/test-vectors/vm_interp/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_interp
+${SOL_COMPAT[@]} dump/test-vectors/vm_interp/fixtures
 
-LOG=$LOG_PATH/test_exec_txn
-find dump/test-vectors/txn/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_txn
+${SOL_COMPAT[@]} dump/test-vectors/txn/fixtures
 
 zstd -df dump/test-vectors/elf_loader/fixtures/*.zst
-LOG=$LOG_PATH/test_elf_loader
-find dump/test-vectors/elf_loader/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_elf_loader
+${SOL_COMPAT[@]} dump/test-vectors/elf_loader/fixtures
 
-LOG=$LOG_PATH/test_exec_instr
-find dump/test-vectors/instr/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_instr
+${SOL_COMPAT[@]} dump/test-vectors/instr/fixtures
 
 echo Test vectors success

src/flamenco/capture/fd_solcap_writer.h

@@ -4,7 +4,6 @@
 #include "fd_solcap_proto.h"
 #include "fd_solcap.pb.h"
 #include "../types/fd_types.h"
-#include "../runtime/fd_runtime_public.h"
 
 /* fd_solcap_writer_t is an opaque handle to a capture writer object.
    Currently, it implements writing SOLCAP_V1_BANK files.  See below

src/flamenco/fd_flamenco_base.h

@@ -75,6 +75,9 @@ fd_base58_enc_64_fmt( char *        out,
 struct fd_bank;
 typedef struct fd_bank fd_bank_t;
 
+struct fd_banks;
+typedef struct fd_banks fd_banks_t;
+
 struct fd_exec_slot_ctx;
 typedef struct fd_exec_slot_ctx fd_exec_slot_ctx_t;
 

src/flamenco/runtime/context/fd_capture_ctx.h

@@ -2,6 +2,7 @@
 #define HEADER_fd_src_flamenco_runtime_context_fd_capture_ctx_h
 
 #include "../../capture/fd_solcap_writer.h"
+#include "../fd_runtime_public.h" /* FD_RUNTIME_PUBLIC_ACCOUNT_UPDATE_MSG_FOOTPRINT */
 
 /* Maximum number of accounts that can be updated in a single transaction */
 #define FD_CAPTURE_CTX_MAX_ACCOUNT_UPDATES      (128UL)

src/flamenco/runtime/fd_executor.h

@@ -10,8 +10,6 @@
 #include "../../ballet/poh/fd_poh.h"
 #include "../types/fd_types_yaml.h"
 #include "../log_collector/fd_log_collector.h"
-#include "tests/harness/generated/invoke.pb.h"
-#include "tests/harness/generated/txn.pb.h"
 #include "../features/fd_features.h"
 #include "fd_runtime.h"
 

src/flamenco/runtime/test_txn_rw_conflicts.c

@@ -1,5 +1,3 @@
-#include "../../util/fd_util_base.h"
-#include "../fd_flamenco_base.h"
 #include "fd_acc_mgr.h"
 #include "fd_runtime.h"
 #include "fd_runtime_err.h"

src/flamenco/runtime/tests/.gitignore

@@ -0,0 +1 @@
+nanopb_venv/

src/flamenco/runtime/tests/Local.mk

@@ -1,18 +1,23 @@
-$(call add-hdrs,harness/generated/context.pb.h,harness/generated/elf.pb.h,harness/generated/invoke.pb.h,harness/generated/txn.pb.h,harness/generated/block.pb.h,harness/generated/vm.pb.h,harness/generated/type.pb.h,harness/generated/shred.pb.h harness/generated/metadata.pb.h harness/generated/pack.pb.h)
-$(call add-objs,harness/generated/context.pb harness/generated/elf.pb harness/generated/invoke.pb harness/generated/txn.pb harness/generated/block.pb harness/generated/vm.pb harness/generated/type.pb harness/generated/shred.pb harness/generated/metadata.pb harness/generated/pack.pb,fd_flamenco)
-
 ifdef FD_HAS_INT128
 ifdef FD_HAS_SECP256K1
-$(call add-hdrs,harness/fd_elf_harness.h harness/fd_instr_harness.h harness/fd_txn_harness.h harness/fd_block_harness.h harness/fd_harness_common.h harness/fd_vm_harness.h harness/fd_pack_harness.h harness/fd_types_harness.h)
-$(call add-objs,harness/fd_elf_harness harness/fd_instr_harness harness/fd_txn_harness harness/fd_block_harness harness/fd_harness_common harness/fd_vm_harness harness/fd_pack_harness harness/fd_types_harness,fd_flamenco_test)
-$(call add-objs,harness/fd_exec_sol_compat,fd_flamenco_test)
+
+$(call add-hdrs,fd_solfuzz.h)
+$(call add-objs,fd_solfuzz fd_solfuzz_exec,fd_flamenco_test)
+
+$(call add-hdrs,fd_instr_harness.h fd_txn_harness.h)
+$(call add-objs,fd_elf_harness fd_instr_harness fd_txn_harness fd_block_harness fd_harness_common fd_vm_harness fd_types_harness,fd_flamenco_test)
+$(call add-objs,fd_sol_compat,fd_flamenco_test)
+
+$(call add-hdrs,generated/context.pb.h,generated/elf.pb.h,generated/invoke.pb.h,generated/txn.pb.h,generated/block.pb.h,generated/vm.pb.h,generated/type.pb.h,generated/shred.pb.h generated/metadata.pb.h)
+$(call add-objs,generated/context.pb generated/elf.pb generated/invoke.pb generated/txn.pb generated/block.pb generated/vm.pb generated/type.pb generated/shred.pb generated/metadata.pb,fd_flamenco)
 
 SOL_COMPAT_FLAGS:=-Wl,--undefined=fd_types_vt_by_name
-$(call make-unit-test,test_exec_sol_compat,test_exec_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS))
-$(call make-shared,libfd_exec_sol_compat.so,harness/fd_exec_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS) $(SOL_COMPAT_FLAGS))
+$(call make-unit-test,test_sol_compat,test_sol_compat,fd_flamenco_test fd_flamenco fd_tango fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS))
+$(call make-shared,libfd_exec_sol_compat.so,fd_sol_compat,fd_flamenco_test fd_flamenco fd_funk fd_ballet fd_util fd_disco,$(SECP256K1_LIBS) $(SOL_COMPAT_FLAGS))
+
+run-runtime-backtest: $(OBJDIR)/bin/fd_ledger
+	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_backtest_ci.sh
 
 endif
 endif
 
-run-runtime-backtest: $(OBJDIR)/bin/fd_ledger
-	OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_backtest_ci.sh

src/flamenco/runtime/tests/README.md

@@ -0,0 +1,46 @@
+# solfuzz APIs
+
+solfuzz wraps the Solana runtime (SVM) in a generic Protobuf format.
+This allows users to execute inputs against different SVM
+implementations such as Agave, Firedancer, Mithril, or Sig.
+
+This directory provides the Firedancer SVM backend for solfuzz.
+
+## Internal design
+
+This integration is layered as follows top to bottom:
+
+- `sol_compat` (solfuzz public ABI)
+- `fd_solfuzz` (internal APIs for solfuzz)
+- `fd_runtime` (internal APIs for the Firedancer SVM)
+
+i.e., if a user executes a Solana transaction via `sol_compat`, it is
+passed down to `fd_solfuzz`, which in turn executes the transaction in
+`fd_runtime`.
+
+### sol_compat layer
+
+`sol_compat` is a C API defined by `solfuzz`.
+It is stable-ABI (no breaking changes to symbol names, struct layouts,
+and function signatures).
+
+The Firedancer build outputs a `libfd_exec_sol_compat.so` shared library
+containing an implementation of the `sol_compat` C API.  External users
+like `solfuzz` or `solana-conformance` use this API.
+
+See [fd_sol_compat.h](./fd_sol_compat.h).
+
+### fd_solfuzz layer
+
+Like `sol_compat`, `fd_solfuzz` uses Protobuf as its input and output
+formats.  `fd_solfuzz` is not a stable API but supports a number of
+advanced features useful for internal use.
+
+Mainly used by command-line tooling and tests in the Firedancer repo
+(e.g. the `fd_exec_sol_compat` executable).
+
+See [fd_solfuzz.h](./fd_solfuzz.h).
+
+### fd_runtime layer
+
+The actual Firedancer SVM.  See `src/flamenco/runtime`.

src/flamenco/runtime/tests/harness/fd_block_harness.c → src/flamenco/runtime/tests/fd_block_harness.c

@@ -1,5 +1,20 @@
-#include "fd_block_harness.h"
-#include "../../fd_cost_tracker.h"
+#include "fd_solfuzz_private.h"
+#include "../fd_cost_tracker.h"
+#include "fd_txn_harness.h"
+#include "../fd_runtime.h"
+#include "../fd_system_ids.h"
+#include "../fd_txn_account.h"
+#include "../context/fd_exec_slot_ctx.h"
+#include "../info/fd_runtime_block_info.h"
+#include "../program/fd_stake_program.h"
+#include "../program/fd_vote_program.h"
+#include "../sysvar/fd_sysvar_epoch_schedule.h"
+#include "../sysvar/fd_sysvar_rent.h"
+#include "../sysvar/fd_sysvar_recent_hashes.h"
+#include "../../rewards/fd_rewards.h"
+#include "../../stakes/fd_stakes.h"
+#include "../../types/fd_types.h"
+#include "generated/block.pb.h"
 
 /* Stripped down version of `fd_refresh_vote_accounts()` that simply refreshes the stake delegation amount
    for each of the vote accounts using the stake delegations cache. */
@@ -180,16 +195,14 @@ fd_runtime_fuzz_block_update_prev_epoch_votes_cache( fd_vote_accounts_pair_globa
 }
 
 static void
-fd_runtime_fuzz_block_ctx_destroy( fd_runtime_fuzz_runner_t * runner,
-                                   fd_wksp_t *                wksp ) {
+fd_runtime_fuzz_block_ctx_destroy( fd_solfuzz_runner_t * runner ) {
   fd_funk_txn_cancel_all( runner->funk, 1 );
-  fd_wksp_detach( wksp );
 }
 
 /* Sets up block execution context from an input test case to execute against the runtime.
    Returns block_info on success and NULL on failure. */
 static fd_runtime_block_info_t *
-fd_runtime_fuzz_block_ctx_create( fd_runtime_fuzz_runner_t *           runner,
+fd_runtime_fuzz_block_ctx_create( fd_solfuzz_runner_t *                runner,
                                   fd_exec_slot_ctx_t *                 slot_ctx,
                                   fd_exec_test_block_context_t const * test_ctx ) {
   fd_funk_t * funk = runner->funk;
@@ -489,7 +502,7 @@ fd_runtime_fuzz_block_ctx_create( fd_runtime_fuzz_runner_t *           runner,
 /* Takes in a block_info created from `fd_runtime_fuzz_block_ctx_create()`
    and executes it against the runtime. Returns the execution result. */
 static int
-fd_runtime_fuzz_block_ctx_exec( fd_runtime_fuzz_runner_t * runner,
+fd_runtime_fuzz_block_ctx_exec( fd_solfuzz_runner_t *      runner,
                                 fd_exec_slot_ctx_t *       slot_ctx,
                                 fd_runtime_block_info_t *  block_info ) {
   int res = 0;
@@ -571,24 +584,23 @@ fd_runtime_fuzz_block_ctx_exec( fd_runtime_fuzz_runner_t * runner,
 }
 
 ulong
-fd_runtime_fuzz_block_run( fd_runtime_fuzz_runner_t * runner,
-                           void const *               input_,
-                           void **                    output_,
-                           void *                     output_buf,
-                           ulong                      output_bufsz ) {
+fd_solfuzz_block_run( fd_solfuzz_runner_t * runner,
+                      void const *          input_,
+                      void **               output_,
+                      void *                output_buf,
+                      ulong                 output_bufsz ) {
   fd_exec_test_block_context_t const * input  = fd_type_pun_const( input_ );
   fd_exec_test_block_effects_t **      output = fd_type_pun( output_ );
 
   FD_SPAD_FRAME_BEGIN( runner->spad ) {
     /* Initialize memory */
-    fd_wksp_t *           wksp          = fd_wksp_attach( "wksp" );
     uchar *               slot_ctx_mem  = fd_spad_alloc( runner->spad, FD_EXEC_SLOT_CTX_ALIGN,  FD_EXEC_SLOT_CTX_FOOTPRINT );
     fd_exec_slot_ctx_t *  slot_ctx      = fd_exec_slot_ctx_join ( fd_exec_slot_ctx_new ( slot_ctx_mem ) );
 
     /* Set up the block execution context */
     fd_runtime_block_info_t * block_info = fd_runtime_fuzz_block_ctx_create( runner, slot_ctx, input );
     if( block_info==NULL ) {
-      fd_runtime_fuzz_block_ctx_destroy( runner, wksp );
+      fd_runtime_fuzz_block_ctx_destroy( runner );
       return 0;
     }
 
@@ -618,7 +630,7 @@ fd_runtime_fuzz_block_run( fd_runtime_fuzz_runner_t * runner,
     fd_memcpy( effects->bank_hash, bank_hash.hash, sizeof(fd_hash_t) );
 
     ulong actual_end = FD_SCRATCH_ALLOC_FINI( l, 1UL );
-    fd_runtime_fuzz_block_ctx_destroy( runner, wksp );
+    fd_runtime_fuzz_block_ctx_destroy( runner );
 
     *output = effects;
     return actual_end - (ulong)output_buf;

src/flamenco/runtime/tests/fd_dump_pb.c

@@ -1,17 +1,15 @@
 #include "fd_dump_pb.h"
-#include "harness/generated/block.pb.h"
-#include "harness/generated/invoke.pb.h"
-#include "harness/generated/txn.pb.h"
-#include "harness/generated/vm.pb.h"
+#include "generated/block.pb.h"
+#include "generated/invoke.pb.h"
+#include "generated/txn.pb.h"
+#include "generated/vm.pb.h"
 #include "../fd_system_ids.h"
 #include "../fd_runtime.h"
 #include "../program/fd_address_lookup_table_program.h"
-#include "../../../ballet/lthash/fd_lthash.h"
 #include "../../../ballet/nanopb/pb_encode.h"
 #include "../program/fd_program_cache.h"
 
 
-#include <errno.h>
 #include <stdio.h> /* fopen */
 #include <sys/mman.h> /* mmap */
 #include <unistd.h> /* ftruncate */

src/flamenco/runtime/tests/fd_dump_pb.h

@@ -55,8 +55,8 @@
 #include "../info/fd_instr_info.h"
 #include "../info/fd_runtime_block_info.h"
 #include "../../vm/fd_vm.h"
-#include "harness/generated/block.pb.h"
-#include "harness/generated/elf.pb.h"
+#include "generated/block.pb.h"
+#include "generated/elf.pb.h"
 
 FD_PROTOTYPES_BEGIN
 

src/flamenco/runtime/tests/harness/fd_elf_harness.c → src/flamenco/runtime/tests/fd_elf_harness.c

@@ -1,11 +1,14 @@
-#include "fd_elf_harness.h"
+#include "fd_solfuzz.h"
+#include "generated/elf.pb.h"
+#include "../../../ballet/sbpf/fd_sbpf_loader.h"
+#include "../../vm/fd_vm_base.h"
 
 ulong
-fd_runtime_fuzz_sbpf_load_run( fd_runtime_fuzz_runner_t * runner,
-                               void const *               input_,
-                               void **                    output_,
-                               void *                     output_buf,
-                               ulong                      output_bufsz ) {
+fd_solfuzz_elf_loader_run( fd_solfuzz_runner_t * runner,
+                           void const *          input_,
+                           void **               output_,
+                           void *                output_buf,
+                           ulong                 output_bufsz ) {
   fd_exec_test_elf_loader_ctx_t const * input  = fd_type_pun_const( input_ );
   fd_exec_test_elf_loader_effects_t **  output = fd_type_pun( output_ );