Merge remote-tracking branch 'origin/main' into asiegel/rpc-resurrect

Author: asiegel-jt

book/api/cli.md

@@ -152,6 +152,20 @@ $ fdctl shred-version
 9065
 ```
 
+## `metrics`
+Prints the current validator metrics to stdout.  Metrics can typically
+be accessed via. HTTP when the `metric` tile is enabled,  but the
+command can be used even if the metrics server is not enabled, or the
+validator has crashed.
+
+```sh [bash]
+$ fdctl metrics --config ~/config.toml
+# HELP tile_pid The process ID of the tile.
+# TYPE tile_pid gauge
+tile_pid{kind="netlnk",kind_id="0"} 627750
+tile_pid{kind="net",kind_id="0"} 627759
+```
+
 ## `set-identity`
 Changes the identity key of a running validator. The `<keypair>`
 argument is required and must be the path to an Agave style

contrib/test/run_test_vectors.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# FIXME This whole file should just really be a firedancer-dev
+#       invocation with parallelism natively implemented in C.
+
 set -ex
 
 DIR="$( dirname -- "${BASH_SOURCE[0]}"; )";   # Get the directory name
@@ -8,6 +11,8 @@ cd $DIR/../..
 
 OBJDIR=${OBJDIR:-build/native/gcc}
 NUM_PROCESSES=${NUM_PROCESSES:-12}
+PAGE_SZ=gigantic
+PAGE_CNT=$(( 6 * $NUM_PROCESSES ))
 
 if [ "$LOG_PATH" == "" ]; then
   LOG_PATH="`mktemp -d`"
@@ -30,27 +35,39 @@ else
   cd dump/test-vectors
 fi
 
-git fetch -q --depth=1 origin $GIT_REF
-git checkout -q $GIT_REF
+if ! git checkout -q $GIT_REF; then
+  git fetch -q --depth=1 origin $GIT_REF
+  git checkout -q FETCH_HEAD
+fi
 cd ../..
 
-LOG=$LOG_PATH/test_exec_block
-find dump/test-vectors/block/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+WKSP=run-test-vectors
+# If workspace already exists, reset it (and hope that it has the correct size)
+if ./$OBJDIR/bin/fd_wksp_ctl query $WKSP --log-path '' >/dev/null 2>/dev/null; then
+  ./$OBJDIR/bin/fd_wksp_ctl reset $WKSP --log-path ''
+else
+  ./$OBJDIR/bin/fd_wksp_ctl new run-test-vectors $PAGE_CNT $PAGE_SZ 0 0644 --log-path ''
+fi
+
+SOL_COMPAT=( "$OBJDIR/unit-test/test_sol_compat" "--wksp" "$WKSP" --tile-cpus "f,0-$(( $NUM_PROCESSES - 1 ))" )
+
+export FD_LOG_PATH=$LOG_PATH/test_exec_block
+${SOL_COMPAT[@]} dump/test-vectors/block/fixtures
 
-LOG=$LOG_PATH/test_exec_syscall
-find dump/test-vectors/syscall/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_syscall
+${SOL_COMPAT[@]} dump/test-vectors/syscall/fixtures
 
-LOG=$LOG_PATH/test_exec_interp
-find dump/test-vectors/vm_interp/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_interp
+${SOL_COMPAT[@]} dump/test-vectors/vm_interp/fixtures
 
-LOG=$LOG_PATH/test_exec_txn
-find dump/test-vectors/txn/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_txn
+${SOL_COMPAT[@]} dump/test-vectors/txn/fixtures
 
 zstd -df dump/test-vectors/elf_loader/fixtures/*.zst
-LOG=$LOG_PATH/test_elf_loader
-find dump/test-vectors/elf_loader/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_elf_loader
+${SOL_COMPAT[@]} dump/test-vectors/elf_loader/fixtures
 
-LOG=$LOG_PATH/test_exec_instr
-find dump/test-vectors/instr/fixtures/* -type f -name '*.fix' | xargs -P $NUM_PROCESSES -n 1000 ./$OBJDIR/unit-test/test_exec_sol_compat --log-path $LOG --wksp-page-sz 1073741824
+export FD_LOG_PATH=$LOG_PATH/test_exec_instr
+${SOL_COMPAT[@]} dump/test-vectors/instr/fixtures
 
 echo Test vectors success

contrib/test/test-vectors-commit-sha.txt

@@ -1 +1 @@
-36b61931456ceaaa8f755fe070e5d1c70a835a2d
+ac1cdc5030baa26590b2d7ef99845c2e70c49d51

src/app/fdctl/main.c

@@ -96,6 +96,7 @@ extern action_t fd_action_set_identity;
 extern action_t fd_action_get_identity;
 extern action_t fd_action_help;
 extern action_t fd_action_version;
+extern action_t fd_action_metrics;
 
 action_t * ACTIONS[] = {
   &fd_action_run,
@@ -111,6 +112,7 @@ action_t * ACTIONS[] = {
   &fd_action_get_identity,
   &fd_action_help,
   &fd_action_version,
+  &fd_action_metrics,
   NULL,
 };
 

src/app/fddev/main.h

@@ -125,6 +125,7 @@ extern action_t fd_action_dev1;
 extern action_t fd_action_dump;
 extern action_t fd_action_flame;
 extern action_t fd_action_help;
+extern action_t fd_action_metrics;
 extern action_t fd_action_load;
 extern action_t fd_action_pktgen;
 extern action_t fd_action_quic_trace;
@@ -145,6 +146,7 @@ action_t * ACTIONS[] = {
   &fd_action_set_identity,
   &fd_action_get_identity,
   &fd_action_help,
+  &fd_action_metrics,
   &fd_action_version,
   &fd_action_bench,
   &fd_action_bundle_client,

src/app/firedancer-dev/main.c

@@ -170,6 +170,7 @@ extern action_t fd_action_dev;
 extern action_t fd_action_dump;
 extern action_t fd_action_flame;
 extern action_t fd_action_help;
+extern action_t fd_action_metrics;
 extern action_t fd_action_load;
 extern action_t fd_action_pktgen;
 extern action_t fd_action_quic_trace;
@@ -195,6 +196,7 @@ action_t * ACTIONS[] = {
   &fd_action_netconf,
   &fd_action_set_identity,
   &fd_action_help,
+  &fd_action_metrics,
   &fd_action_version,
   &fd_action_bench,
   &fd_action_bundle_client,

src/app/firedancer/main.c

@@ -125,6 +125,7 @@ extern action_t fd_action_mem;
 extern action_t fd_action_netconf;
 extern action_t fd_action_set_identity;
 extern action_t fd_action_help;
+extern action_t fd_action_metrics;
 extern action_t fd_action_version;
 extern action_t fd_action_shred_version;
 
@@ -139,6 +140,7 @@ action_t * ACTIONS[] = {
   &fd_action_netconf,
   &fd_action_set_identity,
   &fd_action_help,
+  &fd_action_metrics,
   &fd_action_version,
   &fd_action_shred_version,
   NULL,

src/app/shared/Local.mk

@@ -15,6 +15,7 @@ $(call add-objs,boot/fd_boot,fdctl_shared)
 $(call add-objs,commands/help,fdctl_shared)
 $(call add-objs,commands/keys,fdctl_shared)
 $(call add-objs,commands/mem,fdctl_shared)
+$(call add-objs,commands/metrics,fdctl_shared)
 $(call add-objs,commands/netconf,fdctl_shared)
 $(call add-objs,commands/ready,fdctl_shared)
 $(call add-objs,commands/set_identity,fdctl_shared)

src/app/shared/commands/metrics.c

@@ -0,0 +1,49 @@
+#include "../fd_config.h"
+#include "../fd_action.h"
+
+#include "../../../disco/metrics/fd_prometheus.h"
+#include "../../../waltz/http/fd_http_server_private.h"
+
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+
+extern action_t * ACTIONS[];
+
+void
+metrics_cmd_fn( args_t *   args   FD_PARAM_UNUSED,
+                config_t * config ) {
+  fd_http_server_params_t params = {
+    .max_connection_cnt    = 0UL,
+    .max_ws_connection_cnt = 0UL,
+    .max_request_len       = 0UL,
+    .max_ws_recv_frame_len = 0UL,
+    .max_ws_send_frame_cnt = 0UL,
+    .outgoing_buffer_sz    = (1UL<<28UL), /* 256MiB */
+  };
+
+  fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY );
+  fd_topo_fill( &config->topo );
+
+  void * mem = aligned_alloc( fd_http_server_align(), fd_http_server_footprint( params ) );
+  FD_TEST( mem );
+  fd_http_server_t * http = fd_http_server_new( mem, params, (fd_http_server_callbacks_t){0}, NULL );
+  fd_prometheus_render_all( &config->topo, http );
+
+  ulong bytes_written = 0UL;
+  while( bytes_written<http->oring_sz ) {
+    long written = write( STDOUT_FILENO, http->oring+bytes_written, http->oring_sz-bytes_written );
+    if( FD_UNLIKELY( -1==written ) ) FD_LOG_ERR(( "write() failed: %d (%s)", errno, fd_io_strerror( errno ) ));
+    bytes_written += http->oring_sz;
+  }
+}
+
+action_t fd_action_metrics = {
+  .name          = "metrics",
+  .args          = NULL,
+  .fn            = metrics_cmd_fn,
+  .perm          = NULL,
+  .description   = "Print the current validator Prometheus metrics to STDOUT",
+  .is_immediate  = 0,
+  .is_diagnostic = 1,
+};

src/app/shared/commands/metrics.h

@@ -0,0 +1,14 @@
+#ifndef HEADER_fd_src_app_shared_commands_metrics_h
+#define HEADER_fd_src_app_shared_commands_metrics_h
+
+#include "../fd_config.h"
+
+FD_PROTOTYPES_BEGIN
+
+void metrics_cmd_fn( args_t * args, config_t * config );
+
+FD_PROTOTYPES_END
+
+extern action_t fd_action_metrics;
+
+#endif /* HEADER_fd_src_app_shared_commands_metrics_h */