Merge branch 'main' into asiegel/rpc-resurrect

Author: asiegel-jt

.github/workflows/builds.yml

@@ -4,6 +4,10 @@ name: Builds
 on:
   workflow_call:
     inputs:
+      cid:
+        type: string
+        description: Unique identifier to allow concurrency
+        default: "0"
       gcc:
         type: string
         description: GCC versions to use
@@ -32,16 +36,20 @@ on:
         type: boolean
         description: Print build matrix and exit
         default: false
-      verbose:
+      check_run:
         type: boolean
-        description: Show error outputs
+        description: Run `make check` instead of full builds
         default: false
       exit_on_err:
         type: boolean
         description: Exit on the first error
         default: false
   workflow_dispatch:
     inputs:
+      cid:
+        type: string
+        description: Unique identifier to allow concurrency
+        default: "0"
       gcc:
         type: string
         description: GCC versions to use (comma-separated | none | all)
@@ -70,20 +78,20 @@ on:
         type: boolean
         description: Print build matrix and exit
         default: false
-      verbose:
+      check_run:
         type: boolean
-        description: Show error outputs
+        description: Run `make check` instead of full builds
         default: false
       exit_on_err:
         type: boolean
         description: Exit on the first error
         default: false
 concurrency:
-  group: builds_${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: builds_${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ inputs.cid }}
   cancel-in-progress: true
 jobs:
   build_gcc:
-    runs-on: ci4
+    runs-on: rocky810
     if: ${{ inputs.gcc != 'none' }}
     steps:
       - uses: actions/checkout@v4
@@ -92,16 +100,22 @@ jobs:
 
       - uses: dtolnay/[email protected]
 
+      - uses: ./.github/actions/deps
+        with:
+          compiler: gcc
+          compiler-version: 12.4.0
+          extras: +dev
+
       - name: Build command line args
         run: |
           ARGS=""
           # dry-run
           if [ "${{ inputs.dry_run }}" == "true" ]; then
             ARGS="$ARGS --dry-run"
           fi
-          # verbose
-          if [ "${{ inputs.verbose }}" == "true" ]; then
-            ARGS="$ARGS --verbose"
+          # check-run
+          if [ "${{ inputs.check_run }}" == "true" ]; then
+            ARGS="$ARGS --check-run --no-deps"
           fi
           # exit-on-err
           if [ "${{ inputs.exit_on_err }}" == "true" ]; then
@@ -130,7 +144,7 @@ jobs:
           contrib/build.sh --no-rust --no-clang ${{ env.BUILD_ARGS }}
 
   build_clang:
-    runs-on: ci16
+    runs-on: rocky94
     if: ${{ inputs.clang != 'none' }}
     steps:
       - uses: actions/checkout@v4
@@ -139,16 +153,22 @@ jobs:
 
       - uses: dtolnay/[email protected]
 
+      - uses: ./.github/actions/deps
+        with:
+          compiler: clang
+          compiler-version: 18.1.6
+          extras: +dev
+
       - name: Build command line args
         run: |
           ARGS=""
           # dry-run
           if [ "${{ inputs.dry_run }}" == "true" ]; then
             ARGS="$ARGS --dry-run"
           fi
-          # verbose
-          if [ "${{ inputs.verbose }}" == "true" ]; then
-            ARGS="$ARGS --verbose"
+          # check-run
+          if [ "${{ inputs.check_run }}" == "true" ]; then
+            ARGS="$ARGS --check-run --no-deps"
           fi
           # exit-on-err
           if [ "${{ inputs.exit_on_err }}" == "true" ]; then

.github/workflows/on_nightly.yml

@@ -16,6 +16,7 @@ jobs:
     with:
         # compiler,machine,target
       gcc_exceptions: |
+        ALL,linux_gcc_riscv,ALL;
         ALL,linux_gcc_power9,ALL;
         ALL,linux_gcc_arm_n1,ALL;
         gcc-8.5.0,linux_gcc_zen2,ALL;
@@ -29,7 +30,5 @@ jobs:
         gcc-11.4.0,linux_gcc_zen5,ALL;
         gcc-12.4.0,linux_gcc_zen5,ALL;
         gcc-13.3.0,linux_gcc_zen5,ALL;
-        gcc-14.1.0,linux_gcc_zen5,ALL;
-        ALL,linux_gcc_riscv,ALL
-      verbose: false
+        gcc-14.1.0,linux_gcc_zen5,ALL
       build_arm: true

.github/workflows/on_pull_request.yml

@@ -15,12 +15,29 @@ jobs:
     if: github.event.pull_request.draft == false
     uses: ./.github/workflows/backtest.yml
     secrets: inherit
-
-  # firedancer:
-  #   if: github.event.pull_request.draft == false
-  #   uses: ./.github/workflows/test_firedancer_localnet.yml
-  #   secrets: inherit
-  # firedancer-shredcap:
-  #   if: github.event.pull_request.draft == false
-  #   uses: ./.github/workflows/test_firedancer_testnet_shredcap.yml
-  #   secrets: inherit
+  build_checks_1:
+    if: github.event.pull_request.draft == false
+    uses: ./.github/workflows/builds.yml
+    with:
+      cid: 1
+      build_arm: false
+      check_run: true
+        # compiler,machine,target
+      gcc_exceptions: |
+        gcc-8.5.0,linux_gcc_zen2,ALL;
+        ALL,linux_gcc_riscv,ALL;
+        ALL,linux_gcc_power9,ALL;
+        ALL,linux_gcc_arm_n1,ALL;
+        ALL,linux_gcc_zen4,ALL;
+        ALL,linux_gcc_zen5,ALL;
+        ALL,linux_gcc_icelake,ALL;
+        ALL,linux_gcc_x86_64,ALL
+  build_checks_2:
+    if: github.event.pull_request.draft == false
+    uses: ./.github/workflows/builds.yml
+    with:
+      cid: 2
+      clang: none
+      machine: linux_gcc_icelake,linux_gcc_x86_64
+      build_arm: false
+      check_run: true

.github/workflows/tests.yml

@@ -18,22 +18,11 @@ jobs:
       fail-fast: true
       matrix:
         test-case:
-          - linux_gcc_noarch64        # least capable target
           - linux_gcc_x86_64
           - linux_gcc_icelake         #  most capable target
-          - linux_clang_x86_64
-          - linux_clang_icelake
           - native
-          - native-no-deps
         # Attach additional params to machine types
         include:
-          - test-case: linux_gcc_noarch64
-            machine: linux_gcc_noarch64
-            label: X64
-            deps-extras: "+dev"
-            targets: "check"
-            compiler: gcc
-            compiler-version: 11.4.0
           - test-case: linux_gcc_x86_64
             machine: linux_gcc_x86_64
             label: X64
@@ -52,24 +41,6 @@ jobs:
             compiler: gcc
             compiler-version: 12.4.0
             run-unit-tests: true
-          - test-case: linux_clang_x86_64
-            machine: linux_clang_x86_64
-            label: X64
-            extras: "rpath handholding"
-            deps-extras: "+dev"
-            targets: "all integration-test fdctl firedancer"
-            compiler: clang
-            compiler-version: 15.0.6
-            run-unit-tests: true
-          - test-case: linux_clang_icelake
-            machine: linux_clang_icelake
-            label: icelake
-            extras: "asan ubsan rpath handholding"
-            deps-extras: "+dev"
-            targets: "all integration-test fdctl firedancer"
-            compiler: clang
-            compiler-version: 15.0.6
-            run-unit-tests: true
           - test-case: native
             machine: native
             label: 512G
@@ -80,13 +51,6 @@ jobs:
             compiler-version: 15.0.6
             run-unit-tests: true
             run-integration-tests: true
-          - test-case: native-no-deps
-            machine: native
-            label: X64
-            extras: no-deps
-            targets: check
-            compiler: clang
-            compiler-version: 15.0.6
     runs-on: ${{ matrix.label }}
     env:
       MACHINE: ${{ matrix.machine }}

agave

@@ -672,8 +672,8 @@ potential underflow.
 |----------------------------|----------|-------------|
 | net_in                     | `number` | Network ingress bytes per second (W) |
 | quic                       | `number` | Active QUIC connections (P) |
-| bundle_rtt_smoothed_millis | `number` | The round-trip time for grpc messages sent to the bundle server.  These are mostly ping messages when the validator is not leader.  An exponential moving average ( avg = 1/8 val + 7/8 avg ) is used to filter the signal |
-| bundle_rx_delay_millis_p90 | `number` | An estimate of the 90th percentile of the one-way delay of a bundle dispatched from the bundle server.  Only samples since the start of the most recent leader rotation for this validator are used to compute the percentile |
+| bundle_rtt_smoothed_millis | `number` | The round-trip time for grpc messages sent to the bundle server.  These are mostly ping-pong messages.  An exponential moving average ( avg = 1/8 val + 7/8 avg ) is used to filter the signal (W) |
+| bundle_rx_delay_millis_p90 | `number` | An estimate of the 90th percentile of the one-way delay of a bundle dispatched from the bundle server (W) |
 | verify                     | `number` | Fraction of transactions that failed sigverify (W) |
 | dedup                      | `number` | Fraction of transactions deduplicated (W) |
 | pack                       | `number` | Fraction of pack buffer filled (P) |
@@ -1400,11 +1400,11 @@ The source tpu for a transaction can be one of the following
 
 | TPU    | Description |
 |--------|-------------|
-| quic   | the primary ingress tpu for user transactions.  Utilizes the quic protocol to recieve packets |
-| udp    | ingress transactions recieved as simple UDP packets |
-| gossip | vote transactions recieved from the gossip network |
-| bundle | bundle transacionts recieved by the bundle tile from a block builder.  Utilizes a grpc connection to recieve packets |
-| send   | vote transactions procuded by this validator recieved from the send tile. These transactions are meant for the active cluster leader |
+| quic   | the primary ingress tpu for user transactions.  Utilizes the quic protocol to receive packets |
+| udp    | ingress transactions received as simple UDP packets |
+| gossip | vote transactions received from the gossip network |
+| bundle | bundle transacionts received by the bundle tile from a block builder.  Utilizes a grpc connection to receive packets |
+| send   | vote transactions procuded by this validator received from the send tile. These transactions are meant for the active cluster leader |
 
 These are the possible error codes that might be included in `txn_error_code` and their meanings.
 
@@ -1449,4 +1449,5 @@ These are the possible error codes that might be included in `txn_error_code` an
 | ProgramExecutionTemporarilyRestricted | 36   | Execution of a program referenced by this transaciton is restricted |
 | UnbalancedTransaction                 | 37   | The total accounts balance before the transaction does not equal the total balance after |
 | ProgramCacheHitMaxLimit               | 38   | The program cache allocated for transaction batch for this transaction hit its load limit |
-| CommitCancelled                       | 39   | This transaction was part of a bundle that failed |
+| CommitCancelled                       | 39   | This transaction was aborted during the commit stage |
+| BundlePeer                            | 40   | This transaction was part of a bundle that failed |

book/api/websocket.md

@@ -37,3 +37,6 @@ include config/extra/with-rocksdb.mk
 endif
 
 FD_ARCH_SUPPORTS_SANDBOX:=1
+
+CPPFLAGS+=-DFD_HAS_ARM=1
+FD_HAS_ARM:=1

config/extra/with-arm.mk

@@ -13,6 +13,7 @@ help() {
   echo "   --no-deps            Do not install deps during any builds"
   echo "   --no-rust            Do not install rust"
   echo "   --dry-run            Print build matrix and exit"
+  echo "   --check-run          Run \`make check\` instead of full builds"
   echo "   --verbose            Show output from failed builds"
   echo "   --exit-on-err        Exit upon hitting the first failed build"
   echo "   --help           -h  Show this message and exit"
@@ -111,6 +112,10 @@ while [[ $# -gt 0 ]]; do
     "--dry-run")
       DRY_RUN=1
       ;;
+    # run `make check` instead of full builds
+    "--check-run")
+      CHECK_RUN=1
+      ;;
     # exit upon hitting the first error
     "--exit-on-err")
       EXIT_ON_ERR=1
@@ -227,6 +232,10 @@ if [[ ${#MACHINES[@]} -eq 0 ]]; then
   done
 fi
 
+if [[ $CHECK_RUN -eq 1 ]]; then
+  TARGETS=( check )
+fi
+
 echo "*************************"
 echo "Starting Build Matrix..."
 echo "*************************"
@@ -347,7 +356,7 @@ if [[ $NO_GCC -ne 1 ]]; then
         inf "Skipping all targets for - $compiler $MACHINE\n"
         continue
       fi
-      if [[ "$MACHINE" != *"clang"* ]]; then
+      if [[ "$MACHINE" == *"gcc"* ]]; then
         # override any targets list with supplied --targets
         BUILD_TARGETS=()
         if [[ ${#TARGETS[@]} -eq 0 ]]; then
@@ -465,7 +474,7 @@ if [[ $NO_CLANG -ne 1 ]]; then
         inf "Skipping all targets for - $compiler $MACHINE\n"
         continue
       fi
-      if [[ "$MACHINE" != *"gcc"* ]]; then
+      if [[ "$MACHINE" == *"clang"* ]]; then
         # override any targets list with supplied --targets
         BUILD_TARGETS=()
         if [[ ${#TARGETS[@]} -eq 0 ]]; then

contrib/build.sh

@@ -4,24 +4,26 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-tonic = { version = "0.12.2", features = ["tls-roots", "tls", "tls-webpki-roots"] }
-prost = "0.13.3"
-prost-types = "0.13.3"
+tonic = { version = "0.14", features = ["tls-webpki-roots"] }
+tonic-prost = "0.14"
+prost = "0.14"
+prost-types = "0.14"
 log = "0.4.22"
-tokio = { version = "1.40.0", features = ["rt-multi-thread"] }
+tokio = "1.47"
 tokio-stream = "0.1"
-futures = "0.3.30"
-chrono = "0.4.38"
-thiserror = "1.0.64"
+futures = "0.3"
+chrono = "0.4"
+thiserror = "1.0"
 bs58 = "0.5.1"
-futures-util = "0.3.31"
-env_logger = "0.11.5"
-base64="0.22.1"
+futures-util = "0.3"
+env_logger = "0.11"
+base64 = "0.22"
+rustyline = "17.0"
 
 [build-dependencies]
-tonic-build = "0.12.2"
-protobuf-src = "2.1.0"
-prost-types = "0.13.3"
+tonic-prost-build = "0.14"
+protobuf-src = "2.1"
+prost-types = "0.14"
 
 [dev-dependencies]
 ed25519-dalek = "2.1.1"

contrib/bundle-test-server/Cargo.toml

@@ -1,5 +1,3 @@
-use tonic_build::configure;
-
 fn main() -> Result<(), std::io::Error> {
     const PROTOC_ENVAR: &str = "PROTOC";
     if std::env::var(PROTOC_ENVAR).is_err() {
@@ -23,7 +21,7 @@ fn main() -> Result<(), std::io::Error> {
         protos.push(proto);
     }
 
-    configure()
+    tonic_prost_build::configure()
         .build_client(false)
         .build_server(true)
         .type_attribute(