merge

Author: ibhatt-jumptrading

book/api/metrics-generated.md

@@ -113,10 +113,13 @@
 | <span class="metrics-name">quic_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">abort</span>"} | gauge | The number of QUIC connections in each state. (connection terminating due to error) |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">close_&#8203;pending</span>"} | gauge | The number of QUIC connections in each state. (connection is closing) |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">dead</span>"} | gauge | The number of QUIC connections in each state. (connection about to be freed) |
+| <span class="metrics-name">quic_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">timeout</span>"} | gauge | The number of QUIC connections in each state. (connection timed out) |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;created</span> | counter | The total number of connections that have been created. |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;closed</span> | counter | Number of connections gracefully closed. |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;aborted</span> | counter | Number of connections aborted. |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;timed_&#8203;out</span> | counter | Number of connections timed out. |
+| <span class="metrics-name">quic_&#8203;connections_&#8203;timeout_&#8203;revived</span> | counter | Number of connections revived after timing out. |
+| <span class="metrics-name">quic_&#8203;connections_&#8203;timeout_&#8203;freed</span> | counter | Number of connections freed after timing out. |
 | <span class="metrics-name">quic_&#8203;connections_&#8203;retried</span> | counter | Number of connections established with retry. |
 | <span class="metrics-name">quic_&#8203;connection_&#8203;error_&#8203;no_&#8203;slots</span> | counter | Number of connections that failed to create due to lack of slots. |
 | <span class="metrics-name">quic_&#8203;connection_&#8203;error_&#8203;retry_&#8203;fail</span> | counter | Number of connections that failed during retry (e.g. invalid token). |
@@ -782,10 +785,13 @@
 | <span class="metrics-name">send_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">abort</span>"} | gauge | Number of QUIC connections in each state (connection terminating due to error) |
 | <span class="metrics-name">send_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">close_&#8203;pending</span>"} | gauge | Number of QUIC connections in each state (connection is closing) |
 | <span class="metrics-name">send_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">dead</span>"} | gauge | Number of QUIC connections in each state (connection about to be freed) |
+| <span class="metrics-name">send_&#8203;connections_&#8203;state</span><br/>{quic_&#8203;conn_&#8203;state="<span class="metrics-enum">timeout</span>"} | gauge | Number of QUIC connections in each state (connection timed out) |
 | <span class="metrics-name">send_&#8203;connections_&#8203;created</span> | counter | Total count of QUIC connections created |
 | <span class="metrics-name">send_&#8203;connections_&#8203;closed</span> | counter | Total count of QUIC connections closed |
 | <span class="metrics-name">send_&#8203;connections_&#8203;aborted</span> | counter | Total count of QUIC connections aborted |
 | <span class="metrics-name">send_&#8203;connections_&#8203;timed_&#8203;out</span> | counter | Total count of QUIC connections timed out |
+| <span class="metrics-name">send_&#8203;connections_&#8203;timeout_&#8203;revived</span> | counter | Total count of QUIC connections revived after timing out |
+| <span class="metrics-name">send_&#8203;connections_&#8203;timeout_&#8203;freed</span> | counter | Total count of QUIC connections freed after timing out |
 | <span class="metrics-name">send_&#8203;connections_&#8203;retried</span> | counter | Total count of QUIC connections retried |
 | <span class="metrics-name">send_&#8203;connection_&#8203;error_&#8203;no_&#8203;slots</span> | counter | Total count of connection errors due to no slots |
 | <span class="metrics-name">send_&#8203;connection_&#8203;error_&#8203;retry_&#8203;fail</span> | counter | Total count of connection retry failures |

contrib/codeql/dev/ImplicitVoidPromotion.ql

@@ -0,0 +1,46 @@
+/**
+ * @name Implicit Void Ptr to Typed Ptr promotion
+ * @description Implicit conversion from void* to a typed pointer
+ * @kind problem
+ * @problem.severity recommendation
+ * @precision high
+ * @id firedancer-io/implicit-void-promotion
+ */
+
+import cpp
+
+class VoidPointer extends PointerType {
+  VoidPointer() { this.getBaseType().getUnspecifiedType() instanceof VoidType }
+}
+
+predicate implicitVoidPromotion(Type lvalue, Type rvalue) {
+  not lvalue.getUnderlyingType() instanceof VoidPointer and
+  rvalue.getUnderlyingType() instanceof VoidPointer
+}
+
+predicate allowedSourceFile(File file) { file.getBaseName() != "fd_types.c" }
+
+class BroadAssign extends Locatable {
+  BroadAssign() { this instanceof Variable or this instanceof AssignExpr }
+
+  Expr getRExpr() {
+    result = this.(Variable).getInitializer().getExpr() or
+    result = this.(AssignExpr).getRValue()
+  }
+
+  Type getLType() {
+    result = this.(Variable).getType() or
+    result = this.(AssignExpr).getLValue().getType()
+  }
+
+  Type getRType() { result = this.getRExpr().getType() }
+}
+
+from BroadAssign assign
+where
+  implicitVoidPromotion(assign.getLType(), assign.getRType()) and
+  allowedSourceFile(assign.getLocation().getFile()) and
+  not assign.getRExpr().isInMacroExpansion() and
+  not assign.getRExpr().hasExplicitConversion() and
+  not assign.getRExpr().(FunctionCall).getTarget().getName().matches("fd_type_pun%")
+select assign, "Implicit conversion from void * to " + assign.getLType()

contrib/codeql/nightly/TrivialMemcpy.ql

@@ -6,7 +6,7 @@
                 to weaker typing.
  * @kind problem
  * @id firedancer-io/trivial-memcpy
- * @problem.severity none
+ * @problem.severity recommendation
  * @precision high
  * @tags maintainability
  *       readability

contrib/codeql/nightly/WrongUnused.ql

@@ -4,7 +4,7 @@
  * @id asymmetric-research/used-unused-param
  * @kind problem
  * @precision high
- * @problem.severity none
+ * @problem.severity recommendation
  */
 
 import cpp

contrib/offline-replay/offline_replay_network_parameters.sh

@@ -18,8 +18,8 @@ case $network in
     export BACKTEST_FUNK_PAGES=900
     export INDEX_MAX=1000000000
     export PAGES=250
-    export AGAVE_TAG=v2.2.16
-    export FD_CLUSTER_VERSION=2.2.16
+    export AGAVE_TAG=v2.3.6
+    export FD_CLUSTER_VERSION=2.3.6
     ;;
   "testnet")
     export BUCKET_ENDPOINT="gs://testnet-ledger-us-sv15"
@@ -29,8 +29,8 @@ case $network in
     export BACKTEST_FUNK_PAGES=200
     export INDEX_MAX=200000000
     export PAGES=250
-    export AGAVE_TAG=v2.3.0
-    export FD_CLUSTER_VERSION=2.3.0
+    export AGAVE_TAG=v2.3.6
+    export FD_CLUSTER_VERSION=2.3.6
     ;;
   "devnet")
     export BUCKET_ENDPOINT="gs://solana-devnet-ledger-us-ny5"
@@ -40,8 +40,8 @@ case $network in
     export BACKTEST_FUNK_PAGES=500
     export INDEX_MAX=200000000
     export PAGES=250
-    export AGAVE_TAG=v2.2.16
-    export FD_CLUSTER_VERSION=2.2.16
+    export AGAVE_TAG=v2.3.6
+    export FD_CLUSTER_VERSION=2.3.6
     ;;
   *)
     echo "Unknown network: $network"

contrib/quic/rust_compat/Cargo.toml

@@ -11,7 +11,7 @@ bindgen = "0.71"
 [dependencies]
 env_logger = "0"
 libc = "0.2"
-quiche = { version = "0.22", features = ["qlog"] }
+quiche = { version = "0.24.5", features = ["qlog"] }
 quinn = { version = "0.11", features = ["rustls-aws-lc-rs", "rustls-ring"] }
 rustls = { version = "0.23", features = ["aws_lc_rs", "ring"] }
 rustls-post-quantum = "0.2.1"

contrib/quic/rust_compat/src/quiche.rs

@@ -102,7 +102,7 @@ pub(crate) unsafe fn quiche_to_fdquic() {
         assert!(metrics.net_rx_pkt_cnt < 64);
         assert!(metrics.net_tx_pkt_cnt < metrics.net_rx_pkt_cnt);
         assert!(metrics.net_tx_byte_cnt < metrics.net_rx_byte_cnt);
-        assert!(metrics.conn_active_cnt <= 1);
+        assert!(metrics.conn_state_cnt[3] <= 1);
         assert!(metrics.conn_created_cnt == 1);
         assert!(metrics.conn_closed_cnt <= 1);
         assert!(metrics.conn_aborted_cnt <= 1);

contrib/quic/rust_compat/src/quinn.rs

@@ -151,7 +151,7 @@ pub(crate) unsafe fn quinn_to_fdquic(crypto_provider: CryptoProvider) {
         assert!(metrics.net_rx_pkt_cnt < 64);
         assert!(metrics.net_tx_pkt_cnt <= metrics.net_rx_pkt_cnt);
         assert!(metrics.net_tx_byte_cnt < metrics.net_rx_byte_cnt);
-        assert!(metrics.conn_active_cnt <= 1);
+        assert!(metrics.conn_state_cnt[3] <= 1);
         assert!(metrics.conn_created_cnt == 1);
         assert!(metrics.conn_closed_cnt <= 1);
         assert!(metrics.conn_aborted_cnt <= 1);

src/app/firedancer/topology.c

@@ -1033,6 +1033,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
       tile->exec.dump_instr_to_pb = config->capture.dump_instr_to_pb;
       tile->exec.dump_txn_to_pb = config->capture.dump_txn_to_pb;
       tile->exec.dump_syscall_to_pb = config->capture.dump_syscall_to_pb;
+      tile->exec.dump_elf_to_pb = config->capture.dump_elf_to_pb;
     } else if( FD_UNLIKELY( !strcmp( tile->name, "writer" ) ) ) {
       tile->writer.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
       strncpy( tile->writer.solcap_capture, config->capture.solcap_capture, sizeof(tile->writer.solcap_capture) );

src/app/shared/fd_config.h

@@ -458,6 +458,7 @@ struct fd_config {
     ulong capture_start_slot;
     char  dump_proto_dir[ PATH_MAX ];
     char  solcap_capture[ PATH_MAX ];
+    int   dump_elf_to_pb;
     int   dump_syscall_to_pb;
     int   dump_instr_to_pb;
     int   dump_txn_to_pb;