snapshots: add https (#6354)

* snapshots: add https

* deps: fix multi-threaded OpenSSL usage

Fixes incorrect usage of the OpenSSL 'static' feature which is
intended for statically linking glibc.  The static feature breaks
multi-threaded use of OpenSSL, so it should be disabled.

Note that libssl.a still gets built, so OpenSSL itself is still
statically linked.

---------

Co-authored-by: Richard Patel <[email protected]>

Author: cali-jumptrading

book/api/metrics-generated.md

@@ -900,6 +900,7 @@
 | <span class="metrics-name">snapct_&#8203;predicted_&#8203;slot</span> | gauge | The predicted slot from which replay starts after snapshot loading finishes. Might change if snapshot load is aborted and restarted |
 | <span class="metrics-name">snapct_&#8203;gossip_&#8203;fresh_&#8203;count</span> | gauge | Number of fresh gossip peers seen when collecting gossip peers.  |
 | <span class="metrics-name">snapct_&#8203;gossip_&#8203;total_&#8203;count</span> | gauge | Number of total gossip peers seen when collecting gossip peers.  |
+| <span class="metrics-name">snapct_&#8203;ssl_&#8203;alloc_&#8203;errors</span> | counter | Number of SSL allocation errors encountered.  |
 
 </div>
 
@@ -910,6 +911,7 @@
 | Metric | Type | Description |
 |--------|------|-------------|
 | <span class="metrics-name">snapld_&#8203;state</span> | gauge | State of the tile. 0=IDLE, 1=PROCESSING, 2=FINISHING, 3=ERROR, 4=SHUTDOWN |
+| <span class="metrics-name">snapld_&#8203;ssl_&#8203;alloc_&#8203;errors</span> | counter | Number of SSL allocation errors encountered.  |
 
 </div>
 
@@ -1200,6 +1202,5 @@
 | <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;seen</span> | counter | Number of hard forks we've seen (block ids with multiple candidate bank hashes) |
 | <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;pruned</span> | counter | Number of hard forks (candidate bank hashes) we've pruned |
 | <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;active</span> | gauge | Currently active hard forks |
-| <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;max_&#8203;width</span> | gauge | The max width of hard forks (block id with most candidate bank hashes) we've ever seen |
 
 </div>

deps.sh

@@ -505,10 +505,10 @@ install_openssl () {
 
   echo "[+] Configuring OpenSSL"
   ./config \
-    -static \
     -fPIC \
     --prefix="$PREFIX" \
     --libdir=lib \
+    threads \
     no-engine \
     no-static-engine \
     no-weak-ssl-ciphers \

src/app/firedancer/config/testnet.toml

@@ -10,7 +10,7 @@
     max_account_records = 200_000_000
 [snapshots]
     [snapshots.sources]
-        servers = [ "http://solana-testnet-rpc.jumpisolated.com:8899" ]
+        servers = [ "http://solana-testnet-rpc.jumpisolated.com:8899", "https://solana-testnet-tls.jumpisolated.com" ]
         [snapshots.sources.gossip]
             allow_any = false
             allow_list = []

src/app/firedancer/topology.c

@@ -185,6 +185,9 @@ setup_topo_vinyl_cache( fd_topo_t *    topo,
   return line_obj;
 }
 
+/* Resolves a hostname to a single ip address.  If multiple ip address
+   records are returned by getaddrinfo, only the first IPV4 address is
+   returned via ip_addr. */
 static int
 resolve_address( char const * address,
                  uint       * ip_addr ) {
@@ -209,51 +212,107 @@ resolve_address( char const * address,
   return resolved;
 }
 
+/* Resolves a hostname to multiple ip addresses, specified by
+   ip_addr_cnt.  ip_addrs points to an array of fd_ip4_port_t objects.
+   hints points to an optionally NULL addrinfo hints object.  If hints
+   is NULL, a default hints settings containing the IPV4 address family
+   hint will be used. */
 static int
-resolve_peer( char const *    peer,
-              fd_ip4_port_t * ip4_port ) {
+resolve_addresses( char const *             address,
+                   struct addrinfo const *  hints,
+                   fd_ip4_port_t *          ip_addrs,
+                   ulong                    ip_addr_cnt ) {
+  struct addrinfo default_hints = { .ai_family = AF_INET };
+  if( FD_UNLIKELY( !hints ) ) {
+    hints = &default_hints;
+  }
+
+  struct addrinfo * res;
+  int err = getaddrinfo( address, NULL, hints, &res );
+  if( FD_UNLIKELY( err ) ) {
+    FD_LOG_WARNING(( "cannot resolve address \"%s\": %i-%s", address, err, gai_strerror( err ) ));
+    return 0;
+  }
+
+  int resolved = 0;
+  for( struct addrinfo * cur=res; cur; cur=cur->ai_next ) {
+    if( FD_UNLIKELY( (ulong)resolved>=ip_addr_cnt ) ) break;
+    if( FD_UNLIKELY( cur->ai_addr->sa_family!=AF_INET ) ) continue;
+    struct sockaddr_in const * addr = (struct sockaddr_in const *)cur->ai_addr;
+    ip_addrs[ resolved ].addr = addr->sin_addr.s_addr;
+    resolved++;
+  }
+
+  freeaddrinfo( res );
+  return resolved;
+}
+
+static int
+resolve_peer( char const *            peer,
+              struct addrinfo const * addr_resolve_hints,
+              char const *            config_str,
+              char                    hostname[ static 256UL ],
+              fd_ip4_port_t *         ip4_port,
+              ulong                   ip4_port_cnt,
+              int *                   is_https ) {
 
   /* Split host:port */
+  int          https     = 0;
   char const * host_port = peer;
   if( FD_LIKELY( strncmp( peer, "http://", 7UL )==0 ) ) {
+    if( FD_LIKELY( is_https ) ) *is_https  = 0;
     host_port += 7UL;
   } else if( FD_LIKELY( strncmp( peer, "https://", 8UL )==0 ) ) {
+    if( FD_LIKELY( is_https ) ) *is_https  = 1;
     host_port += 8UL;
+    https      = 1;
   }
 
-  char const * colon = strrchr( host_port, ':' );
-  if( FD_UNLIKELY( !colon ) ) {
-    FD_LOG_ERR(( "invalid [gossip.entrypoints] entry \"%s\": no port number", host_port ));
+  char const * colon    = strrchr( host_port, ':' );
+  char const * host_end = colon;
+  if( FD_LIKELY( FD_UNLIKELY( !colon && !https )  ) ) {
+    FD_LOG_ERR(( "invalid [%s] entry \"%s\": no port number", config_str, host_port ));
+    host_end = colon;
+  } else if( FD_LIKELY( !colon && https ) ) {
+    host_end = host_port + strlen( host_port );
   }
 
-  char fqdn[ 255 ];
-  ulong fqdn_len = (ulong)( colon-host_port );
-  if( FD_UNLIKELY( fqdn_len>254 ) ) {
-    FD_LOG_ERR(( "invalid [gossip.entrypoints] entry \"%s\": hostname too long", host_port ));
+  ulong fqdn_len = (ulong)( host_end-host_port );
+  if( FD_UNLIKELY( fqdn_len>255 ) ) {
+    FD_LOG_ERR(( "invalid [%s] entry \"%s\": hostname too long", config_str, host_port ));
   }
-  fd_memcpy( fqdn, host_port, fqdn_len );
-  fqdn[ fqdn_len ] = '\0';
+  fd_memcpy( hostname, host_port, fqdn_len );
+  hostname[ fqdn_len ] = '\0';
+
+  /* Resolve hostname */
+  int resolved = resolve_addresses( hostname, addr_resolve_hints, ip4_port, ip4_port_cnt );
 
   /* Parse port number */
 
-  char const * port_str = colon+1;
-  char const * endptr   = NULL;
-  ulong port = strtoul( port_str, (char **)&endptr, 10 );
-  if( FD_UNLIKELY( !endptr || !port || port>USHORT_MAX || *endptr!='\0' ) ) {
-    FD_LOG_ERR(( "invalid [gossip.entrypoints] entry \"%s\": invalid port number", host_port ));
+  if( FD_LIKELY( colon ) ) {
+    char const * port_str = host_end+1;
+    char const * endptr   = NULL;
+    ulong port = strtoul( port_str, (char **)&endptr, 10 );
+    if( FD_UNLIKELY( endptr==port_str || !port || port>USHORT_MAX || *endptr!='\0' ) ) {
+      FD_LOG_ERR(( "invalid [%s] entry \"%s\": invalid port number", config_str, host_port ));
+    }
+    for( ulong i=0UL; i<(ulong)resolved; i++ ) ip4_port[ i ].port = fd_ushort_bswap( (ushort)port );
+  } else if( FD_LIKELY( !colon && https ) ) {
+    /* use default https port */
+    for( ulong i=0UL; i<(ulong)resolved; i++ ) ip4_port[ i ].port = fd_ushort_bswap( 443U );
+  } else {
+    FD_LOG_ERR(( "invalid [%s] entry \"%s\": no port number", config_str, host_port ));
   }
-  ip4_port->port = (ushort)fd_ushort_bswap( (ushort)port );
 
-  /* Resolve hostname */
-  int resolved = resolve_address( fqdn, &ip4_port->addr );
   return resolved;
 }
 
 static void
 resolve_gossip_entrypoints( config_t * config ) {
   ulong entrypoint_cnt = config->gossip.entrypoints_cnt;
   for( ulong i=0UL; i<entrypoint_cnt; i++ ) {
-    if( FD_UNLIKELY( 0==resolve_peer( config->gossip.entrypoints[ i ], &config->gossip.resolved_entrypoints[ i ] ) ) ) {
+    char hostname[ 256UL ];
+    if( FD_UNLIKELY( 0==resolve_peer( config->gossip.entrypoints[ i ], NULL, "gossip.entrypoints", hostname, &config->gossip.resolved_entrypoints[ i ], 1, NULL ) ) ) {
       FD_LOG_ERR(( "failed to resolve address of [gossip.entrypoints] entry \"%s\"", config->gossip.entrypoints[ i ] ));
     }
   }
@@ -1080,12 +1139,32 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
         FD_LOG_ERR(( "[snapshots.sources.gossip.block_list[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.gossip.block_list[ i ] ));
       }
     }
+
+    ulong resolved_peers_cnt = 0UL;
     for( ulong i=0UL; i<tile->snapct.sources.servers_cnt; i++ ) {
-      if( FD_UNLIKELY( !resolve_peer( config->firedancer.snapshots.sources.servers[ i ], &tile->snapct.sources.servers[ i ] ) ) ) {
+      fd_ip4_port_t resolved_addrs[ FD_TOPO_MAX_RESOLVED_ADDRS ];
+      struct addrinfo hints = { .ai_family = AF_INET, .ai_socktype = SOCK_STREAM };
+      int num_resolved = resolve_peer( config->firedancer.snapshots.sources.servers[ i ],
+                                       &hints,
+                                       "snapshots.sources.servers",
+                                       tile->snapct.sources.servers[ resolved_peers_cnt ].hostname,
+                                       resolved_addrs,
+                                       FD_TOPO_MAX_RESOLVED_ADDRS,
+                                       &tile->snapct.sources.servers[ resolved_peers_cnt ].is_https );
+      if( FD_UNLIKELY( 0==num_resolved ) ) {
         FD_LOG_ERR(( "[snapshots.sources.servers[%lu] invalid (%s)", i, config->firedancer.snapshots.sources.servers[ i ] ));
+      } else {
+        for( ulong i=0UL; i<(ulong)num_resolved; i++ ) tile->snapct.sources.servers[ resolved_peers_cnt+i ].addr = resolved_addrs[ i ];
+        for( ulong i=1UL; i<(ulong)num_resolved; i++ ) {
+          tile->snapct.sources.servers[ resolved_peers_cnt+i ].is_https = tile->snapct.sources.servers[ resolved_peers_cnt ].is_https;
+          fd_memcpy( tile->snapct.sources.servers[ resolved_peers_cnt+i ].hostname,
+                     tile->snapct.sources.servers[ resolved_peers_cnt ].hostname,
+                     sizeof(tile->snapct.sources.servers[ resolved_peers_cnt ].hostname) );
+        }
+        resolved_peers_cnt += (ulong)num_resolved;
       }
     }
-
+    tile->snapct.sources.servers_cnt = resolved_peers_cnt;
   } else if( FD_UNLIKELY( !strcmp( tile->name, "snapld" ) ) ) {
 
     fd_memcpy( tile->snapld.snapshots_path, config->paths.snapshots, PATH_MAX );

src/disco/bundle/fd_bundle_tile.c

@@ -5,6 +5,7 @@
 #include "../keyguard/fd_keyload.h"
 #include "../plugin/fd_plugin.h"
 #include "../../waltz/http/fd_url.h"
+#include "../../waltz/openssl/fd_openssl_tile.h"
 
 #include <errno.h>
 #include <dirent.h> /* opendir */
@@ -55,6 +56,9 @@ metrics_write( fd_bundle_tile_t * ctx ) {
   FD_MCNT_SET( BUNDLE, ERRORS_PROTOBUF,        ctx->metrics.decode_fail_cnt           );
   FD_MCNT_SET( BUNDLE, ERRORS_TRANSPORT,       ctx->metrics.transport_fail_cnt        );
   FD_MCNT_SET( BUNDLE, ERRORS_NO_FEE_INFO,     ctx->metrics.missing_builder_info_fail_cnt );
+#if FD_HAS_OPENSSL
+  FD_MCNT_SET( BUNDLE, ERRORS_SSL_ALLOC,       fd_ossl_alloc_errors                 );
+#endif
 
   FD_MGAUGE_SET( BUNDLE, RTT_SAMPLE,   (ulong)ctx->rtt->latest_rtt   );
   FD_MGAUGE_SET( BUNDLE, RTT_SMOOTHED, (ulong)ctx->rtt->smoothed_rtt );
@@ -247,65 +251,6 @@ fd_bundle_tile_parse_endpoint( fd_bundle_tile_t *     ctx,
 
 #if FD_HAS_OPENSSL
 
-/* OpenSSL allows us to specify custom memory allocation functions,
-   which we want to point to an fd_alloc_t, but it does not let us use a
-   context object.  Instead we stash it in this thread local, which is
-   OK because the parent workspace exists for the duration of the SSL
-   context, and the process only has one thread.
-
-   Currently fd_alloc doesn't support realloc, so it's implemented on
-   top of malloc and free, and then also it doesn't support getting the
-   size of an allocation from the pointer, which we need for realloc, so
-   we pad each alloc by 8 bytes and stuff the size into the first 8
-   bytes. */
-static FD_TL fd_alloc_t * fd_quic_ssl_mem_function_ctx = NULL;
-
-static void *
-crypto_malloc( ulong        num,
-               char const * file,
-               int          line ) {
-  (void)file; (void)line;
-  void * result = fd_alloc_malloc( fd_quic_ssl_mem_function_ctx, 16UL, num + 8UL );
-  if( FD_UNLIKELY( !result ) ) {
-    FD_MCNT_INC( BUNDLE, ERRORS_SSL_ALLOC, 1UL );
-    return NULL;
-  }
-  *(ulong *)result = num;
-  return (uchar *)result + 8UL;
-}
-
-static void
-crypto_free( void *       addr,
-             char const * file,
-             int          line ) {
-  (void)file;
-  (void)line;
-
-  if( FD_UNLIKELY( !addr ) ) return;
-  fd_alloc_free( fd_quic_ssl_mem_function_ctx, (uchar *)addr - 8UL );
-}
-
-static void *
-crypto_realloc( void *       addr,
-                ulong        num,
-                char const * file,
-                int          line ) {
-  if( FD_UNLIKELY( !addr ) ) return crypto_malloc( num, file, line );
-  if( FD_UNLIKELY( !num ) ) {
-    crypto_free( addr, file, line );
-    return NULL;
-  }
-
-  void * new = fd_alloc_malloc( fd_quic_ssl_mem_function_ctx, 16UL, num + 8UL );
-  if( FD_UNLIKELY( !new ) ) return NULL;
-
-  ulong old_num = *(ulong *)( (uchar *)addr - 8UL );
-  fd_memcpy( (uchar*)new + 8, (uchar*)addr, fd_ulong_min( old_num, num ) );
-  fd_alloc_free( fd_quic_ssl_mem_function_ctx, (uchar *)addr - 8UL );
-  *(ulong *)new = num;
-  return (uchar*)new + 8UL;
-}
-
 static void
 fd_ossl_keylog_callback( SSL const *  ssl,
                          char const * line ) {
@@ -321,58 +266,6 @@ fd_ossl_keylog_callback( SSL const *  ssl,
   }
 }
 
-static void
-fd_bundle_tile_load_certs( SSL_CTX * ssl_ctx ) {
-  X509_STORE * ca_certs = X509_STORE_new();
-  if( FD_UNLIKELY( !ca_certs ) ) {
-    FD_LOG_ERR(( "X509_STORE_new failed" ));
-  }
-
-  static char const default_dir[] = "/etc/ssl/certs/";
-  DIR * dir = opendir( default_dir );
-  if( FD_UNLIKELY( !dir ) ) {
-    FD_LOG_ERR(( "opendir(%s) failed (%i-%s)", default_dir, errno, fd_io_strerror( errno ) ));
-  }
-
-  struct dirent * entry;
-  errno = 0; // clear old value since entry can be NULL when reaching end of directory.
-  while( (entry = readdir( dir )) ) {
-    if( !strcmp( entry->d_name, "." ) || !strcmp( entry->d_name, ".." ) ) continue;
-
-    char cert_path[ PATH_MAX ];
-    char * p = fd_cstr_init( cert_path );
-    p = fd_cstr_append_text( p, default_dir, sizeof(default_dir)-1 );
-    p = fd_cstr_append_cstr_safe( p, entry->d_name, (ulong)(cert_path+sizeof(cert_path)-1) - (ulong)p );
-    fd_cstr_fini( p );
-
-    if( !X509_STORE_load_locations( ca_certs, cert_path, NULL ) ) {
-      /* Not all files in /etc/ssl/certs are valid certs, so ignore errors */
-      continue;
-    }
-    errno = 0;
-  }
-
-  if( FD_UNLIKELY( errno && errno!=ENOENT ) ) {
-    FD_LOG_ERR(( "readdir(%s) failed (%i-%s)", default_dir, errno, fd_io_strerror( errno ) ));
-  }
-
-  STACK_OF(X509) * cert_list = X509_STORE_get1_all_certs( ca_certs );
-  FD_LOG_INFO(( "Loaded %d CA certs from %s into OpenSSL", sk_X509_num( cert_list ), default_dir ));
-  if( fd_log_level_logfile()==0 ) {
-    for( int i=0; i<sk_X509_num( cert_list ); i++ ) {
-      X509 * cert = sk_X509_value( cert_list, i );
-      FD_LOG_DEBUG(( "Loaded CA cert \"%s\"", X509_NAME_oneline( X509_get_subject_name( cert ), NULL, 0 ) ));
-    }
-  }
-  sk_X509_pop_free( cert_list, X509_free );
-
-  SSL_CTX_set_cert_store( ssl_ctx, ca_certs );
-
-  if( FD_UNLIKELY( 0!=closedir( dir ) ) ) {
-    FD_LOG_ERR(( "closedir(%s) failed (%i-%s)", default_dir, errno, fd_io_strerror( errno ) ));
-  }
-}
-
 static void
 fd_bundle_tile_init_openssl( fd_bundle_tile_t * ctx,
                              void *             alloc_mem,
@@ -381,19 +274,8 @@ fd_bundle_tile_init_openssl( fd_bundle_tile_t * ctx,
   if( FD_UNLIKELY( !alloc ) ) {
     FD_LOG_ERR(( "fd_alloc_new failed" ));
   }
-  ctx->ssl_alloc               = alloc;
-  fd_quic_ssl_mem_function_ctx = alloc;
-
-  if( FD_UNLIKELY( !CRYPTO_set_mem_functions( crypto_malloc, crypto_realloc, crypto_free ) ) ) {
-    FD_LOG_ERR(( "CRYPTO_set_mem_functions failed" ));
-  }
-
-  OPENSSL_init_ssl(
-      OPENSSL_INIT_LOAD_SSL_STRINGS |
-      OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
-      OPENSSL_INIT_NO_LOAD_CONFIG,
-      NULL
-  );
+  ctx->ssl_alloc = alloc;
+  fd_ossl_tile_init( alloc );
 
   SSL_CTX * ssl_ctx = SSL_CTX_new( TLS_client_method() );
   if( FD_UNLIKELY( !ssl_ctx ) ) {
@@ -417,8 +299,7 @@ fd_bundle_tile_init_openssl( fd_bundle_tile_t * ctx,
   }
 
   if( tls_cert_verify ) {
-    fd_bundle_tile_load_certs( ssl_ctx );
-    SSL_CTX_set_verify( ssl_ctx, SSL_VERIFY_PEER, NULL );
+    fd_ossl_load_certs( ssl_ctx );
   }
 
   if( FD_LIKELY( ctx->keylog_fd >= 0 ) ) {

src/disco/metrics/generated/fd_metrics_snapct.c

@@ -16,4 +16,5 @@ const fd_metrics_meta_t FD_METRICS_SNAPCT[FD_METRICS_SNAPCT_TOTAL] = {
     DECLARE_METRIC( SNAPCT_PREDICTED_SLOT, GAUGE ),
     DECLARE_METRIC( SNAPCT_GOSSIP_FRESH_COUNT, GAUGE ),
     DECLARE_METRIC( SNAPCT_GOSSIP_TOTAL_COUNT, GAUGE ),
+    DECLARE_METRIC( SNAPCT_SSL_ALLOC_ERRORS, COUNTER ),
 };