diff --git a/pingora-core/src/connectors/mod.rs b/pingora-core/src/connectors/mod.rs index 0e3c727c..3de0b441 100644 --- a/pingora-core/src/connectors/mod.rs +++ b/pingora-core/src/connectors/mod.rs @@ -80,6 +80,16 @@ pub struct ConnectorOptions { pub bind_to_v4: Vec, /// Bind to any of the given source IPv4 addresses pub bind_to_v6: Vec, + /// Optional custom server certificate verifier for the rustls backend. + /// + /// When set, the connector uses `dangerous().with_custom_certificate_verifier()` instead of + /// building a `RootCertStore` and using webpki for server cert validation. This is necessary + /// for certificates with custom critical extensions that webpki rejects (e.g. IEEE 2030.5). + /// + /// The verifier must handle TLS signature verification as well, since webpki is bypassed. + /// `ca_file` is ignored when this is set. + #[cfg(feature = "rustls")] + pub server_cert_verifier: Option>, } impl ConnectorOptions { @@ -120,6 +130,8 @@ impl ConnectorOptions { offload_threadpool, bind_to_v4, bind_to_v6, + #[cfg(feature = "rustls")] + server_cert_verifier: None, } } @@ -135,6 +147,8 @@ impl ConnectorOptions { offload_threadpool: None, bind_to_v4: vec![], bind_to_v6: vec![], + #[cfg(feature = "rustls")] + server_cert_verifier: None, } } } diff --git a/pingora-core/src/connectors/tls/rustls/mod.rs b/pingora-core/src/connectors/tls/rustls/mod.rs index 58ea4085..661db7ee 100644 --- a/pingora-core/src/connectors/tls/rustls/mod.rs +++ b/pingora-core/src/connectors/tls/rustls/mod.rs @@ -18,7 +18,7 @@ use log::debug; use pingora_error::{ Error, ErrorType::{ConnectTimedout, InvalidCert}, - OrErr, Result, + OkOrErr, OrErr, Result, }; use pingora_rustls::{ load_ca_file_into_store, load_certs_and_key_files, load_platform_certs_incl_env_into_store, @@ -54,6 +54,8 @@ impl Connector { pub struct TlsConnector { config: Arc, ca_certs: Arc, + /// Custom server cert verifier, stored for per-connection config rebuilds in connect(). + custom_verifier: Option>, } impl TlsConnector { @@ -68,15 +70,24 @@ impl TlsConnector { // - set supported ciphers/algorithms/curves // - add options for CRL/OCSP validation + let custom_verifier = options + .as_ref() + .and_then(|o| o.server_cert_verifier.clone()); + let (ca_certs, certs_key) = { let mut ca_certs = RootCertStore::empty(); let mut certs_key = None; if let Some(conf) = options.as_ref() { - if let Some(ca_file_path) = conf.ca_file.as_ref() { - load_ca_file_into_store(ca_file_path, &mut ca_certs)?; - } else { - load_platform_certs_incl_env_into_store(&mut ca_certs)?; + // When a custom verifier is provided, skip RootCertStore loading entirely. + // The custom verifier handles CA validation without webpki, which would + // reject certificates with unrecognized critical extensions. + if custom_verifier.is_none() { + if let Some(ca_file_path) = conf.ca_file.as_ref() { + load_ca_file_into_store(ca_file_path, &mut ca_certs)?; + } else { + load_platform_certs_incl_env_into_store(&mut ca_certs)?; + } } if let Some((cert, key)) = conf.cert_key_file.as_ref() { certs_key = load_certs_and_key_files(cert, key)?; @@ -88,23 +99,57 @@ impl TlsConnector { (ca_certs, certs_key) }; - // TODO: WebPkiServerVerifier for CRL/OCSP validation - let builder = - RusTlsClientConfig::builder_with_protocol_versions(&[&version::TLS12, &version::TLS13]) - .with_root_certificates(ca_certs.clone()); - - let mut config = match certs_key { - Some((certs, key)) => { - match builder.with_client_auth_cert(certs.clone(), key.clone_key()) { - Ok(config) => config, - Err(err) => { - // TODO: is there a viable alternative to the panic? - // falling back to no client auth... does not seem to be reasonable. - panic!("Failed to configure client auth cert/key. Error: {}", err); + let mut config = if let Some(ref verifier) = custom_verifier { + // Use the custom verifier — bypasses webpki for CA and server cert validation. + let builder = RusTlsClientConfig::builder_with_protocol_versions(&[ + &version::TLS12, + &version::TLS13, + ]) + .dangerous() + .with_custom_certificate_verifier(Arc::clone(verifier)); + + match certs_key { + Some((certs, key)) => { + // Bypass with_client_auth_cert() to skip webpki on the client chain, + // but still verify the key matches the leaf cert. + let provider = builder.crypto_provider().clone(); + let signing_key = provider + .key_provider + .load_private_key(key) + .or_err(InvalidCert, "Failed to load client private key")?; + let leaf = certs + .first() + .or_err(InvalidCert, "client cert chain is empty")?; + crate::protocols::tls::verify_cert_key_match(leaf, signing_key.as_ref())?; + let certified_key = + Arc::new(pingora_rustls::sign::CertifiedKey::new(certs, signing_key)); + builder.with_client_cert_resolver(Arc::new(SingleCertClientResolver( + certified_key, + ))) + } + None => builder.with_no_client_auth(), + } + } else { + // Default webpki path (unchanged from original behavior) + let builder = RusTlsClientConfig::builder_with_protocol_versions(&[ + &version::TLS12, + &version::TLS13, + ]) + .with_root_certificates(ca_certs.clone()); + + match certs_key { + Some((certs, key)) => { + match builder.with_client_auth_cert(certs.clone(), key.clone_key()) { + Ok(config) => config, + Err(err) => { + // TODO: is there a viable alternative to the panic? + // falling back to no client auth... does not seem to be reasonable. + panic!("Failed to configure client auth cert/key. Error: {}", err); + } } } + None => builder.with_no_client_auth(), } - None => builder.with_no_client_auth(), }; // Enable SSLKEYLOGFILE support for debugging TLS traffic @@ -118,6 +163,7 @@ impl TlsConnector { ctx: Arc::new(TlsConnector { config: Arc::new(config), ca_certs: Arc::new(ca_certs), + custom_verifier, }), }) } @@ -179,20 +225,48 @@ where let private_key: PrivateKeyDer = key_arc.key().as_slice().to_owned().try_into().unwrap(); - let builder = RusTlsClientConfig::builder_with_protocol_versions(&[ - &version::TLS12, - &version::TLS13, - ]) - .with_root_certificates(Arc::clone(effective_ca_store)); - debug!("added root ca certificates"); - - let mut updated_config = builder.with_client_auth_cert(certs, private_key).or_err( - InvalidCert, - "Failed to use peer cert/key to update Rustls config", - )?; - // Preserve keylog setting from original config - updated_config.key_log = Arc::clone(&config.key_log); - Some(updated_config) + if let Some(ref verifier) = tls_ctx.custom_verifier { + // Custom verifier path: bypass webpki for both server and client certs. + let builder = RusTlsClientConfig::builder_with_protocol_versions(&[ + &version::TLS12, + &version::TLS13, + ]) + .dangerous() + .with_custom_certificate_verifier(Arc::clone(verifier)); + + let provider = builder.crypto_provider().clone(); + let signing_key = provider + .key_provider + .load_private_key(private_key) + .or_err(InvalidCert, "Failed to load peer client private key")?; + let leaf = certs + .first() + .or_err(InvalidCert, "peer client cert chain is empty")?; + crate::protocols::tls::verify_cert_key_match(leaf, signing_key.as_ref())?; + let certified_key = + Arc::new(pingora_rustls::sign::CertifiedKey::new(certs, signing_key)); + + let mut updated_config = builder + .with_client_cert_resolver(Arc::new(SingleCertClientResolver(certified_key))); + updated_config.key_log = Arc::clone(&config.key_log); + Some(updated_config) + } else { + // Default webpki path + let builder = RusTlsClientConfig::builder_with_protocol_versions(&[ + &version::TLS12, + &version::TLS13, + ]) + .with_root_certificates(Arc::clone(effective_ca_store)); + debug!("added root ca certificates"); + + let mut updated_config = builder.with_client_auth_cert(certs, private_key).or_err( + InvalidCert, + "Failed to use peer cert/key to update Rustls config", + )?; + // Preserve keylog setting from original config + updated_config.key_log = Arc::clone(&config.key_log); + Some(updated_config) + } } }; @@ -242,15 +316,23 @@ where // Builds the custom_verifier when verification_mode is set. if let Some(mode) = verification_mode { - let delegate = WebPkiServerVerifier::builder(Arc::clone(effective_ca_store)) - .build() - .or_err(InvalidCert, "Failed to build WebPkiServerVerifier")?; - - let custom_verifier = Arc::new(CustomServerCertVerifier::new(delegate, mode)); - - updated_config - .dangerous() - .set_certificate_verifier(custom_verifier); + if let Some(ref verifier) = tls_ctx.custom_verifier { + // Wrap the custom verifier with verification mode logic + let custom_verifier = + Arc::new(CustomServerCertVerifier::new(Arc::clone(verifier), mode)); + updated_config + .dangerous() + .set_certificate_verifier(custom_verifier); + } else { + // Default: wrap the WebPkiServerVerifier + let delegate = WebPkiServerVerifier::builder(Arc::clone(effective_ca_store)) + .build() + .or_err(InvalidCert, "Failed to build WebPkiServerVerifier")?; + let custom_verifier = Arc::new(CustomServerCertVerifier::new(delegate, mode)); + updated_config + .dangerous() + .set_certificate_verifier(custom_verifier); + } } } @@ -290,6 +372,26 @@ where } } +/// Client cert resolver that returns a pre-loaded CertifiedKey without webpki validation. +/// Used when `ConnectorOptions::server_cert_verifier` is set, to avoid webpki rejecting +/// certificates with unrecognized critical extensions. +#[derive(Debug)] +struct SingleCertClientResolver(Arc); + +impl pingora_rustls::ResolvesClientCert for SingleCertClientResolver { + fn resolve( + &self, + _root_hint_subjects: &[&[u8]], + _sigschemes: &[SignatureScheme], + ) -> Option> { + Some(Arc::clone(&self.0)) + } + + fn has_certs(&self) -> bool { + true + } +} + #[allow(dead_code)] #[derive(Debug)] pub enum VerificationMode { @@ -302,12 +404,15 @@ pub enum VerificationMode { #[derive(Debug)] pub struct CustomServerCertVerifier { - delegate: Arc, + delegate: Arc, verification_mode: VerificationMode, } impl CustomServerCertVerifier { - pub fn new(delegate: Arc, verification_mode: VerificationMode) -> Self { + pub fn new( + delegate: Arc, + verification_mode: VerificationMode, + ) -> Self { Self { delegate, verification_mode, @@ -316,7 +421,7 @@ impl CustomServerCertVerifier { } // CustomServerCertVerifier delegates TLS signature verification and allows 3 VerificationMode: -// Full: delegates all verification to the original WebPkiServerVerifier +// Full: delegates all verification to the underlying ServerCertVerifier // SkipHostname: same as "Full" but ignores "NotValidForName" certificate errors // SkipAll: all certificate verification checks are skipped. impl RusTlsServerCertVerifier for CustomServerCertVerifier { diff --git a/pingora-core/src/listeners/tls/rustls/mod.rs b/pingora-core/src/listeners/tls/rustls/mod.rs index 0ca94d51..2393b863 100644 --- a/pingora-core/src/listeners/tls/rustls/mod.rs +++ b/pingora-core/src/listeners/tls/rustls/mod.rs @@ -15,12 +15,14 @@ use std::sync::Arc; use crate::listeners::TlsAcceptCallbacks; -use crate::protocols::tls::{server::handshake, server::handshake_with_callback, TlsStream}; +use crate::protocols::tls::{ + server::handshake, server::handshake_with_callback, verify_cert_key_match, TlsStream, +}; use log::debug; -use pingora_error::ErrorType::InternalError; -use pingora_error::{Error, OrErr, Result}; +use pingora_error::{Error, ErrorType::InvalidCert, Result}; use pingora_rustls::load_certs_and_key_files; use pingora_rustls::ClientCertVerifier; +use pingora_rustls::ResolvesServerCert; use pingora_rustls::ServerConfig; use pingora_rustls::{version, TlsAcceptor as RusTlsAcceptor}; @@ -31,7 +33,9 @@ pub struct TlsSettings { alpn_protocols: Option>>, cert_path: String, key_path: String, + cert_resolver: Option>, client_cert_verifier: Option>, + callbacks: Option, } pub struct Acceptor { @@ -48,27 +52,63 @@ impl TlsSettings { /// /// Todo: Return a result instead of panicking XD pub fn build(self) -> Acceptor { - let Ok(Some((certs, key))) = load_certs_and_key_files(&self.cert_path, &self.key_path) - else { - panic!( - "Failed to load provided certificates \"{}\" or key \"{}\".", - self.cert_path, self.key_path - ) - }; - let builder = ServerConfig::builder_with_protocol_versions(&[&version::TLS12, &version::TLS13]); + let provider = builder.crypto_provider().clone(); let builder = if let Some(verifier) = self.client_cert_verifier { builder.with_client_cert_verifier(verifier) } else { builder.with_no_client_auth() }; - let mut config = builder - .with_single_cert(certs, key) - .explain_err(InternalError, |e| { - format!("Failed to create server listener config: {e}") - }) - .unwrap(); + + let resolver: Arc = match self.cert_resolver { + Some(r) => r, + None => { + assert!( + !self.cert_path.is_empty() && !self.key_path.is_empty(), + "Either set_cert_resolver() or both set_certificate_chain_file() and \ + set_private_key_file() must be called before build()." + ); + + let Ok(Some((certs, key))) = + load_certs_and_key_files(&self.cert_path, &self.key_path) + else { + panic!( + "Failed to load provided certificates \"{}\" or key \"{}\".", + self.cert_path, self.key_path + ) + }; + + // Bypass with_single_cert() because its webpki policy validation rejects + // certificates with unrecognized critical extensions; keep the key match check. + let signing_key = provider + .key_provider + .load_private_key(key) + .expect("Failed to load server private key"); + let leaf = certs + .first() + .expect("load_certs_and_key_files returned an empty cert chain"); + verify_cert_key_match(leaf, signing_key.as_ref()) + .expect("Server certificate and private key do not match"); + let certified_key = + Arc::new(pingora_rustls::sign::CertifiedKey::new(certs, signing_key)); + + #[derive(Debug)] + struct SingleCert(Arc); + impl ResolvesServerCert for SingleCert { + fn resolve( + &self, + _client_hello: pingora_rustls::ClientHello<'_>, + ) -> Option> { + Some(Arc::clone(&self.0)) + } + } + + Arc::new(SingleCert(certified_key)) + } + }; + + let mut config = builder.with_cert_resolver(resolver); if let Some(alpn_protocols) = self.alpn_protocols { config.alpn_protocols = alpn_protocols; @@ -76,7 +116,7 @@ impl TlsSettings { Acceptor { acceptor: RusTlsAcceptor::from(Arc::new(config)), - callbacks: None, + callbacks: self.callbacks, } } @@ -95,6 +135,43 @@ impl TlsSettings { self.client_cert_verifier = Some(verifier); } + /// Install a user-provided server certificate resolver. + /// + /// When set, any certificate/key paths are ignored at `build()`. Useful for + /// dynamic SNI-based selection, where the cert cannot be chosen ahead of the + /// handshake. + pub fn set_cert_resolver(&mut self, resolver: Arc) { + self.cert_resolver = Some(resolver); + } + + /// Set the path to the certificate chain file (PEM format). + /// + /// Returns an error if the file cannot be opened or contains no X.509 + /// certificate, matching the OpenSSL backend's behavior. + pub fn set_certificate_chain_file(&mut self, path: &str) -> Result<()> { + let path_str = path.to_string(); + let bytes = pingora_rustls::load_pem_file_ca(&path_str)?; + if bytes.is_empty() { + return Error::e_explain(InvalidCert, format!("No X.509 certificate found in {path}")); + } + self.cert_path = path_str; + Ok(()) + } + + /// Set the path to the private key file (PEM format). + /// + /// Returns an error if the file cannot be opened or contains no private + /// key, matching the OpenSSL backend's behavior. + pub fn set_private_key_file(&mut self, path: &str) -> Result<()> { + let path_str = path.to_string(); + let bytes = pingora_rustls::load_pem_file_private_key(&path_str)?; + if bytes.is_empty() { + return Error::e_explain(InvalidCert, format!("No private key found in {path}")); + } + self.key_path = path_str; + Ok(()) + } + pub fn intermediate(cert_path: &str, key_path: &str) -> Result where Self: Sized, @@ -103,19 +180,29 @@ impl TlsSettings { alpn_protocols: None, cert_path: cert_path.to_string(), key_path: key_path.to_string(), + cert_resolver: None, client_cert_verifier: None, + callbacks: None, }) } - pub fn with_callbacks() -> Result + /// Create a new [`TlsSettings`] with post-handshake callbacks. + /// + /// Before calling `build()`, supply either a cert/key pair via + /// `set_certificate_chain_file` + `set_private_key_file`, or a custom + /// resolver via `set_cert_resolver`. + pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result where Self: Sized, { - // TODO: verify if/how callback in handshake can be done using Rustls - Error::e_explain( - InternalError, - "Certificate callbacks are not supported with feature \"rustls\".", - ) + Ok(TlsSettings { + alpn_protocols: None, + cert_path: String::new(), + key_path: String::new(), + cert_resolver: None, + client_cert_verifier: None, + callbacks: Some(callbacks), + }) } } diff --git a/pingora-core/src/protocols/tls/rustls/mod.rs b/pingora-core/src/protocols/tls/rustls/mod.rs index c7c81fc8..e5537003 100644 --- a/pingora-core/src/protocols/tls/rustls/mod.rs +++ b/pingora-core/src/protocols/tls/rustls/mod.rs @@ -19,7 +19,76 @@ mod stream; pub use stream::*; use crate::utils::tls::WrappedX509; +use pingora_error::{Error, ErrorType::InvalidCert, Result}; +use pingora_rustls::sign::SigningKey; +use pingora_rustls::CertificateDer; pub type CaType = [WrappedX509]; -pub struct TlsRef; +/// Verify the leaf certificate's public key matches the given signing key. +/// Stands in for rustls's `with_single_cert` consistency check, which we +/// bypass to skip webpki policy validation on our own cert chain. +pub(crate) fn verify_cert_key_match( + cert: &CertificateDer<'_>, + signing_key: &dyn SigningKey, +) -> Result<()> { + let Some(key_spki) = signing_key.public_key() else { + return Ok(()); + }; + + let (_, parsed) = x509_parser::parse_x509_certificate(cert.as_ref()) + .map_err(|_| Error::explain(InvalidCert, "Failed to parse certificate for key match"))?; + + if parsed.tbs_certificate.subject_pki.raw != key_spki.as_ref() { + return Error::e_explain( + InvalidCert, + "Certificate public key does not match private key", + ); + } + Ok(()) +} + +/// TLS connection state exposed to post-handshake callbacks. +/// +/// Provides access to peer certificates and negotiated cipher suite +/// after a TLS handshake completes. This is the rustls equivalent of +/// the OpenSSL `SslRef` that is used as `TlsRef` in the boringssl/openssl path. +#[derive(Debug)] +pub struct TlsRef { + pub(super) peer_certs: Option>>, + pub(super) cipher: Option<&'static str>, + pub(super) version: Option<&'static str>, + pub(super) server_name: Option, +} + +impl TlsRef { + /// Returns the peer's leaf certificate in DER encoding, if present. + pub fn peer_certificate_der(&self) -> Option<&[u8]> { + self.peer_certs + .as_ref() + .and_then(|certs| certs.first()) + .map(|cert| cert.as_ref()) + } + + /// Returns the full peer certificate chain in DER encoding. + /// The first entry is the leaf; subsequent entries are intermediates. + pub fn peer_cert_chain_der(&self) -> Option<&[CertificateDer<'static>]> { + self.peer_certs.as_deref() + } + + /// Returns the negotiated cipher suite name, if available. + pub fn current_cipher_name(&self) -> Option<&'static str> { + self.cipher + } + + /// Returns the negotiated TLS protocol version (e.g. "TLSv1.3"), if available. + pub fn version(&self) -> Option<&'static str> { + self.version + } + + /// Returns the SNI hostname sent by the peer, if any. Only populated on + /// server-side connections; always `None` on client streams. + pub fn server_name(&self) -> Option<&str> { + self.server_name.as_deref() + } +} diff --git a/pingora-core/src/protocols/tls/rustls/server.rs b/pingora-core/src/protocols/tls/rustls/server.rs index 4367f75a..622d53ca 100644 --- a/pingora-core/src/protocols/tls/rustls/server.rs +++ b/pingora-core/src/protocols/tls/rustls/server.rs @@ -16,8 +16,7 @@ use crate::listeners::TlsAcceptCallbacks; use crate::protocols::tls::rustls::TlsStream; -use crate::protocols::tls::TlsRef; -use crate::protocols::IO; +use crate::protocols::{Ssl, IO}; use crate::{listeners::tls::Acceptor, protocols::Shutdown}; use async_trait::async_trait; use log::warn; @@ -65,7 +64,6 @@ pub async fn handshake(acceptor: &Acceptor, io: S) -> Result } /// Perform TLS handshake for the given connection with the given configuration and callbacks -/// callbacks are currently not supported within pingora Rustls and are ignored pub async fn handshake_with_callback( acceptor: &Acceptor, io: S, @@ -74,20 +72,21 @@ pub async fn handshake_with_callback( let mut tls_stream = prepare_tls_stream(acceptor, io).await?; let done = Pin::new(&mut tls_stream).start_accept().await?; if !done { - // TODO: verify if/how callback in handshake can be done using Rustls - warn!("Callacks are not supported with feature \"rustls\"."); - + // NOTE: certificate_callback is not invoked for rustls. Dynamic cert selection + // should use a custom ResolvesServerCert instead. + warn!("certificate_callback is not supported with the rustls backend; use ResolvesServerCert for dynamic cert selection"); Pin::new(&mut tls_stream) .resume_accept() .await .explain_err(TLSHandshakeFailure, |e| format!("TLS accept() failed: {e}"))?; } - { - let tls_ref = TlsRef; - if let Some(extension) = callbacks.handshake_complete_callback(&tls_ref).await { - if let Some(digest_mut) = tls_stream.ssl_digest_mut() { - digest_mut.extension.set(extension); - } + let extension = match tls_stream.get_ssl() { + Some(tls_ref) => callbacks.handshake_complete_callback(tls_ref).await, + None => None, + }; + if let Some(extension) = extension { + if let Some(digest_mut) = tls_stream.ssl_digest_mut() { + digest_mut.extension.set(extension); } } Ok(tls_stream) @@ -108,8 +107,100 @@ where } } -#[ignore] -#[tokio::test] -async fn test_async_cert() { - todo!("callback support and test for Rustls") +#[cfg(test)] +mod tests { + use crate::listeners::tls::TlsSettings; + use crate::listeners::TlsAccept; + use crate::protocols::tls::TlsRef; + use async_trait::async_trait; + use pingora_rustls::{ + ClientConfig, HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier, ServerName, + }; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, DuplexStream}; + + #[derive(Debug)] + struct NoVerify; + + impl ServerCertVerifier for NoVerify { + fn verify_server_cert( + &self, + _: &rustls::pki_types::CertificateDer<'_>, + _: &[rustls::pki_types::CertificateDer<'_>], + _: &ServerName<'_>, + _: &[u8], + _: pingora_rustls::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _: &[u8], + _: &rustls::pki_types::CertificateDer<'_>, + _: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _: &[u8], + _: &rustls::pki_types::CertificateDer<'_>, + _: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + } + + async fn client_task(client: DuplexStream) { + let config = ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerify)) + .with_no_client_auth(); + let connector = pingora_rustls::TlsConnector::from(Arc::new(config)); + let server_name = ServerName::try_from("openrusty.org").unwrap(); + let mut stream = connector.connect(server_name, client).await.unwrap(); + let mut buf = [0u8; 1]; + let _ = stream.read(&mut buf).await; + } + + #[tokio::test] + async fn test_handshake_complete_callback() { + struct CipherName(String); + struct Callback; + + #[async_trait] + impl TlsAccept for Callback { + async fn handshake_complete_callback( + &self, + tls: &TlsRef, + ) -> Option> { + let name = tls.current_cipher_name()?.to_string(); + Some(Arc::new(CipherName(name))) + } + } + + let cert = format!("{}/tests/keys/server.crt", env!("CARGO_MANIFEST_DIR")); + let key = format!("{}/tests/keys/key.pem", env!("CARGO_MANIFEST_DIR")); + + let mut settings = TlsSettings::with_callbacks(Box::new(Callback)).unwrap(); + settings.set_certificate_chain_file(&cert).unwrap(); + settings.set_private_key_file(&key).unwrap(); + let acceptor = settings.build(); + + let (client, server) = tokio::io::duplex(4096); + tokio::spawn(client_task(client)); + + let stream = acceptor.tls_handshake(server).await.unwrap(); + let digest = stream.ssl_digest().unwrap(); + let cipher = digest.extension.get::().unwrap(); + assert!(!cipher.0.is_empty()); + } } diff --git a/pingora-core/src/protocols/tls/rustls/stream.rs b/pingora-core/src/protocols/tls/rustls/stream.rs index f2a0ddae..37e61948 100644 --- a/pingora-core/src/protocols/tls/rustls/stream.rs +++ b/pingora-core/src/protocols/tls/rustls/stream.rs @@ -21,6 +21,7 @@ use std::time::{Duration, SystemTime}; use crate::listeners::tls::Acceptor; use crate::protocols::raw_connect::ProxyDigest; +use crate::protocols::tls::TlsRef; use crate::protocols::{tls::SslDigest, Peek, TimingDigest, UniqueIDType}; use crate::protocols::{ GetProxyDigest, GetSocketDigest, GetTimingDigest, SocketDigest, Ssl, UniqueID, ALPN, @@ -46,6 +47,7 @@ pub struct InnerStream { pub struct TlsStream { tls: InnerStream, digest: Option>, + ssl: Option, timing: TimingDigest, } @@ -69,6 +71,7 @@ where Ok(TlsStream { tls, digest: None, + ssl: None, timing: Default::default(), }) } @@ -85,6 +88,7 @@ where Ok(TlsStream { tls, digest: None, + ssl: None, timing: Default::default(), }) } @@ -164,6 +168,7 @@ where self.tls.connect().await?; self.timing.established_ts = SystemTime::now(); self.digest = self.tls.digest(); + self.ssl = self.tls.tls_ref(); Ok(()) } @@ -172,6 +177,7 @@ where self.tls.accept().await?; self.timing.established_ts = SystemTime::now(); self.digest = self.tls.digest(); + self.ssl = self.tls.tls_ref(); Ok(()) } } @@ -228,6 +234,10 @@ where } impl Ssl for TlsStream { + fn get_ssl(&self) -> Option<&TlsRef> { + self.ssl.as_ref() + } + fn get_ssl_digest(&self) -> Option> { self.ssl_digest() } @@ -310,6 +320,26 @@ impl InnerStream { pub(crate) fn digest(&mut self) -> Option> { Some(Arc::new(SslDigest::from_stream(&self.stream))) } + + pub(crate) fn tls_ref(&self) -> Option { + let stream = self.stream.as_ref()?; + let (_io, session) = stream.get_ref(); + let peer_certs = session.peer_certificates().map(|certs| certs.to_vec()); + let cipher = session + .negotiated_cipher_suite() + .and_then(|suite| suite.suite().as_str()); + let version = session.protocol_version().and_then(|v| v.as_str()); + let server_name = match stream { + RusTlsStream::Server(s) => s.get_ref().1.server_name().map(|n| n.to_string()), + RusTlsStream::Client(_) => None, + }; + Some(TlsRef { + peer_certs, + cipher, + version, + server_name, + }) + } } impl GetSocketDigest for InnerStream diff --git a/pingora-proxy/src/lib.rs b/pingora-proxy/src/lib.rs index e5433efa..6b495eab 100644 --- a/pingora-proxy/src/lib.rs +++ b/pingora-proxy/src/lib.rs @@ -174,8 +174,23 @@ where SV: ProxyHttp + Send + Sync + 'static, SV::CTX: Send + Sync, { - let client_upstream = - Connector::new_custom(Some(ConnectorOptions::from_server_conf(&conf)), connector); + let opts = ConnectorOptions::from_server_conf(&conf); + Self::new_custom_with_options(inner, conf, connector, on_custom, server_options, opts) + } + + fn new_custom_with_options( + inner: SV, + conf: Arc, + connector: C, + on_custom: Option>, + server_options: Option, + connector_options: ConnectorOptions, + ) -> Self + where + SV: ProxyHttp + Send + Sync + 'static, + SV::CTX: Send + Sync, + { + let client_upstream = Connector::new_custom(Some(connector_options), connector); HttpProxy { inner, @@ -1383,6 +1398,7 @@ where connector: C, custom: Option>, server_options: Option, + connector_options: Option, } impl ProxyServiceBuilder @@ -1407,6 +1423,7 @@ where connector: (), custom: None, server_options: None, + connector_options: None, } } } @@ -1441,6 +1458,7 @@ where inner, name, server_options, + connector_options, .. } = self; ProxyServiceBuilder { @@ -1450,6 +1468,7 @@ where connector, custom: Some(on_custom), server_options, + connector_options, } } @@ -1461,6 +1480,15 @@ where self } + /// Override the default [ConnectorOptions] derived from [ServerConf]. + /// + /// This allows setting custom options such as a custom server certificate verifier + /// for the upstream TLS connector. + pub fn connector_options(mut self, options: ConnectorOptions) -> Self { + self.connector_options = Some(options); + self + } + /// Builds a new [Service] from the [ProxyServiceBuilder]. /// /// This function takes ownership of the [ProxyServiceBuilder] and returns a new [Service] with @@ -1475,9 +1503,18 @@ where connector, custom, server_options, + connector_options, } = self; - let mut proxy = HttpProxy::new_custom(inner, conf, connector, custom, server_options); + let opts = connector_options.unwrap_or_else(|| ConnectorOptions::from_server_conf(&conf)); + let mut proxy = HttpProxy::new_custom_with_options( + inner, + conf, + connector, + custom, + server_options, + opts, + ); proxy.handle_init_modules(); Service::new(name, proxy) diff --git a/pingora-rustls/src/lib.rs b/pingora-rustls/src/lib.rs index 097a8da5..bcb7047a 100644 --- a/pingora-rustls/src/lib.rs +++ b/pingora-rustls/src/lib.rs @@ -26,7 +26,9 @@ pub use no_debug::{Ellipses, NoDebug, WithTypeInfo}; use pingora_error::{Error, ErrorType, OrErr, Result}; pub use rustls::server::danger::{ClientCertVerified, ClientCertVerifier}; -pub use rustls::server::{ClientCertVerifierBuilder, WebPkiClientVerifier}; +pub use rustls::server::{ + ClientCertVerifierBuilder, ClientHello, ResolvesServerCert, WebPkiClientVerifier, +}; pub use rustls::{ client::WebPkiServerVerifier, version, CertificateError, ClientConfig, DigitallySignedStruct, Error as RusTlsError, KeyLogFile, RootCertStore, ServerConfig, SignatureScheme, Stream, @@ -38,6 +40,9 @@ pub use tokio_rustls::client::TlsStream as ClientTlsStream; pub use tokio_rustls::server::TlsStream as ServerTlsStream; pub use tokio_rustls::{Accept, Connect, TlsAcceptor, TlsConnector, TlsStream}; +pub use rustls::client::ResolvesClientCert; +pub use rustls::sign; + // This allows to skip certificate verification. Be highly cautious. pub use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};