Skip to content

Commit abfa0c9

Browse files
danzh2010danzh1989
andauthored
use QUICHE built-in port and Server Preferred Address migration (#41798)
Commit Message: switch to use QUICHE built-in migration to handle port migration upon path degrading and migration to Server Preferred Address(SPA) . This change introduces a `QuicClientPacketWriterFactory` to abstract the creation of QUIC packet writers and their associated connection sockets. It also change EnvoyQuicClientSession to construct with a different QuicSpdyClientSession constructor which takes migration-related parameters. It also implements QUICHE migration related interfaces: `QuicMigrationHelper` and `QuicPathContextFactory` for Envoy to provide path validation context and to retrieve networks which is needed for QUICHE built-in migration. Added two more interfaces `getOrCreateMigrationHelper()` and `setWriterFactory()` to `EnvoyQuicClientConnection`. Depending on which one is called, the connection will conduct port migration and SPA migration by itself or defer migration to QUICHE and act as a migration helper. For upstream connections created by the connection pool,`EnvoyQuicClientSession` and `EnvoyQuicClientConnection` will optionally be constructed and setup to enable QUICHE built-in migration. A runtime feature, `envoy.reloadable_features.use_migration_in_quiche`, is added for this purpose. Risk Level: medium, new QUICHE code will be exercised Testing: existing tests pass Docs Changes: N/A Release Notes: Y Runtime guard: envoy.reloadable_features.use_migration_in_quiche --------- Signed-off-by: Dan Zhang <[email protected]> Co-authored-by: Dan Zhang <[email protected]>
1 parent b6ace88 commit abfa0c9

21 files changed

+572
-120
lines changed

changelogs/current.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ minor_behavior_changes:
4242
change: |
4343
Check that the response header count is less than the configured limits before applying mutations, and do not
4444
add new headers if not.
45+
- area: quic
46+
change: |
47+
Switch to use QUICHE provided migration logic to handle port migration upon path degrading and migration to Server
48+
Preferred Address. This behavior can be reverted by setting ``envoy.reloadable_features.use_migration_in_quiche``.
4549
- area: mobile
4650
change: |
4751
Use mobile specific network observer registries to propagate network change signals. This behavior can be reverted by

source/common/quic/BUILD

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ envoy_cc_library(
198198
":envoy_quic_connection_helper_lib",
199199
":envoy_quic_proof_verifier_lib",
200200
":envoy_quic_utils_lib",
201+
":quic_client_packet_writer_factory_impl_lib",
201202
"//envoy/http:codec_interface",
202203
"//envoy/http:persistent_quic_info_interface",
203204
"//envoy/registry",
@@ -395,6 +396,18 @@ envoy_cc_library(
395396
]),
396397
)
397398

399+
envoy_cc_library(
400+
name = "quic_client_packet_writer_factory_impl_lib",
401+
srcs = envoy_select_enable_http3(["quic_client_packet_writer_factory_impl.cc"]),
402+
hdrs = envoy_select_enable_http3(["quic_client_packet_writer_factory_impl.h"]),
403+
deps = envoy_select_enable_http3([
404+
":envoy_quic_packet_writer_lib",
405+
":envoy_quic_utils_lib",
406+
":envoy_quic_client_connection_lib",
407+
"//source/common/network:udp_packet_writer_handler_lib",
408+
]),
409+
)
410+
398411
envoy_cc_library(
399412
name = "envoy_quic_client_connection_lib",
400413
srcs = envoy_select_enable_http3(["envoy_quic_client_connection.cc"]),
@@ -407,6 +420,7 @@ envoy_cc_library(
407420
"//source/common/network:udp_packet_writer_handler_lib",
408421
"//source/common/runtime:runtime_lib",
409422
"@com_github_google_quiche//:quic_core_connection_lib",
423+
"@com_github_google_quiche//:quic_core_http_client_lib",
410424
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
411425
]),
412426
)
@@ -479,6 +493,7 @@ envoy_cc_library(
479493
"@com_github_google_quiche//:quic_core_config_lib",
480494
"@com_github_google_quiche//:quic_core_http_header_list_lib",
481495
"@com_github_google_quiche//:quic_platform",
496+
"@com_github_google_quiche//:quic_core_http_client_lib",
482497
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
483498
"@envoy_api//envoy/config/listener/v3:pkg_cc_proto",
484499
]),

source/common/quic/client_connection_factory_impl.cc

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
#include "source/common/quic/client_connection_factory_impl.h"
22

3+
#include "source/common/network/udp_packet_writer_handler_impl.h"
4+
#include "source/common/quic/envoy_quic_packet_writer.h"
5+
#include "source/common/quic/envoy_quic_utils.h"
6+
#include "source/common/quic/quic_client_packet_writer_factory_impl.h"
7+
#include "source/common/quic/quic_transport_socket_factory.h"
38
#include "source/common/runtime/runtime_features.h"
49

510
namespace Envoy {
@@ -10,6 +15,9 @@ PersistentQuicInfoImpl::PersistentQuicInfoImpl(Event::Dispatcher& dispatcher, ui
1015
: conn_helper_(dispatcher), alarm_factory_(dispatcher, *conn_helper_.GetClock()),
1116
buffer_limit_(buffer_limit), max_packet_length_(max_packet_length) {
1217
quiche::FlagRegistry::getInstance();
18+
// Allow migration to server preferred address by default.
19+
migration_config_.allow_server_preferred_address = true;
20+
migration_config_.max_port_migrations_per_session = kMaxNumSocketSwitches;
1321
migration_config_.migrate_session_on_network_change = false;
1422
}
1523

@@ -31,6 +39,11 @@ createPersistentQuicInfoForCluster(Event::Dispatcher& dispatcher,
3139
}
3240
quic_info->max_packet_length_ =
3341
PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_config, max_packet_length, 0);
42+
uint32_t num_timeouts_to_trigger_port_migration =
43+
PROTOBUF_GET_WRAPPED_OR_DEFAULT(quic_config, num_timeouts_to_trigger_port_migration, 0);
44+
quic_info->migration_config_.allow_port_migration = (num_timeouts_to_trigger_port_migration > 0);
45+
// TODO: make this an extension point.
46+
quic_info->writer_factory_ = std::make_unique<QuicClientPacketWriterFactoryImpl>();
3447
return quic_info;
3548
}
3649

@@ -52,14 +65,44 @@ std::unique_ptr<Network::ClientConnection> createQuicNetworkConnection(
5265
PersistentQuicInfoImpl* info_impl = reinterpret_cast<PersistentQuicInfoImpl*>(&info);
5366
quic::ParsedQuicVersionVector quic_versions = quic::CurrentSupportedHttp3Versions();
5467
ASSERT(!quic_versions.empty());
68+
ASSERT(info_impl->writer_factory_ != nullptr);
69+
QuicClientPacketWriterFactory::CreationResult creation_result =
70+
info_impl->writer_factory_->createSocketAndQuicPacketWriter(
71+
server_addr, quic::kInvalidNetworkHandle, local_addr, options);
72+
const bool use_migration_in_quiche =
73+
Runtime::runtimeFeatureEnabled("envoy.reloadable_features.use_migration_in_quiche");
74+
quic::QuicForceBlockablePacketWriter* wrapper = nullptr;
75+
if (use_migration_in_quiche) {
76+
wrapper = new quic::QuicForceBlockablePacketWriter();
77+
// Owns the inner writer.
78+
wrapper->set_writer(creation_result.writer_.release());
79+
}
5580
auto connection = std::make_unique<EnvoyQuicClientConnection>(
56-
quic::QuicUtils::CreateRandomConnectionId(), server_addr, info_impl->conn_helper_,
57-
info_impl->alarm_factory_, quic_versions, local_addr, dispatcher, options, generator);
81+
quic::QuicUtils::CreateRandomConnectionId(), info_impl->conn_helper_,
82+
info_impl->alarm_factory_,
83+
(use_migration_in_quiche
84+
? wrapper
85+
: static_cast<quic::QuicPacketWriter*>(creation_result.writer_.release())),
86+
/*owns_writer=*/true, quic_versions, dispatcher, std::move(creation_result.socket_),
87+
generator);
5888
// Override the max packet length of the QUIC connection if the option value is not 0.
5989
if (info_impl->max_packet_length_ > 0) {
6090
connection->SetMaxPacketLength(info_impl->max_packet_length_);
6191
}
6292

93+
EnvoyQuicClientConnection::EnvoyQuicMigrationHelper* migration_helper = nullptr;
94+
quic::QuicConnectionMigrationConfig migration_config = info_impl->migration_config_;
95+
if (use_migration_in_quiche) {
96+
migration_helper = &connection->getOrCreateMigrationHelper(
97+
*info_impl->writer_factory_,
98+
makeOptRefFromPtr<EnvoyQuicNetworkObserverRegistry>(network_observer_registry));
99+
} else {
100+
// The connection needs to be aware of the writer factory so it can create migration probing
101+
// sockets.
102+
connection->setWriterFactory(*info_impl->writer_factory_);
103+
// Disable all kinds of migration in QUICHE as the session won't be setup to handle it.
104+
migration_config = quicConnectionMigrationDisableAllConfig();
105+
}
63106
// TODO (danzh) move this temporary config and initial RTT configuration to h3 pool.
64107
quic::QuicConfig config = info_impl->quic_config_;
65108
// Update config with latest srtt, if available.
@@ -75,9 +118,10 @@ std::unique_ptr<Network::ClientConnection> createQuicNetworkConnection(
75118

76119
// QUICHE client session always use the 1st version to start handshake.
77120
auto session = std::make_unique<EnvoyQuicClientSession>(
78-
config, quic_versions, std::move(connection), server_id, std::move(crypto_config), dispatcher,
79-
info_impl->buffer_limit_, info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache,
80-
scope, transport_socket_options, transport_socket_factory);
121+
config, quic_versions, std::move(connection), wrapper, migration_helper, migration_config,
122+
server_id, std::move(crypto_config), dispatcher, info_impl->buffer_limit_,
123+
info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, scope,
124+
transport_socket_options, transport_socket_factory);
81125
if (network_observer_registry != nullptr) {
82126
session->registerNetworkObserver(*network_observer_registry);
83127
}

source/common/quic/client_connection_factory_impl.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo {
3737
quic::QuicByteCount max_packet_length_;
3838
// TODO(danzh): Add a config knob to configure connection migration.
3939
quic::QuicConnectionMigrationConfig migration_config_;
40+
QuicClientPacketWriterFactoryPtr writer_factory_;
4041
};
4142

4243
std::unique_ptr<PersistentQuicInfoImpl>

source/common/quic/envoy_quic_client_connection.cc

Lines changed: 138 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66

77
#include "source/common/network/socket_option_factory.h"
88
#include "source/common/network/udp_packet_writer_handler_impl.h"
9-
#include "source/common/quic/envoy_quic_utils.h"
109
#include "source/common/runtime/runtime_features.h"
1110

1211
namespace Envoy {
@@ -24,29 +23,107 @@ class DeferredDeletableSocket : public Event::DeferredDeletable {
2423
std::unique_ptr<Network::ConnectionSocket> socket_;
2524
};
2625

27-
EnvoyQuicClientConnection::EnvoyQuicClientConnection(
28-
const quic::QuicConnectionId& server_connection_id,
29-
Network::Address::InstanceConstSharedPtr& initial_peer_address,
30-
quic::QuicConnectionHelperInterface& helper, quic::QuicAlarmFactory& alarm_factory,
31-
const quic::ParsedQuicVersionVector& supported_versions,
32-
Network::Address::InstanceConstSharedPtr local_addr, Event::Dispatcher& dispatcher,
33-
const Network::ConnectionSocket::OptionsSharedPtr& options,
34-
quic::ConnectionIdGeneratorInterface& generator)
35-
: EnvoyQuicClientConnection(
36-
server_connection_id, helper, alarm_factory, supported_versions, dispatcher,
37-
createConnectionSocket(initial_peer_address, local_addr, options), generator) {}
26+
class EnvoyQuicClientPathValidationContext : public quic::QuicClientPathValidationContext {
27+
public:
28+
EnvoyQuicClientPathValidationContext(quic::QuicSocketAddress self_address,
29+
quic::QuicSocketAddress peer_address,
30+
quic::QuicNetworkHandle network,
31+
std::unique_ptr<EnvoyQuicPacketWriter>&& writer,
32+
Network::ConnectionSocketPtr&& socket,
33+
Event::Dispatcher& dispatcher)
34+
: quic::QuicClientPathValidationContext(self_address, peer_address, network),
35+
writer_(std::make_unique<quic::QuicForceBlockablePacketWriter>()),
36+
socket_(std::move(socket)), dispatcher_(dispatcher) {
37+
// Owns the writer.
38+
writer_->set_writer(writer.release());
39+
}
3840

39-
EnvoyQuicClientConnection::EnvoyQuicClientConnection(
40-
const quic::QuicConnectionId& server_connection_id, quic::QuicConnectionHelperInterface& helper,
41-
quic::QuicAlarmFactory& alarm_factory, const quic::ParsedQuicVersionVector& supported_versions,
42-
Event::Dispatcher& dispatcher, Network::ConnectionSocketPtr&& connection_socket,
43-
quic::ConnectionIdGeneratorInterface& generator)
44-
: EnvoyQuicClientConnection(
45-
server_connection_id, helper, alarm_factory,
46-
new EnvoyQuicPacketWriter(
47-
std::make_unique<Network::UdpDefaultWriter>(connection_socket->ioHandle())),
48-
/*owns_writer=*/true, supported_versions, dispatcher, std::move(connection_socket),
49-
generator) {}
41+
~EnvoyQuicClientPathValidationContext() override {
42+
if (socket_ != nullptr) {
43+
// The socket wasn't used by the connection, the path validation must have failed. Now
44+
// deferred delete it to avoid deleting IoHandle in a read loop.
45+
dispatcher_.deferredDelete(std::make_unique<DeferredDeletableSocket>(std::move(socket_)));
46+
}
47+
}
48+
49+
bool ShouldConnectionOwnWriter() const override { return true; }
50+
quic::QuicForceBlockablePacketWriter* ForceBlockableWriterToUse() override {
51+
return writer_.get();
52+
}
53+
54+
Network::ConnectionSocket& probingSocket() { return *socket_; }
55+
56+
quic::QuicForceBlockablePacketWriter* releaseWriter() { return writer_.release(); }
57+
std::unique_ptr<Network::ConnectionSocket> releaseSocket() { return std::move(socket_); }
58+
59+
private:
60+
std::unique_ptr<quic::QuicForceBlockablePacketWriter> writer_;
61+
Network::ConnectionSocketPtr socket_;
62+
Event::Dispatcher& dispatcher_;
63+
};
64+
65+
void EnvoyQuicClientConnection::EnvoyQuicClinetPathContextFactory::CreatePathValidationContext(
66+
quic::QuicNetworkHandle network, quic::QuicSocketAddress peer_address,
67+
std::unique_ptr<quic::QuicPathContextFactory::CreationResultDelegate> result_delegate) {
68+
Network::Address::InstanceConstSharedPtr new_local_address;
69+
if (network == quic::kInvalidNetworkHandle) {
70+
// If there isn't a meaningful network handle to bind to, bind to the
71+
// local address of the current socket.
72+
Network::Address::InstanceConstSharedPtr current_local_address =
73+
connection_.connectionSocket()->connectionInfoProvider().localAddress();
74+
if (current_local_address->ip()->version() == Network::Address::IpVersion::v4) {
75+
new_local_address = std::make_shared<Network::Address::Ipv4Instance>(
76+
current_local_address->ip()->addressAsString(),
77+
&current_local_address->socketInterface());
78+
} else {
79+
new_local_address = std::make_shared<Network::Address::Ipv6Instance>(
80+
current_local_address->ip()->addressAsString(),
81+
&current_local_address->socketInterface());
82+
}
83+
}
84+
Network::Address::InstanceConstSharedPtr remote_address =
85+
(connection_.peer_address() == peer_address)
86+
? connection_.connectionSocket()->connectionInfoProvider().remoteAddress()
87+
: quicAddressToEnvoyAddressInstance(peer_address);
88+
// new_local_address will be re-assigned if it is nullptr.
89+
QuicClientPacketWriterFactory::CreationResult result =
90+
writer_factory_.createSocketAndQuicPacketWriter(remote_address, network, new_local_address,
91+
connection_.connectionSocket()->options());
92+
connection_.setUpConnectionSocket(*result.socket_, connection_.delegate_);
93+
result_delegate->OnCreationSucceeded(std::make_unique<EnvoyQuicClientPathValidationContext>(
94+
envoyIpAddressToQuicSocketAddress(new_local_address->ip()), peer_address, network,
95+
std::move(result.writer_), std::move(result.socket_), connection_.dispatcher_));
96+
}
97+
98+
quic::QuicNetworkHandle EnvoyQuicClientConnection::EnvoyQuicMigrationHelper::FindAlternateNetwork(
99+
quic::QuicNetworkHandle /*network*/) {
100+
return quic::kInvalidNetworkHandle;
101+
}
102+
103+
quic::QuicNetworkHandle EnvoyQuicClientConnection::EnvoyQuicMigrationHelper::GetDefaultNetwork() {
104+
return quic::kInvalidNetworkHandle;
105+
}
106+
107+
void EnvoyQuicClientConnection::EnvoyQuicMigrationHelper::OnMigrationToPathDone(
108+
std::unique_ptr<quic::QuicClientPathValidationContext> context, bool success) {
109+
if (success) {
110+
auto* envoy_context = static_cast<EnvoyQuicClientPathValidationContext*>(context.get());
111+
// Connection already owns the writer.
112+
envoy_context->releaseWriter();
113+
++connection_.num_socket_switches_;
114+
connection_.setConnectionSocket(envoy_context->releaseSocket());
115+
// Previous writer may have been force blocked and write events on it may have been dropped.
116+
// Synthesize a write event in case this case to unblock the connection.
117+
connection_.connectionSocket()->ioHandle().activateFileEvents(Event::FileReadyType::Write);
118+
// Send something to notify the peer of the address change immediately.
119+
connection_.SendPing();
120+
}
121+
}
122+
123+
std::unique_ptr<quic::QuicPathContextFactory>
124+
EnvoyQuicClientConnection::EnvoyQuicMigrationHelper::CreateQuicPathContextFactory() {
125+
return std::make_unique<EnvoyQuicClinetPathContextFactory>(writer_factory_, connection_);
126+
}
50127

51128
EnvoyQuicClientConnection::EnvoyQuicClientConnection(
52129
const quic::QuicConnectionId& server_connection_id, quic::QuicConnectionHelperInterface& helper,
@@ -153,8 +230,12 @@ void EnvoyQuicClientConnection::switchConnectionSocket(
153230
}
154231

155232
void EnvoyQuicClientConnection::OnPathDegradingDetected() {
233+
// This will trigger connection migration or port migration in QUICHE if
234+
// migration_helper_ is initialized. Otherwise do it in this class.
156235
QuicConnection::OnPathDegradingDetected();
157-
maybeMigratePort();
236+
if (migration_helper_ == nullptr) {
237+
maybeMigratePort();
238+
}
158239
}
159240

160241
void EnvoyQuicClientConnection::maybeMigratePort() {
@@ -181,18 +262,20 @@ void EnvoyQuicClientConnection::probeWithNewPort(const quic::QuicSocketAddress&
181262
}
182263

183264
// The probing socket will have the same host but a different port.
184-
auto probing_socket = createConnectionSocket(
185-
peer_addr == peer_address() ? connectionSocket()->connectionInfoProvider().remoteAddress()
186-
: quicAddressToEnvoyAddressInstance(peer_addr),
187-
new_local_address, connectionSocket()->options());
188-
setUpConnectionSocket(*probing_socket, delegate_);
189-
auto writer = std::make_unique<EnvoyQuicPacketWriter>(
190-
std::make_unique<Network::UdpDefaultWriter>(probing_socket->ioHandle()));
265+
ASSERT(migration_helper_ == nullptr && writer_factory_.has_value());
266+
QuicClientPacketWriterFactory::CreationResult creation_result =
267+
writer_factory_->createSocketAndQuicPacketWriter(
268+
(peer_addr == peer_address()
269+
? connectionSocket()->connectionInfoProvider().remoteAddress()
270+
: quicAddressToEnvoyAddressInstance(peer_addr)),
271+
quic::kInvalidNetworkHandle, new_local_address, connectionSocket()->options());
272+
setUpConnectionSocket(*creation_result.socket_, delegate_);
273+
auto writer = std::move(creation_result.writer_);
191274
quic::QuicSocketAddress self_address = envoyIpAddressToQuicSocketAddress(
192-
probing_socket->connectionInfoProvider().localAddress()->ip());
275+
creation_result.socket_->connectionInfoProvider().localAddress()->ip());
193276

194277
auto context = std::make_unique<EnvoyQuicPathValidationContext>(
195-
self_address, peer_addr, std::move(writer), std::move(probing_socket));
278+
self_address, peer_addr, std::move(writer), std::move(creation_result.socket_));
196279
ValidatePath(std::move(context), std::make_unique<EnvoyPathValidationResultDelegate>(*this),
197280
reason);
198281
}
@@ -241,15 +324,21 @@ void EnvoyQuicClientConnection::onFileEvent(uint32_t events,
241324
ASSERT(events & (Event::FileReadyType::Read | Event::FileReadyType::Write));
242325

243326
if (events & Event::FileReadyType::Write) {
244-
OnBlockedWriterCanWrite();
327+
writer()->SetWritable();
328+
// The writer might still be force blocked for migration in progress, in
329+
// which case no write should be attempted.
330+
WriteIfNotBlocked();
245331
}
246332

247-
bool is_probing_socket =
333+
// Check if the event is on the probing socket before read.
334+
const bool is_probing_socket =
248335
HasPendingPathValidation() &&
249336
(&connection_socket ==
250-
&static_cast<EnvoyQuicClientConnection::EnvoyQuicPathValidationContext*>(
251-
GetPathValidationContext())
252-
->probingSocket());
337+
(writer_factory_
338+
? &static_cast<EnvoyQuicPathValidationContext*>(GetPathValidationContext())
339+
->probingSocket()
340+
: &static_cast<EnvoyQuicClientPathValidationContext*>(GetPathValidationContext())
341+
->probingSocket()));
253342

254343
// It's possible for a write event callback to close the connection, in such case ignore read
255344
// event processing.
@@ -331,8 +420,19 @@ void EnvoyQuicClientConnection::OnCanWrite() {
331420
onWriteEventDone();
332421
}
333422

423+
EnvoyQuicClientConnection::EnvoyQuicMigrationHelper&
424+
EnvoyQuicClientConnection::getOrCreateMigrationHelper(
425+
QuicClientPacketWriterFactory& writer_factory,
426+
OptRef<EnvoyQuicNetworkObserverRegistry> registry) {
427+
if (migration_helper_ == nullptr) {
428+
migration_helper_ = std::make_unique<EnvoyQuicMigrationHelper>(*this, registry, writer_factory);
429+
}
430+
return *migration_helper_;
431+
}
432+
334433
void EnvoyQuicClientConnection::probeAndMigrateToServerPreferredAddress(
335434
const quic::QuicSocketAddress& server_preferred_address) {
435+
ASSERT(migration_helper_ == nullptr);
336436
probeWithNewPort(server_preferred_address,
337437
quic::PathValidationReason::kServerPreferredAddressMigration);
338438
}

0 commit comments

Comments
 (0)