WIP: improve BBR throughput on links with very low latency

Author: haproxyFred

include/haproxy/quic_cc-t.h

@@ -91,7 +91,7 @@ struct quic_cc {
 	/* <conn> is there only for debugging purpose. */
 	struct quic_conn *qc;
 	struct quic_cc_algo *algo;
-	uint32_t priv[158];
+	uint32_t priv[164];
 };
 
 struct quic_cc_path {

include/haproxy/quic_cc_drs.h

@@ -10,10 +10,10 @@ struct quic_cc_rs {
 	uint64_t lost;
 	uint64_t prior_lost;
 	int64_t last_end_seq;
-	uint32_t interval;
-	uint32_t prior_time;
-	uint32_t send_elapsed;
-	uint32_t ack_elapsed;
+	uint64_t interval_ns;
+	uint64_t prior_time_ns;
+	uint64_t send_elapsed_ns;
+	uint64_t ack_elapsed_ns;
 	uint32_t is_app_limited;
 };
 
@@ -26,8 +26,8 @@ struct quic_cc_drs {
 	uint64_t delivered;
 	uint64_t lost;
 	int64_t last_seq;
-	uint32_t delivered_time;
-	uint32_t first_sent_time;
+	uint64_t delivered_time_ns;
+	uint64_t first_sent_time_ns;
 	int is_cwnd_limited; /* boolean */
 	int app_limited; /* boolean */
 };
@@ -36,6 +36,6 @@ void quic_cc_drs_init(struct quic_cc_drs *drs);
 void quic_cc_drs_on_pkt_sent(struct quic_cc_path *path,
                              struct quic_tx_packet *pkt, struct quic_cc_drs *drs);
 void quic_cc_drs_update_rate_sample(struct quic_cc_drs *drs,
-                                    struct quic_tx_packet *pkt);
+                                    struct quic_tx_packet *pkt, uint64_t time_ns);
 void quic_cc_drs_on_ack_recv(struct quic_cc_drs *drs, struct quic_cc_path *path,
                              uint64_t pkt_delivered);

include/haproxy/quic_tx-t.h

@@ -51,7 +51,8 @@ struct quic_tx_packet {
 	/* The list of frames of this packet. */
 	struct list frms;
 	/* The time this packet was sent (ms). */
-	unsigned int time_sent;
+	unsigned int time_sent_ms;
+	uint64_t time_sent_ns;
 	/* Packet number spakce. */
 	struct quic_pktns *pktns;
 	/* Flags. */
@@ -70,8 +71,8 @@ struct quic_tx_packet {
 		uint64_t tx_in_flight;
 		uint64_t lost;
 		int64_t end_seq;
-		uint32_t delivered_time;
-		uint32_t first_sent_time;
+		uint64_t delivered_time_ns;
+		uint64_t first_sent_time_ns;
 		int is_app_limited;
 	} rs;
 	unsigned char type;

src/quic_cc_drs.c

@@ -4,20 +4,21 @@
 #include <haproxy/quic_cc-t.h>
 #include <haproxy/quic_cc_drs.h>
 #include <haproxy/quic_tx-t.h>
+#include <haproxy/task.h>
 #include <haproxy/ticks.h>
 #include <haproxy/window_filter.h>
 
 static void quic_cc_rs_init(struct quic_cc_rs *rs)
 {
-	rs->interval = UINT32_MAX;
+	rs->interval_ns = UINT64_MAX;
 	rs->delivered = 0;
 	rs->prior_delivered = 0;
-	rs->prior_time = TICK_ETERNITY;
+	rs->prior_time_ns = 0;
 	rs->tx_in_flight = 0;
 	rs->lost = 0;
 	rs->prior_lost = 0;
-	rs->send_elapsed = 0;
-	rs->ack_elapsed = 0;
+	rs->send_elapsed_ns = 0;
+	rs->ack_elapsed_ns = 0;
 	rs->last_end_seq = -1;
 	rs->is_app_limited = 0;
 }
@@ -31,8 +32,8 @@ void quic_cc_drs_init(struct quic_cc_drs *drs)
 	drs->delivered = 0;
 	drs->lost = 0;
 	drs->last_seq = -1;
-	drs->delivered_time = TICK_ETERNITY;
-	drs->first_sent_time = TICK_ETERNITY;
+	drs->delivered_time_ns = 0;
+	drs->first_sent_time_ns = 0;
 	drs->app_limited = 0;
 	drs->is_cwnd_limited = 0;
 }
@@ -44,12 +45,12 @@ void quic_cc_drs_on_pkt_sent(struct quic_cc_path *path,
                              struct quic_tx_packet *pkt, struct quic_cc_drs *drs)
 {
 	if (!path->in_flight)
-		drs->first_sent_time  = drs->delivered_time = pkt->time_sent;
+		drs->first_sent_time_ns = drs->delivered_time_ns = pkt->time_sent_ns;
 
-	pkt->rs.first_sent_time = drs->first_sent_time;
-	pkt->rs.delivered_time  = drs->delivered_time;
-	pkt->rs.delivered       = drs->delivered;
-	pkt->rs.is_app_limited  = drs->app_limited != 0;
+	pkt->rs.first_sent_time_ns = drs->first_sent_time_ns;
+	pkt->rs.delivered_time_ns  = drs->delivered_time_ns;
+	pkt->rs.delivered          = drs->delivered;
+	pkt->rs.is_app_limited     = drs->app_limited != 0;
 
 	pkt->rs.tx_in_flight = path->in_flight + pkt->len;
 	pkt->rs.lost = drs->lost;
@@ -62,8 +63,8 @@ void quic_cc_drs_on_pkt_sent(struct quic_cc_path *path,
 static inline int quic_cc_drs_is_newest_packet(struct quic_cc_drs *drs,
                                                struct quic_tx_packet *pkt)
 {
-	return tick_is_lt(drs->first_sent_time, pkt->time_sent) ||
-		(pkt->time_sent == drs->first_sent_time &&
+	return drs->first_sent_time_ns < pkt->time_sent_ns ||
+		(pkt->time_sent_ns == drs->first_sent_time_ns &&
 		 pkt->rs.end_seq > drs->rs.last_end_seq);
 }
 
@@ -83,32 +84,32 @@ static inline int quic_cc_drs_is_newest_packet(struct quic_cc_drs *drs,
  * RFC UpdateRateSample() called from first part of GenerateRateSample().
  */
 void quic_cc_drs_update_rate_sample(struct quic_cc_drs *drs,
-                                    struct quic_tx_packet *pkt)
+                                    struct quic_tx_packet *pkt, uint64_t time_ns)
 {
 	struct quic_cc_rs *rs = &drs->rs;
 
-	if (!tick_isset(pkt->rs.delivered_time))
+	if (!pkt->rs.delivered_time_ns)
 		return;
 
 	drs->delivered += pkt->len;
-	drs->delivered_time = now_ms;
+	drs->delivered_time_ns = time_ns;
 	/* Update info using the newest packet. */
-	if (tick_isset(rs->prior_time) && !quic_cc_drs_is_newest_packet(drs, pkt))
+	if (rs->prior_time_ns && !quic_cc_drs_is_newest_packet(drs, pkt))
 		return;
 
 	rs->prior_delivered  = pkt->rs.delivered;
-	rs->prior_time       = pkt->rs.delivered_time;
+	rs->prior_time_ns    = pkt->rs.delivered_time_ns;
 	rs->is_app_limited   = pkt->rs.is_app_limited;
-	rs->send_elapsed     = pkt->time_sent - pkt->rs.first_sent_time;
-	rs->ack_elapsed      = drs->delivered_time - pkt->rs.delivered_time;
+	rs->send_elapsed_ns  = pkt->time_sent_ns - pkt->rs.first_sent_time_ns;
+	rs->ack_elapsed_ns   = drs->delivered_time_ns - pkt->rs.delivered_time_ns;
 	rs->tx_in_flight     = pkt->rs.tx_in_flight;
 	rs->prior_lost       = pkt->rs.lost;
 	rs->last_end_seq     = pkt->rs.end_seq;
-	drs->first_sent_time = pkt->time_sent;
+	drs->first_sent_time_ns = pkt->time_sent_ns;
 	/* Mark the packet as delivered once it's SACKed to
 	 * avoid being used again when it's cumulatively acked.
 	 */
-	pkt->rs.delivered_time = TICK_ETERNITY;
+	pkt->rs.delivered_time_ns = 0;
 }
 
 /* RFC https://datatracker.ietf.org/doc/draft-ietf-ccwg-bbr/
@@ -131,25 +132,25 @@ void quic_cc_drs_on_ack_recv(struct quic_cc_drs *drs, struct quic_cc_path *path,
 		++drs->round_count;
 	}
 
-	if (!tick_isset(rs->prior_time))
+	if (!rs->prior_time_ns)
 		return;
 
-	rs->interval = MAX(rs->send_elapsed, rs->ack_elapsed);
+	rs->interval_ns = MAX(rs->send_elapsed_ns, rs->ack_elapsed_ns);
 
 	BUG_ON(drs->delivered <= rs->prior_delivered);
 	rs->delivered = drs->delivered - rs->prior_delivered;
 	BUG_ON(drs->lost < rs->prior_lost);
 	rs->lost = drs->lost - rs->prior_lost;
 
-	if (rs->interval < path->loss.rtt_min) {
-		rs->interval = UINT32_MAX;
+	if (rs->interval_ns < path->loss.rtt_min * 1000000) {
+		rs->interval_ns = UINT64_MAX;
 		return;
 	}
 
-	if (!rs->interval)
+	if (!rs->interval_ns)
 		return;
 
-	rate = rs->delivered * 1000 / rs->interval;
+	rate = rs->delivered * 1000000000 / rs->interval_ns;
 	if (rate >= wf_get_max(&drs->wf) || !drs->app_limited)
 		path->delivery_rate = wf_max_update(&drs->wf, rate, drs->round_count);
 }

src/quic_conn.c

@@ -1247,6 +1247,7 @@ struct quic_conn *qc_new_conn(const struct quic_version *qv, int ipv4,
 	}
 	qc->wait_event.tasklet->process = quic_conn_io_cb;
 	qc->wait_event.tasklet->context = qc;
+	qc->wait_event.tasklet->state |= TASK_F_WANTS_TIME;
 	qc->wait_event.events = 0;
 	qc->subs = NULL;
 

src/quic_loss.c

@@ -206,7 +206,7 @@ void qc_packet_loss_lookup(struct quic_pktns *pktns, struct quic_conn *qc,
 		if ((int64_t)pkt->pn_node.key > largest_acked_pn)
 			break;
 
-		time_sent = pkt->time_sent;
+		time_sent = pkt->time_sent_ms;
 		loss_time_limit = tick_add(time_sent, loss_delay);
 
 		reordered = (int64_t)largest_acked_pn >= pkt->pn_node.key + pktthresh;
@@ -290,12 +290,12 @@ int qc_release_lost_pkts(struct quic_conn *qc, struct quic_pktns *pktns,
 			struct quic_cc_event ev = { };
 
 			ev.type = QUIC_CC_EVT_LOSS;
-			ev.loss.time_sent = newest_lost->time_sent;
+			ev.loss.time_sent = newest_lost->time_sent_ms;
 			ev.loss.count = tot_lost;
 
 			quic_cc_event(cc, &ev);
 			if (cc->algo->congestion_event)
-			    cc->algo->congestion_event(cc, newest_lost->time_sent);
+			    cc->algo->congestion_event(cc, newest_lost->time_sent_ms);
 		}
 
 		/* If an RTT have been already sampled, <rtt_min> has been set.
@@ -304,7 +304,7 @@ int qc_release_lost_pkts(struct quic_conn *qc, struct quic_pktns *pktns,
 		 * slow start state.
 		 */
 		if (qc->path->loss.rtt_min && newest_lost != oldest_lost) {
-			unsigned int period = newest_lost->time_sent - oldest_lost->time_sent;
+			unsigned int period = newest_lost->time_sent_ms - oldest_lost->time_sent_ms;
 
 			if (quic_loss_persistent_congestion(&qc->path->loss, period,
 							    now_ms, qc->max_ack_delay) &&

src/quic_rx.c

@@ -432,9 +432,13 @@ static void qc_notify_cc_of_newly_acked_pkts(struct quic_conn *qc,
 	struct quic_cc_drs *drs =
 		p->cc.algo->get_drs ? p->cc.algo->get_drs(&p->cc) : NULL;
 	unsigned int bytes_delivered = 0, pkt_delivered = 0;
+	uint64_t time_ns;
 
 	TRACE_ENTER(QUIC_EV_CONN_PRSAFRM, qc);
 
+	if (drs)
+		time_ns = task_mono_time();
+
 	list_for_each_entry_safe(pkt, tmp, newly_acked_pkts, list) {
 		pkt->pktns->tx.in_flight -= pkt->in_flight_len;
 		p->prep_in_flight -= pkt->in_flight_len;
@@ -450,12 +454,12 @@ static void qc_notify_cc_of_newly_acked_pkts(struct quic_conn *qc,
 		bytes_delivered += pkt->len;
 		pkt_delivered = pkt->rs.delivered;
 		ev.ack.acked = pkt->in_flight_len;
-		ev.ack.time_sent = pkt->time_sent;
+		ev.ack.time_sent = pkt->time_sent_ms;
 		ev.ack.pn = pkt->pn_node.key;
 		/* Note that this event is not emitted for BBR. */
 		quic_cc_event(&p->cc, &ev);
 		if (drs && (pkt->flags & QUIC_FL_TX_PACKET_ACK_ELICITING))
-			quic_cc_drs_update_rate_sample(drs, pkt);
+			quic_cc_drs_update_rate_sample(drs, pkt, time_ns);
 		LIST_DEL_INIT(&pkt->list);
 		quic_tx_packet_refdec(pkt);
 	}
@@ -523,7 +527,7 @@ static int qc_parse_ack_frm(struct quic_conn *qc,
 		}
 		else {
 			time_sent = eb64_entry(largest_node,
-			                       struct quic_tx_packet, pn_node)->time_sent;
+			                       struct quic_tx_packet, pn_node)->time_sent_ms;
 			new_largest_acked_pn = 1;
 		}
 	}

src/quic_tx.c

@@ -295,7 +295,8 @@ static int qc_send_ppkts(struct buffer *buf, struct ssl_sock_ctx *ctx)
 		struct buffer tmpbuf = { };
 		struct quic_tx_packet *first_pkt, *pkt, *next_pkt;
 		uint16_t dglen, gso = 0, gso_fallback = 0;
-		unsigned int time_sent;
+		uint64_t time_sent_ns;
+		unsigned int time_sent_ms;
 
 		pos = (unsigned char *)b_head(buf);
 		dglen = read_u16(pos);
@@ -360,7 +361,8 @@ static int qc_send_ppkts(struct buffer *buf, struct ssl_sock_ctx *ctx)
 
 		b_del(buf, dglen + QUIC_DGRAM_HEADLEN);
 		qc->bytes.tx += tmpbuf.data;
-		time_sent = now_ms;
+		time_sent_ms = now_ms;
+		time_sent_ns = task_mono_time();
 
 		for (pkt = first_pkt; pkt; pkt = next_pkt) {
 			struct quic_cc *cc = &qc->path->cc;
@@ -391,9 +393,10 @@ static int qc_send_ppkts(struct buffer *buf, struct ssl_sock_ctx *ctx)
 
 			qc->cntrs.sent_pkt++;
 
-			pkt->time_sent = time_sent;
+			pkt->time_sent_ns = time_sent_ns;
+			pkt->time_sent_ms = time_sent_ms;
 			if (pkt->flags & QUIC_FL_TX_PACKET_ACK_ELICITING) {
-				pkt->pktns->tx.time_of_last_eliciting = time_sent;
+				pkt->pktns->tx.time_of_last_eliciting = time_sent_ms;
 				qc->path->ifae_pkts++;
 				if (qc->flags & QUIC_FL_CONN_IDLE_TIMER_RESTARTED_AFTER_READ)
 					qc_idle_timer_rearm(qc, 0, 0);
@@ -2202,7 +2205,8 @@ static inline void quic_tx_packet_init(struct quic_tx_packet *pkt, int type)
 	pkt->in_flight_len = 0;
 	pkt->pn_node.key = (uint64_t)-1;
 	LIST_INIT(&pkt->frms);
-	pkt->time_sent = TICK_ETERNITY;
+	pkt->time_sent_ns = 0;
+	pkt->time_sent_ms = TICK_ETERNITY;
 	pkt->next = NULL;
 	pkt->prev = NULL;
 	pkt->largest_acked_pn = -1;