fix(dns-resolve): add a lower-bound TTL for dns refreshing (#3807)

DNS servers may return extremely low TTLs in some cases. When we're polling DNS to power a load balancer, we need to enforce a minimum duration to prevent tight-looping DNS queries.

This change adds a 5s minimum time between DNS lookups when resolving control plane components.

fixes linkerd/linkerd2#13508

Author: cratelyn

linkerd/dns/src/lib.rs

@@ -85,7 +85,7 @@ impl Resolver {
         &self,
         name: NameRef<'_>,
         default_port: u16,
-    ) -> Result<(Vec<net::SocketAddr>, time::Sleep), ResolveError> {
+    ) -> Result<(Vec<net::SocketAddr>, Instant), ResolveError> {
         match self.resolve_srv(name).await {
             Ok(res) => Ok(res),
             Err(srv_error) => {
@@ -108,18 +108,18 @@ impl Resolver {
     async fn resolve_a_or_aaaa(
         &self,
         name: NameRef<'_>,
-    ) -> Result<(Vec<net::IpAddr>, time::Sleep), ARecordError> {
+    ) -> Result<(Vec<net::IpAddr>, Instant), ARecordError> {
         debug!(%name, "Resolving an A/AAAA record");
         let lookup = self.dns.lookup_ip(name.as_str()).await?;
-        let valid_until = Instant::from_std(lookup.valid_until());
         let ips = lookup.iter().collect::<Vec<_>>();
-        Ok((ips, time::sleep_until(valid_until)))
+        let valid_until = Instant::from_std(lookup.valid_until());
+        Ok((ips, valid_until))
     }
 
     async fn resolve_srv(
         &self,
         name: NameRef<'_>,
-    ) -> Result<(Vec<net::SocketAddr>, time::Sleep), SrvRecordError> {
+    ) -> Result<(Vec<net::SocketAddr>, Instant), SrvRecordError> {
         debug!(%name, "Resolving a SRV record");
         let srv = self.dns.srv_lookup(name.as_str()).await?;
 
@@ -128,9 +128,9 @@ impl Resolver {
             .into_iter()
             .map(Self::srv_to_socket_addr)
             .collect::<Result<_, InvalidSrv>>()?;
-        debug!(ttl = ?valid_until - time::Instant::now(), ?addrs);
+        debug!(ttl = ?valid_until - Instant::now(), ?addrs);
 
-        Ok((addrs, time::sleep_until(valid_until)))
+        Ok((addrs, valid_until))
     }
 
     // XXX We need to convert the SRV records to an IP addr manually,

linkerd/proxy/dns-resolve/src/lib.rs

@@ -80,7 +80,7 @@ async fn resolution(dns: dns::Resolver, na: NameAddr) -> Result<UpdateStream, Er
                 trace!("Closed");
                 return;
             }
-            expiry.await;
+            sleep_until_expired(expiry).await;
 
             loop {
                 match dns.resolve_addrs(na.name().as_ref(), na.port()).await {
@@ -91,7 +91,7 @@ async fn resolution(dns: dns::Resolver, na: NameAddr) -> Result<UpdateStream, Er
                             trace!("Closed");
                             return;
                         }
-                        expiry.await;
+                        sleep_until_expired(expiry).await;
                     }
                     Err(error) => {
                         debug!(%error);
@@ -107,3 +107,26 @@ async fn resolution(dns: dns::Resolver, na: NameAddr) -> Result<UpdateStream, Er
 
     Ok(Box::pin(ReceiverStream::new(rx)))
 }
+
+/// Sleep for the provided [`Duration`][tokio::time::Duration].
+///
+/// NB: This enforces a lower-bound for TTL's to prevent [`resolution()`], above, from spinning
+/// in a hot-loop.
+#[tracing::instrument(level = "debug")]
+async fn sleep_until_expired(valid_until: tokio::time::Instant) {
+    use tokio::time::{sleep_until, Duration, Instant};
+
+    /// The minimum TTL duration that will be respected.
+    const MINIMUM_TTL: Duration = Duration::from_secs(5);
+    let minimum = Instant::now() + MINIMUM_TTL;
+
+    // Choose a deadline; if the expiry is too short, fall back to the minimum TTL.
+    let deadline = if valid_until >= minimum {
+        valid_until
+    } else {
+        debug!(ttl.min = ?MINIMUM_TTL, "Given TTL too short, using a minimum TTL");
+        minimum
+    };
+
+    sleep_until(deadline).await;
+}