outbound: Emit INFO-level logs on failure accrual changes (#2373)

There are no explicit indications of failure accrual related behavior.

This change adds INFO-level logs when a failure accrual breaker is
closed or reopened. These log messages are scoped within the
'endpoint' log context so that log lines include endpoint and balancer
addresses.

    INFO balance{addr=logical.test.svc.cluster.local:666}:endpoint{addr=192.0.2.41:666}:consecutive_failures: linkerd_app_outbound::http::breaker::consecutive_failures: Consecutive failure-accrual breaker closed
    INFO balance{addr=logical.test.svc.cluster.local:666}:endpoint{addr=192.0.2.41:666}:consecutive_failures: linkerd_app_outbound::http::breaker::consecutive_failures: Consecutive failure-accrual breaker reopened

Author: olix0r

linkerd/app/outbound/src/http/breaker/consecutive_failures.rs

@@ -33,9 +33,12 @@ impl ConsecutiveFailures {
                 return;
             }
 
+            tracing::info!("Consecutive failure-accrual breaker closed");
             if self.closed().await.is_err() {
                 return;
             }
+
+            tracing::info!("Consecutive failure-accrual breaker reopened");
         }
     }
 

linkerd/app/outbound/src/http/concrete.rs

@@ -120,15 +120,6 @@ impl<N> Outbound<N> {
                 .instrument(|e: &Endpoint<T>| info_span!("forward", addr = %e.addr));
 
             let endpoint = inner
-                .push_on_service(
-                    rt.metrics
-                        .proxy
-                        .stack
-                        .layer(stack_labels("http", "endpoint")),
-                )
-                .instrument(|e: &Endpoint<T>| info_span!("endpoint", addr = %e.addr));
-
-            let balance = endpoint
                 .push_map_target({
                     let inbound_ips = inbound_ips.clone();
                     move |((addr, metadata), target): ((SocketAddr, Metadata), Balance<T>)| {
@@ -164,6 +155,17 @@ impl<N> Outbound<N> {
                         }
                     }),
                 )
+                .push_on_service(svc::OnServiceLayer::new(
+                    rt.metrics
+                        .proxy
+                        .stack
+                        .layer(stack_labels("http", "endpoint")),
+                ))
+                .push_on_service(svc::NewInstrumentLayer::new(
+                    |(addr, _): &(SocketAddr, _)| info_span!("endpoint", %addr),
+                ));
+
+            let balance = endpoint
                 .push(http::NewBalancePeakEwma::layer(resolve))
                 .push(svc::NewMapErr::layer_from_target::<ConcreteError, _>())
                 .push_on_service(http::BoxResponse::layer())