Add tracing spans to controller clients (#1288)

It can be difficult to identify when log messages are emitted from
controller clients. This change adds tracing spans to identify the
client and its endpoints.

Author: olix0r

linkerd/app/core/src/control.rs

@@ -36,17 +36,23 @@ impl fmt::Display for ControlAddr {
 type BalanceBody =
     http::balance::PendingUntilFirstDataBody<tower::load::peak_ewma::Handle, hyper::Body>;
 
-type RspBody = linkerd_http_metrics::requests::ResponseBody<BalanceBody, classify::Eos>;
-
-pub type Client = svc::Buffer<http::Request<tonic::body::BoxBody>, http::Response<RspBody>, Error>;
+pub type RspBody = linkerd_http_metrics::requests::ResponseBody<BalanceBody, classify::Eos>;
 
 impl Config {
     pub fn build<L>(
         self,
         dns: dns::Resolver,
         metrics: metrics::ControlHttp,
         identity: Option<L>,
-    ) -> svc::ArcNewService<(), Client>
+    ) -> svc::ArcNewService<
+        (),
+        impl svc::Service<
+                http::Request<tonic::body::BoxBody>,
+                Response = http::Response<RspBody>,
+                Error = Error,
+                Future = impl Send,
+            > + Clone,
+    >
     where
         L: Clone + svc::Param<tls::client::Config> + Send + Sync + 'static,
     {
@@ -87,12 +93,14 @@ impl Config {
             // Ensure individual endpoints are driven to readiness so that the balancer need not
             // drive them all directly.
             .push_on_service(svc::layer::mk(svc::SpawnReady::new))
+            .instrument(|t: &self::client::Target| tracing::info_span!("endpoint", addr = %t.addr))
             .push(self::resolve::layer(dns, resolve_backoff))
             .push_on_service(self::control::balance::layer())
             .into_new_service()
             .push(metrics.to_layer::<classify::Response, _, _>())
             .push(self::add_origin::layer())
             .push_on_service(svc::layers().push_spawn_buffer(self.buffer_capacity))
+            .instrument(|c: &ControlAddr| tracing::info_span!("controller", addr = %c.addr))
             .push_map_target(move |()| addr.clone())
             .push(svc::ArcNewService::layer())
             .into_inner()
@@ -238,7 +246,7 @@ mod client {
 
     #[derive(Clone, Hash, Debug, Eq, PartialEq)]
     pub struct Target {
-        addr: SocketAddr,
+        pub(super) addr: SocketAddr,
         server_id: tls::ConditionalClientTls,
     }
 

linkerd/app/src/dst.rs

@@ -3,8 +3,8 @@ use linkerd_app_core::{
     exp_backoff::{ExponentialBackoff, ExponentialBackoffStream},
     metrics,
     profiles::{self, DiscoveryRejected},
-    proxy::{api_resolve as api, identity::LocalCrtKey, resolve::recover},
-    svc::NewService,
+    proxy::{api_resolve as api, http, identity::LocalCrtKey, resolve::recover},
+    svc::{self, NewService},
     Error, Recover,
 };
 
@@ -15,15 +15,15 @@ pub struct Config {
 }
 
 /// Handles to destination service clients.
-pub struct Dst {
+pub struct Dst<S> {
     /// The address of the destination service, used for logging.
     pub addr: control::ControlAddr,
 
     /// Resolves profiles.
-    pub profiles: profiles::Client<BackoffUnlessInvalidArgument, control::Client>,
+    pub profiles: profiles::Client<BackoffUnlessInvalidArgument, S>,
 
     /// Resolves endpoints.
-    pub resolve: recover::Resolve<BackoffUnlessInvalidArgument, api::Resolve<control::Client>>,
+    pub resolve: recover::Resolve<BackoffUnlessInvalidArgument, api::Resolve<S>>,
 }
 
 #[derive(Copy, Clone, Debug, Default)]
@@ -37,7 +37,17 @@ impl Config {
         dns: dns::Resolver,
         metrics: metrics::ControlHttp,
         identity: Option<LocalCrtKey>,
-    ) -> Result<Dst, Error> {
+    ) -> Result<
+        Dst<
+            impl svc::Service<
+                    http::Request<tonic::body::BoxBody>,
+                    Response = http::Response<control::RspBody>,
+                    Error = Error,
+                    Future = impl Send,
+                > + Clone,
+        >,
+        Error,
+    > {
         let addr = self.control.addr.clone();
         let backoff = BackoffUnlessInvalidArgument(self.control.connect.backoff);
         let svc = self.control.build(dns, metrics, identity).new_service(());