trace: tweak tracing & test support for the multithreaded runtime (#616)

This branch makes a couple minor tweaks to improve tracing with the
multithreaded Tokio runtime. 

It enables the option to print thread IDs when logging trace output,
which should make it somewhat clearer when events occur on different
threads.

Additionally, there's currently an issue where traces are not logged
from the proxy during integration tests. Only traces from the test
support code are logged. This is because we are using `tracing`'s scoped
thread-local dispatcher in the test proxy, so that each test can create
its own subscriber. We set the test's subscriber as the default for the
test proxy thread, but *not* as the global default for all threads.
However, when the multithreaded runtime is enabled, the default runtime
returned by `tokio::runtime::Runtime::new` is now a multithreaded
runtime. Therefore, the test proxy runs on the runtime's worker threads,
rather than the main thread spawned by the tests, and the dispatcher is
not set for the worker threads. 

This branch fixes that issue by explicitly using the basic (single
threaded) scheduler when creating the test proxy. This should
(hopefully) also improve test flakiness a bit by not spawning a whole
bunch of workers.

Author: hawkw

linkerd/app/core/src/trace.rs

@@ -63,7 +63,9 @@ pub fn with_filter_and_format(
     // Set up the subscriber
     let start_time = clock::now();
     let filter = tracing_subscriber::EnvFilter::new(filter);
-
+    let formatter = tracing_subscriber::fmt::format()
+        .with_timer(Uptime { start_time })
+        .with_thread_ids(true);
     let (dispatch, level, tasks) = match format.as_ref().to_uppercase().as_ref() {
         "JSON" => {
             let (tasks, tasks_layer) = TasksLayer::<format::JsonFields>::new();
@@ -72,7 +74,7 @@ pub fn with_filter_and_format(
                 .with(tasks_layer)
                 .with(
                     tracing_subscriber::fmt::layer()
-                        .with_timer(Uptime { start_time })
+                        .event_format(formatter)
                         .json()
                         .with_span_list(true),
                 )
@@ -86,7 +88,7 @@ pub fn with_filter_and_format(
             let (filter, level) = tracing_subscriber::reload::Layer::new(filter);
             let dispatch = tracing_subscriber::registry()
                 .with(tasks_layer)
-                .with(tracing_subscriber::fmt::layer().with_timer(Uptime { start_time }))
+                .with(tracing_subscriber::fmt::layer().event_format(formatter))
                 .with(filter)
                 .into();
             let handle = LevelHandle::Plain(level);

linkerd/app/integration/src/proxy.rs

@@ -310,7 +310,10 @@ async fn run(proxy: Proxy, mut env: TestEnv, random_ports: bool) -> Listening {
                 let span = info_span!("proxy", test = %thread_name());
                 let _enter = span.enter();
 
-                tokio::runtime::Runtime::new()
+                tokio::runtime::Builder::new()
+                    .enable_all()
+                    .basic_scheduler()
+                    .build()
                     .expect("proxy")
                     .block_on(async move {
                         let main = config.build(trace_handle).await.expect("config");