internal: Spilt app from linkerd2-proxy (#375)

* internal: internal: Spilt app from linkerd2-proxy

The `linkerd2-proxy crate currently comprises the entirety of the
application logic for the proxy. This unfortunately leads to exceedingly
high compile times (35+ minutes to compile the application with tests).

Specifically:
* Any change to the inbound or outbound proxy configuration necessitated
  recompiling the other; and this compilation could not be parallelized.
* Integration tests depended on the `linkerd2-proxy` executable, adding
  about 10 minutes to every build.
* The tests/support module (which is also extremely costly to build) was
  compiled _for each integration test_.

This change restructures the crates in this repository to allow `cargo`
to cache intermediate code that was otherwise being compiled
redundantly or serially:

* The `linkerd2-proxy` crate now contains _only_ the executable and need
  not be built during tests.
* The `linkerd2-app` crate exposes the app's `Main`, but uses
  `linkerd2-app-inbound` and `linkerd2-app-outbound` subcrates to
  improve parellization/cacheability.
* The rest of the  top-level application code
* The `linkerd2-app-integration` crate now contains all of the
  integration test support code (as well as the tests themselves), so
  that the tests only need to compile the support library once.

All in all, this reduces compile time to under 20 minutes.

Author: olix0r

Cargo.lock

@@ -1,142 +1,40 @@
 [workspace]
 members = [
-    ".",
-    "lib/hyper-balance",
-    "lib/linkerd2-addr",
-    "lib/linkerd2-conditional",
-    "lib/linkerd2-dns-name",
-    "lib/linkerd2-drain",
-    "lib/linkerd2-error",
-    "lib/linkerd2-exp-backoff",
-    "lib/linkerd2-fallback",
-    "lib/linkerd2-identity",
-    "lib/linkerd2-metrics",
-    "lib/linkerd2-opencensus",
-    "lib/linkerd2-proxy-api-resolve",
-    "lib/linkerd2-proxy-core",
-    "lib/linkerd2-proxy-discover",
-    "lib/linkerd2-proxy-resolve",
-    "lib/linkerd2-proxy-transport",
-    "lib/linkerd2-request-filter",
-    "lib/linkerd2-reconnect",
-    "lib/linkerd2-router",
-    "lib/linkerd2-signal",
-    "lib/linkerd2-stack",
-    "lib/linkerd2-task",
-    "lib/linkerd2-timeout",
-    "lib/opencensus-proto",
+    "hyper-balance",
+    "linkerd/addr",
+    "linkerd/app/core",
+    "linkerd/app/inbound",
+    "linkerd/app/integration",
+    "linkerd/app/outbound",
+    "linkerd/app",
+    "linkerd/conditional",
+    "linkerd/dns/name",
+    "linkerd/dns",
+    "linkerd/drain",
+    "linkerd/duplex",
+    "linkerd/error",
+    "linkerd/exp-backoff",
+    "linkerd/fallback",
+    "linkerd/identity",
+    "linkerd/metrics",
+    "linkerd/opencensus",
+    "linkerd/proxy/api-resolve",
+    "linkerd/proxy/core",
+    "linkerd/proxy/discover",
+    "linkerd/proxy/http",
+    "linkerd/proxy/resolve",
+    "linkerd/proxy/transport",
+    "linkerd/request-filter",
+    "linkerd/reconnect",
+    "linkerd/router",
+    "linkerd/signal",
+    "linkerd/stack",
+    "linkerd/task",
+    "linkerd/timeout",
+    "linkerd2-proxy",
+    "opencensus-proto",
 ]
 
-[package]
-name = "linkerd2-proxy"
-version = "0.1.0"
-authors = ["Linkerd Developers <[email protected]>"]
-edition = "2018"
-publish = false
-
-[features]
-default = ["flaky_tests"]
-# Disable to skip certain tests that should not be run on CI.
-flaky_tests = []
-
-[dependencies]
-hyper-balance = { path = "lib/hyper-balance" }
-linkerd2-addr        = { path = "lib/linkerd2-addr" }
-linkerd2-conditional = { path = "lib/linkerd2-conditional" }
-linkerd2-dns-name    = { path = "lib/linkerd2-dns-name" }
-linkerd2-drain       = { path = "lib/linkerd2-drain" }
-linkerd2-error       = { path = "lib/linkerd2-error" }
-linkerd2-exp-backoff = { path = "lib/linkerd2-exp-backoff" }
-linkerd2-fallback    = { path = "lib/linkerd2-fallback" }
-linkerd2-identity    = { path = "lib/linkerd2-identity" }
-linkerd2-metrics     = { path = "lib/linkerd2-metrics" }
-linkerd2-opencensus  = { path = "lib/linkerd2-opencensus" }
-linkerd2-proxy-core  = { path = "lib/linkerd2-proxy-core" }
-linkerd2-proxy-api-resolve = { path = "lib/linkerd2-proxy-api-resolve" }
-linkerd2-proxy-discover    = { path = "lib/linkerd2-proxy-discover" }
-linkerd2-proxy-resolve     = { path = "lib/linkerd2-proxy-resolve" }
-linkerd2-proxy-transport   = { path = "lib/linkerd2-proxy-transport" }
-linkerd2-reconnect         = { path = "lib/linkerd2-reconnect" }
-linkerd2-request-filter    = { path = "lib/linkerd2-request-filter" }
-linkerd2-router  = { path = "lib/linkerd2-router" }
-linkerd2-signal  = { path = "lib/linkerd2-signal" }
-linkerd2-stack   = { path = "lib/linkerd2-stack" }
-linkerd2-task    = { path = "lib/linkerd2-task" }
-linkerd2-timeout = { path = "lib/linkerd2-timeout" }
-linkerd2-trace-context = { path = "lib/linkerd2-trace-context" }
-opencensus-proto       = { path = "lib/opencensus-proto" }
-
-linkerd2-proxy-api = { git = "https://github.com/linkerd/linkerd2-proxy-api", tag = "v0.1.10" }
-
-bytes = "0.4"
-futures = "0.1"
-h2 = "0.1.15"
-http = "0.1"
-http-body = "0.1"
-httparse = "1.2"
-hyper = "0.12.3"
-ipnet = "1.0"
-log = { version = "0.4.1", features = ["std"] }
-indexmap = "1.0.0"
-prost = "0.5.0"
-prost-types = "0.5.0"
-rand = "0.6.3"
-try-lock = "0.2"
-
-# for config parsing
-regex = "1.0.0"
-
-# networking
-tokio = "0.1.14"
-tokio-timer = "0.2.6"   # for tokio_timer::clock
-tower = "0.1"
-tower-discover = "0.1"
-tower-service = "0.2"
-tower-util = "0.1"
-tokio-connect          = { git = "https://github.com/carllerche/tokio-connect" }
-tower-balance          = { git = "https://github.com/tower-rs/tower" }
-tower-load             = { git = "https://github.com/tower-rs/tower" }
-tower-request-modifier = { git = "https://github.com/tower-rs/tower-http" }
-tower-spawn-ready      = { git = "https://github.com/tower-rs/tower" }
-tower-grpc             = { version = "0.1", default-features = false, features = ["protobuf"] }
-
-# FIXME update to a release when available (>0.11)
-trust-dns-resolver = { git = "https://github.com/bluejekyll/trust-dns", rev = "7c8a0739dad495bf5a4fddfe86b8bbe2aa52d060", default-features = false }
-
-# tracing
-tracing            = "0.1.9"
-tracing-futures    = "0.1"
-tracing-log        = "0.1"
-
-# tls
-ring = "0.16"
-webpki = "0.21"
-rustls = "0.16"
-tokio-rustls = "0.10"
-untrusted = "0.7"
-
-[dependencies.tracing-subscriber]
-version = "0.1.4"
-# we don't need ANSI colors or `chrono` time formatting
-default-features = false
-features = ["env-filter", "fmt", "smallvec", "tracing-log"]
-
-[target.'cfg(target_os = "linux")'.dependencies]
-libc = "0.2"
-procinfo = "0.4.2"
-
-[dev-dependencies]
-net2 = "0.2"
-quickcheck = { version = "0.8", default-features = false }
-linkerd2-metrics = { path = "./lib/linkerd2-metrics", features = ["test_util"] }
-linkerd2-task    = { path = "lib/linkerd2-task", features = ["test_util"] }
-linkerd2-proxy-api = { git = "https://github.com/linkerd/linkerd2-proxy-api", features = ["arbitrary"], tag = "v0.1.10" }
-flate2 = { version = "1.0.1", default-features = false, features = ["rust_backend"] }
-# `tokio-io` is needed for TCP tests, because `tokio::io` doesn't re-export
-# the `read` function.
-tokio-io = "0.1.6"
-tokio-current-thread = "0.1.4"
-
 # Debug symbols end up chewing up several GB of disk space, so better to just
 # disable them.
 [profile.dev]
@@ -146,4 +44,3 @@ debug = false
 
 [patch.crates-io]
 webpki = { git = "https://github.com/seanmonstar/webpki", branch = "cert-dns-names-0.21" }
-

Cargo.toml

@@ -16,17 +16,12 @@ FROM $RUST_IMAGE as build
 
 WORKDIR /usr/src/linkerd2-proxy
 
-# Fetch external dependencies.
-RUN mkdir -p src && touch src/lib.rs
-COPY Cargo.toml Cargo.lock ./
-COPY lib lib
+COPY . .
 RUN cargo fetch --locked
-
-COPY src src
 RUN if [ -n "$PROXY_UNOPTIMIZED" ]; \
     then \
     cargo build -p linkerd2-proxy --bin linkerd2-proxy --frozen && \
-    mv target/debug/linkerd1-proxy target/linkerd2-proxy ; \
+    mv target/debug/linkerd2-proxy target/linkerd2-proxy ; \
     else \
     cargo build -p linkerd2-proxy --bin linkerd2-proxy --frozen --release && \
     mv target/release/linkerd2-proxy target/linkerd2-proxy ; \

Dockerfile

@@ -18,7 +18,7 @@ SHASUM = shasum -a 256
 
 CARGO ?= cargo
 CARGO_BUILD = $(CARGO) build --frozen $(RELEASE)
-CARGO_TEST = $(CARGO) test --all --frozen $(RELEASE)
+CARGO_TEST = $(CARGO) test --frozen $(RELEASE)
 CARGO_FMT = $(CARGO) fmt --all
 
 DOCKER = docker
@@ -74,18 +74,18 @@ fmt:
 
 .PHONY: test-lib
 test-lib:: fetch
-	$(CARGO_TEST) --lib --no-default-features
+	$(CARGO_TEST) --lib
 
 .PHONY: test-integration
 test-integration: fetch
-	$(CARGO_TEST) --tests --no-default-features
+	$(CARGO_TEST) --tests
 
 .PHONY: test
 test: test-lib test-integration
 
 .PHONY: test-flakey
 test-flakey: fetch
-	$(CARGO_TEST)
+	$(CARGO_TEST) --features linkerd-app-integration/flaky_tests
 
 .PHONY: package
 package: $(PKG_ROOT)/$(PKG)

Makefile

@@ -52,6 +52,21 @@ Usually, [Cargo][cargo], Rust's package manager, is used to build and test this
 project. If you don't have Cargo installed, we suggest getting it via
 https://rustup.rs/.
 
+### Repository Structure
+
+This project is broken into many small libraries, or _crates_, so that
+components may be compiled & tested independently. The following crate
+targets are especially important:
+
+* [`linkerd2-proxy`] contains the proxy executable;
+* [`linkerd2-app-integration`] contains the proxy's integration tests;
+* [`linkerd2-app`] bundles the [`linkerd2-app-inbound`] and [`linkerd2-app-outbound`] crates so that they may be run by the executable or integration tests.
+
+[`linkerd2-proxy`]: linkerd2-proxy
+[`linkerd2-app`]: linkerd/app
+[`linkerd2-app-integration`]: linkerd/app/integration
+[`linkerd2-app-inbound`]: linkerd/app/inbound
+[`linkerd2-app-outbound`]: linkerd/app/outbound
 
 ## Code of conduct
 

README.md

@@ -7,4 +7,4 @@ publish = false
 
 [dependencies]
 http = "0.1"
-linkerd2-dns-name = { path = "../linkerd2-dns-name" }
+linkerd2-dns-name = { path = "../dns/name" }