inbound: Do not cache gateway services (#549)

When the inbound caches gateway services, it eagerly obtains an
outbound service to cache. If the outbound service employs a traffic
split, this inbound service is pinned to a specific leaf, and requests
will never be routed to the other leaf.

This change moves the gateway fallback to be outside all of the inbound
caches, so that outbound splits work as intended.

Author: olix0r

linkerd/app/core/src/errors.rs

@@ -405,7 +405,7 @@ impl HttpError {
 
     pub fn gateway_loop() -> Self {
         Self {
-            message: "gateway loop detcted",
+            message: "gateway loop detected",
             http: http::StatusCode::LOOP_DETECTED,
             grpc: Code::Aborted,
             reason: Reason::GatewayLoop,

linkerd/app/gateway/src/gateway.rs

@@ -61,13 +61,14 @@ where
             } => {
                 // Check forwarded headers to see if this request has already
                 // transited through this gateway.
-                for fwd in request
+                for forwarded in request
                     .headers()
                     .get_all(http::header::FORWARDED)
                     .into_iter()
                     .filter_map(|h| h.to_str().ok())
                 {
-                    if let Some(by) = fwd_by(fwd) {
+                    if let Some(by) = fwd_by(forwarded) {
+                        tracing::info!(%forwarded);
                         if by == local_identity.as_ref() {
                             return Box::new(future::err(HttpError::gateway_loop().into()));
                         }

linkerd/app/inbound/src/lib.rs

@@ -216,13 +216,6 @@ impl Config {
             // Normalizes the URI, i.e. if it was originally in
             // absolute-form on the outbound side.
             .push(normalize_uri::layer())
-            .push(admit::AdmitLayer::new(prevent_loop))
-            .push_on_response(svc::layers().box_http_response())
-            .push_fallback_on_error::<prevent_loop::LoopPrevented, _>(
-                svc::stack(http_loopback)
-                    .push_on_response(svc::layers().box_http_request())
-                    .into_inner(),
-            )
             .push(http_target_observability)
             .check_service::<Target>()
             .into_new_service()
@@ -279,13 +272,20 @@ impl Config {
         svc::stack(http_profile_cache)
             .push_on_response(svc::layers().box_http_response())
             .push_make_ready()
-            // Don't resolve profiles for inbound-targetted requests. They'll be
-            // resolved on the outbound side if necsesary.
-            .push(admit::AdmitLayer::new(prevent_loop))
             .push_fallback(
                 http_target_cache
                     .push_on_response(svc::layers().box_http_response().box_http_request()),
             )
+            // If the traffic is targeted at the inbound port, send it through
+            // the loopback service (i.e. as a gateway). This is done before
+            // caching so that the loopback stack can determine whether it
+            // should cache or not.
+            .push(admit::AdmitLayer::new(prevent_loop))
+            .push_fallback_on_error::<prevent_loop::LoopPrevented, _>(
+                svc::stack(http_loopback)
+                    .push_on_response(svc::layers().box_http_request())
+                    .into_inner(),
+            )
             .check_service::<Target>()
             .into_inner()
     }