outbound: synthesize client policies on Unimplemented (#2396)

If the policy controller is from a Linkerd version earlier than 2.13.x,
it will return the `Unimplemented` gRPC status code for requests to the
`OutboundPolicies` API. The proxy's outbound policy client will
currently retry this error code, rather than synthesizing a default
policy. Since 2.13.x proxies require an `OutboundPolicy` to be
discovered before handling outbound traffic, this means that 2.13.x
proxies cannot handle outbound connections when the control plane
is on an earlier version. Therefore, installing Linkerd 2.13 and then
downgrading to 2.12 can potentially break the data plane's ability to
route traffic.

In order to support downgrade scenarios, the proxy should also
synthesize a default policy when receiving an `Unimplemented` gRPC
status code from the policy controller. This branch changes the proxy to
do that. A warning is logged which indicates that the control plane
version is older than the proxy's.

Author: hawkw

linkerd/app/core/src/errors.rs

@@ -9,3 +9,12 @@ pub use tonic::Code as Grpc;
 #[derive(Debug, thiserror::Error)]
 #[error("connect timed out after {0:?}")]
 pub struct ConnectTimeout(pub(crate) std::time::Duration);
+
+/// Returns `true` if `error` was caused by a gRPC error with the provided
+/// status code.
+#[inline]
+pub fn has_grpc_status(error: &crate::Error, code: tonic::Code) -> bool {
+    cause_ref::<tonic::Status>(error.as_ref())
+        .map(|s| s.code() == code)
+        .unwrap_or(false)
+}

linkerd/app/gateway/src/discover.rs

@@ -23,13 +23,6 @@ impl Gateway {
         Error = Error,
         Future = impl Send + Unpin,
     > + Clone {
-        #[inline]
-        fn is_not_found(e: &Error) -> bool {
-            errors::cause_ref::<tonic::Status>(e.as_ref())
-                .map(|s| s.code() == tonic::Code::NotFound)
-                .unwrap_or(false)
-        }
-
         use futures::future;
 
         let allowlist = self.config.allow_discovery.clone();
@@ -71,7 +64,14 @@ impl Gateway {
                     // names that are not Services, so we will get a `NotFound`
                     // error if we looked up a pod DNS name. In this case, we
                     // will synthesize a default policy.
-                    Err(error) if is_not_found(&error) => tracing::debug!("Policy not found"),
+                    Err(error) if errors::has_grpc_status(&error, tonic::Code::NotFound) =>
+                        tracing::debug!("Policy not found"),
+                    // Earlier versions of the Linkerd control plane (e.g.
+                    // 2.12.x) will return `Unimplemented` for requests to the
+                    // OutboundPolicy API. Log a warning and synthesize a policy
+                    // for backwards compatibility.
+                    Err(error) if errors::has_grpc_status(&error, tonic::Code::Unimplemented) =>
+                        tracing::warn!("Policy controller returned `Unimplemented`, the control plane may be out of date."),
                     Err(error) => return Err(error),
                 }
 

linkerd/app/outbound/src/discover.rs

@@ -108,7 +108,13 @@ impl<N> Outbound<N> {
                     // XXX(ver) The policy controller may (for the time being) reject
                     // our lookups, since it doesn't yet serve endpoint metadata for
                     // forwarding.
-                    Err(error) if is_not_found(&error) => tracing::debug!("Policy not found"),
+                    Err(error) if errors::has_grpc_status(&error, tonic::Code::NotFound) => tracing::debug!("Policy not found"),
+                    // Earlier versions of the Linkerd control plane (e.g.
+                    // 2.12.x) will return `Unimplemented` for requests to the
+                    // OutboundPolicy API. Log a warning and synthesize a policy
+                    // for backwards compatibility.
+                    Err(error) if errors::has_grpc_status(&error, tonic::Code::Unimplemented) =>
+                        tracing::warn!("Policy controller returned `Unimplemented`, the control plane may be out of date."),
                     Err(error) => return Err(error),
                 }
 
@@ -143,13 +149,6 @@ impl<N> Outbound<N> {
     }
 }
 
-#[inline]
-fn is_not_found(e: &Error) -> bool {
-    errors::cause_ref::<tonic::Status>(e.as_ref())
-        .map(|s| s.code() == tonic::Code::NotFound)
-        .unwrap_or(false)
-}
-
 pub fn spawn_synthesized_profile_policy(
     mut profile: watch::Receiver<profiles::Profile>,
     synthesize: impl Fn(&profiles::Profile) -> policy::ClientPolicy + Send + 'static,

linkerd/app/outbound/src/policy/api.rs

@@ -112,7 +112,7 @@ impl Recover<tonic::Status> for GrpcRecover {
             // Non-retryable
             tonic::Code::InvalidArgument | tonic::Code::FailedPrecondition => Err(status),
             // Indicates no policy for this target
-            tonic::Code::NotFound => Err(status),
+            tonic::Code::NotFound | tonic::Code::Unimplemented => Err(status),
             _ => {
                 tracing::debug!(%status, "Recovering");
                 Ok(self.0.stream())