identity: add a metric tracking when the proxy's cert expires (#787)

This branch adds a new `identity_cert_expiration_timestamp_seconds` to the
proxy's Prometheus metric. When identity is enabled, this metric will
report the time at which the proxy's current certificate expires, in
seconds since the Unix epoch.

Closes linkerd/linkerd2#5379

Author: hawkw

Cargo.lock

@@ -1417,6 +1417,7 @@ dependencies = [
  "http-body",
  "linkerd2-error",
  "linkerd2-identity",
+ "linkerd2-metrics",
  "linkerd2-proxy-api",
  "linkerd2-proxy-transport",
  "pin-project 0.4.22",

linkerd/app/src/identity.rs

@@ -1,5 +1,5 @@
 pub use linkerd2_app_core::proxy::identity::{
-    certify, Crt, CrtKey, Csr, InvalidName, Key, Local, Name, TokenSource, TrustAnchors,
+    certify, metrics, Crt, CrtKey, Csr, InvalidName, Key, Local, Name, TokenSource, TrustAnchors,
 };
 use linkerd2_app_core::{
     control, dns,
@@ -80,6 +80,13 @@ impl Identity {
         }
     }
 
+    pub fn metrics(&self) -> metrics::Report {
+        match self {
+            Identity::Disabled => metrics::Report::disabled(),
+            Identity::Enabled { ref local, .. } => local.metrics(),
+        }
+    }
+
     pub fn task(self) -> Task {
         match self {
             Identity::Disabled => Box::pin(async {}),

linkerd/app/src/lib.rs

@@ -75,6 +75,8 @@ impl Config {
     /// It is currently required that this be run on a Tokio runtime, since some
     /// services are created eagerly and must spawn tasks to do so.
     pub async fn build(self, log_level: trace::Handle) -> Result<App, Error> {
+        use metrics::FmtMetrics;
+
         let Config {
             admin,
             dns,
@@ -94,6 +96,7 @@ impl Config {
 
         let identity = info_span!("identity")
             .in_scope(|| identity.build(dns.resolver.clone(), metrics.control.clone()))?;
+        let report = report.and_then(identity.metrics());
 
         let (drain_tx, drain_rx) = drain::channel();
 

linkerd/identity/src/lib.rs

@@ -331,6 +331,10 @@ impl CrtKey {
     pub fn tls_server_config(&self) -> Arc<rustls::ServerConfig> {
         self.server_config.clone()
     }
+
+    pub fn expiry(&self) -> SystemTime {
+        self.expiry
+    }
 }
 
 impl fmt::Debug for CrtKey {

linkerd/proxy/identity/Cargo.toml

@@ -9,6 +9,7 @@ publish = false
 futures = "0.3"
 linkerd2-error = { path = "../../error" }
 linkerd2-identity = { path = "../../identity" }
+linkerd2-metrics = { path = "../../metrics" }
 linkerd2-proxy-api = { git = "https://github.com/linkerd/linkerd2-proxy-api", tag = "v0.1.16" }
 linkerd2-proxy-transport = { path = "../transport" }
 tokio = { version = "0.3", features = ["time", "sync"] }

linkerd/proxy/identity/src/certify.rs

@@ -164,6 +164,10 @@ impl Local {
         }
         Ok(self)
     }
+
+    pub fn metrics(&self) -> crate::metrics::Report {
+        crate::metrics::Report::new(self.crt_key.clone())
+    }
 }
 
 impl tls::client::HasConfig for Local {

linkerd/proxy/identity/src/lib.rs

@@ -1,6 +1,7 @@
 #![deny(warnings, rust_2018_idioms)]
 
 pub mod certify;
+pub mod metrics;
 
 pub use self::certify::{AwaitCrt, CrtKeySender, Local};
 pub use linkerd2_identity::{Crt, CrtKey, Csr, InvalidName, Key, Name, TokenSource, TrustAnchors};

linkerd/proxy/identity/src/metrics.rs

@@ -0,0 +1,53 @@
+use crate::CrtKey;
+use linkerd2_metrics::{metrics, FmtMetrics, Gauge};
+use std::{fmt, time::UNIX_EPOCH};
+use tokio::sync::watch;
+
+#[derive(Debug, Clone)]
+pub struct Report {
+    crt_key_watch: Option<watch::Receiver<Option<CrtKey>>>,
+}
+
+metrics! {
+    identity_cert_expiration_timestamp_seconds: Gauge {
+        "Time when the this proxy's current mTLS identity certificate will expire (in seconds since the UNIX epoch)"
+    }
+}
+
+impl Report {
+    pub(crate) fn new(watch: watch::Receiver<Option<CrtKey>>) -> Self {
+        Self {
+            crt_key_watch: Some(watch),
+        }
+    }
+
+    pub fn disabled() -> Self {
+        Self {
+            crt_key_watch: None,
+        }
+    }
+}
+
+impl FmtMetrics for Report {
+    fn fmt_metrics(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let dur = if let Some(watch) = self.crt_key_watch.as_ref() {
+            if let Some(ref crt_key) = *(watch.borrow()) {
+                crt_key
+                .expiry()
+                .duration_since(UNIX_EPOCH)
+                .map_err(|error| {
+                    tracing::warn!(%error, "an identity would expire before the beginning of the UNIX epoch, something is probably wrong");
+                    fmt::Error
+                })?
+            } else {
+                return Ok(());
+            }
+        } else {
+            return Ok(());
+        };
+
+        identity_cert_expiration_timestamp_seconds.fmt_help(f)?;
+        identity_cert_expiration_timestamp_seconds.fmt_metric(f, &Gauge::from(dur.as_secs()))?;
+        Ok(())
+    }
+}