feat: Add stackable-telemetry utility crate (#758)

* chore: Add initial crate structure

* feat: Add initial CRD structs

* feat: Add exporter config to sub keys

* feat: Add initial trace layer and service

* feat: Finish initial trace layer

* feat: Add doc comments, implement server_socket_address

* refactor: Rename observability struct

* refactor: Use raw identifier function

* chore: Add various notes and todo comments

* Rename crate to stackable-telemetry

* Swap out trace layer with our own

* Add header injector, pre-register span fields

* chore: Add comment about constant span field names

* feat: Add header extractor

* chore: Add comments about HTTP requeste header fields

* docs: Add doc comments

* test: Add trace layer unit test

* feat: Add tracing instrumentation

* docs: Add comment about implementation guide

* add tracing subscriber (otlp exporter) initialisation

* refactor(stackable-telemetry): add typestate builder, and initialise configured subscribers.

* improve tracing docs

* docs(stackable-telemetry): improve crate docs

* docs(stackable-telemetry): improve layer docs

* Add missing doc comments

* Enable previously ignored doc tests

* Adjust cargo-udeps ignore list for doc tests

* Move modules, add re-exports

* Add doc comment about injector and extractor

* Fix doc test

* chore(stackable-telemetry): re-export LevelFilter

* Revert "chore(stackable-telemetry): re-export LevelFilter"

This reverts commit ec1b909.

* Use new module structure

* Revert "Use new module structure"

This reverts commit 5769544.

* chore(stackable-certs): skip the self field in spans

* chore(stackable-certs): skip the self field in spans

* chore(stackable-telemetry): add comment about log::Record to tracing::Event conversion

* perf: make rsa fast again

* Revert "perf: make rsa fast again"

This reverts commit 9a30a02.

* chore(stackable-webhook): adjust the severity of a failed tls handshake event

* fix(stackable-telemetry): Inject the traceparent at span finalization

* feat(stackable-webhook): Add default health endpoint

* Update crates/stackable-telemetry/src/tracing.rs

Co-authored-by: Malte Sander <[email protected]>

* Apply suggestions

This batch of changes includes various typos.

Co-authored-by: Andrew Kenworthy <[email protected]>
Co-authored-by: Malte Sander <[email protected]>

* docs(stackable-telemetry): Add doc comments for CRD structs

* chore(stackable-telemetry): Remove comment prefix

* chore: Remove CRD structs

Remove the OpenTelemetry related CRD structs for
now, because we currently don't fully understand
all requirements. The current work will be saved
in a separate branch for future reference.

* chore(stackable-telemetry): remove tracing-log, it only appears to work when called from the application code, not in a library

---------

Co-authored-by: Nick Larsen <[email protected]>
Co-authored-by: Malte Sander <[email protected]>
Co-authored-by: Andrew Kenworthy <[email protected]>

Author: 4 people

Cargo.toml

@@ -33,16 +33,20 @@ kube = { version = "0.89.0", default-features = false, features = ["client", "js
 lazy_static = "1.4.0"
 opentelemetry = "0.22.0"
 opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"] }
+opentelemetry-appender-tracing = "0.3.0"
 opentelemetry-jaeger = { version = "0.21.0", features = ["rt-tokio"] }
+opentelemetry-otlp = "0.15.0"
+opentelemetry-semantic-conventions = "0.14.0"
 p256 = { version = "0.13.2", features = ["ecdsa"] }
+pin-project = "1.1.5"
 proc-macro2 = "1.0.79"
 quote = "1.0.35"
 rand = "0.8.5"
 rand_core = "0.6.4"
 regex = "1.10.4"
 rsa = { version = "0.9.6", features = ["sha2"] }
 rstest = "0.18.2"
-schemars = "0.8.16"
+schemars = { version = "0.8.16", features = ["url"] }
 semver = "1.0.22"
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.115"
@@ -63,7 +67,7 @@ tower-http = { version = "0.5.2", features = ["trace"] }
 tracing = "0.1.40"
 tracing-opentelemetry = "0.23.0"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
-url = "2.5.0"
+url = { version = "2.5.0", features = ["serde"] }
 x509-cert = { version = "0.2.5", features = ["builder"] }
 zeroize = "1.7.0"
 

crates/stackable-certs/src/ca/mod.rs

@@ -270,7 +270,7 @@ where
     /// It is also possible to directly greate RSA or ECDSA-based leaf
     /// certificates using [`CertificateAuthority::generate_rsa_leaf_certificate`]
     /// and [`CertificateAuthority::generate_ecdsa_leaf_certificate`].
-    #[instrument(skip(key_pair))]
+    #[instrument(skip(self, key_pair))]
     pub fn generate_leaf_certificate<T>(
         &mut self,
         key_pair: T,
@@ -343,7 +343,7 @@ where
     ///
     /// See [`CertificateAuthority::generate_leaf_certificate`] for more
     /// information.
-    #[instrument]
+    #[instrument(skip(self))]
     pub fn generate_rsa_leaf_certificate(
         &mut self,
         name: &str,
@@ -358,7 +358,7 @@ where
     ///
     /// See [`CertificateAuthority::generate_leaf_certificate`] for more
     /// information.
-    #[instrument]
+    #[instrument(skip(self))]
     pub fn generate_ecdsa_leaf_certificate(
         &mut self,
         name: &str,

crates/stackable-telemetry/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "stackable-telemetry"
+version = "0.1.0"
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+repository.workspace = true
+
+[dependencies]
+axum.workspace = true
+futures-util.workspace = true
+opentelemetry = { workspace = true, features = ["logs"] }
+opentelemetry-appender-tracing.workspace = true
+opentelemetry-otlp = { workspace = true, features = ["gzip-tonic", "logs"] }
+opentelemetry-semantic-conventions.workspace = true
+opentelemetry_sdk = { workspace = true, features = ["logs", "rt-tokio"] }
+pin-project.workspace = true
+snafu.workspace = true
+tokio.workspace = true
+tower.workspace = true
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+
+[dev-dependencies]
+tokio.workspace = true
+tracing-opentelemetry.workspace = true
+stackable-webhook = { path = "../stackable-webhook" }
+
+[package.metadata.cargo-udeps.ignore]
+# Required for doc tests in stackable-telemetry
+development = ["stackable-webhook"]

crates/stackable-telemetry/src/instrumentation/axum/extractor.rs

@@ -0,0 +1,45 @@
+use axum::http::{HeaderMap, HeaderName};
+use opentelemetry::{propagation::Extractor, Context};
+
+/// Extracts the [`TextMapPropagator`][1] to access trace parent information in
+/// HTTP headers.
+///
+/// This propagation is useful when an HTTP request already has a trace parent
+/// which can be picked up by the Tower [`Layer`][2] to link both spans together.
+/// A concrete usage example is available in [`SpanExt::from_request`][3].
+///
+/// This is pretty much a copy-pasted version of the [`HeaderExtractor`][4] from
+/// the `opentelemetry_http` crate. However, we cannot use this crate, as it
+/// uses an outdated version of the underlying `http` crate.
+///
+/// [1]: opentelemetry::propagation::TextMapPropagator
+/// [2]: tower::Layer
+/// [3]: crate::instrumentation::axum::SpanExt::from_request
+/// [4]: https://docs.rs/opentelemetry-http/latest/opentelemetry_http/struct.HeaderExtractor.html
+pub struct HeaderExtractor<'a>(pub(crate) &'a HeaderMap);
+
+impl<'a> Extractor for HeaderExtractor<'a> {
+    fn get(&self, key: &str) -> Option<&str> {
+        self.0
+            .get(key)
+            .and_then(|header_value| header_value.to_str().ok())
+    }
+
+    fn keys(&self) -> Vec<&str> {
+        self.0.keys().map(HeaderName::as_str).collect()
+    }
+}
+
+impl<'a> HeaderExtractor<'a> {
+    /// Create a new header extractor from a reference to a [`HeaderMap`].
+    pub fn new(headers: &'a HeaderMap) -> Self {
+        Self(headers)
+    }
+
+    /// Extracts the [`TextMapPropagator`][1] from the HTTP headers.
+    ///
+    /// [1]: opentelemetry::propagation::TextMapPropagator
+    pub fn extract_context(&self) -> Context {
+        opentelemetry::global::get_text_map_propagator(|propagator| propagator.extract(self))
+    }
+}

crates/stackable-telemetry/src/instrumentation/axum/injector.rs

@@ -0,0 +1,47 @@
+use axum::http::{HeaderMap, HeaderName, HeaderValue};
+use opentelemetry::{propagation::Injector, Context};
+
+/// Injects the [`TextMapPropagator`][1] to propagate trace parent information
+/// in HTTP headers.
+///
+/// This propagation is useful when consumers of the HTTP response want to link
+/// up their span data with the data produced in the Tower [`Layer`][2].
+/// A concrete usage example is available in the [`TraceService::call`][3]
+/// implementation for [`TraceService`][4].
+///
+/// This is pretty much a copy-pasted version of the [`HeaderInjector`][5] from
+/// the `opentelemetry_http` crate. However, we cannot use this crate, as it
+/// uses an outdated version of the underlying `http` crate.
+///
+/// [1]: opentelemetry::propagation::TextMapPropagator
+/// [2]: tower::Layer
+/// [3]: tower::Service::call
+/// [4]: crate::instrumentation::axum::TraceService
+/// [5]: https://docs.rs/opentelemetry-http/latest/opentelemetry_http/struct.HeaderInjector.html
+pub struct HeaderInjector<'a>(pub(crate) &'a mut HeaderMap);
+
+impl<'a> Injector for HeaderInjector<'a> {
+    fn set(&mut self, key: &str, value: String) {
+        if let Ok(header_name) = HeaderName::from_bytes(key.as_bytes()) {
+            if let Ok(header_value) = HeaderValue::from_str(&value) {
+                self.0.insert(header_name, header_value);
+            }
+        }
+    }
+}
+
+impl<'a> HeaderInjector<'a> {
+    /// Create a new header injecttor from a mutable reference to [`HeaderMap`].
+    pub fn new(headers: &'a mut HeaderMap) -> Self {
+        Self(headers)
+    }
+
+    /// Inject the [`TextMapPropagator`][1] into the HTTP headers.
+    ///
+    /// [1]: opentelemetry::propagation::TextMapPropagator
+    pub fn inject_context(&mut self, cx: &Context) {
+        opentelemetry::global::get_text_map_propagator(|propagator| {
+            propagator.inject_context(cx, self)
+        })
+    }
+}