debug and fix telemetry init and shutdown

Author: KSDaemon

rust/cubestore/cubestore/src/bin/cubestored.rs

@@ -55,9 +55,6 @@ fn main() {
             ))
         })
         .unwrap();
-    if enable_telemetry {
-        init_tracing_telemetry();
-    }
     init_cube_logger(enable_telemetry);
 
     log::info!("Cube Store version {}", version);
@@ -83,6 +80,10 @@ fn main() {
     }
     let runtime = tokio_builder.build().unwrap();
     runtime.block_on(async move {
+        if enable_telemetry {
+            init_tracing_telemetry(version);
+        }
+        // TODO: Should this be avoided if otel is configured?
         init_agent_sender().await;
 
         validate_config(config.config_obj().as_ref()).report_and_abort_on_errors();
@@ -99,6 +100,12 @@ fn main() {
 
         stop_on_ctrl_c(&services).await;
         services.wait_processing_loops().await.unwrap();
+
+        if enable_telemetry {
+            // This still doesn't prevent errors:
+            // OpenTelemetry trace error occurred. cannot send message to batch processor as the channel is closed
+            opentelemetry::global::shutdown_tracer_provider();
+        }
     });
 }
 

rust/cubestore/cubestore/src/telemetry/otel_tracing.rs

@@ -1,18 +1,26 @@
 use log::{Log, Metadata, Record};
 use opentelemetry::trace::TracerProvider;
+use opentelemetry::KeyValue;
+use opentelemetry_sdk::Resource;
 use tracing_log::LogTracer;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::Registry;
 
 const OTEL_SERVICE_NAME: &str = "cubestore";
 
-pub fn init_tracing_telemetry() {
+pub fn init_tracing_telemetry(version: String) {
     let otlp_exporter = opentelemetry_otlp::new_exporter()
         .http()
         .with_http_client(reqwest::Client::new());
     let tracer_provider = opentelemetry_otlp::new_pipeline()
         .tracing()
         .with_exporter(otlp_exporter)
+        .with_trace_config(opentelemetry_sdk::trace::Config::default().with_resource(
+            Resource::new(vec![
+                KeyValue::new("service.name", OTEL_SERVICE_NAME),
+                KeyValue::new("service.version", version),
+            ]),
+        ))
         .install_batch(opentelemetry_sdk::runtime::Tokio)
         .expect("Should be able to initialise the tracer_provider");