feat: single source of truth for headers (fixes issue in profiling with missing headers) (#1493)

# What does this PR do?

Unifies the headers between common and profiling

# Motivation

We had missed some when we implemented our own exporter, this makes sure the two implementations stay in sync. 
# Additional Notes

Anything else we should know when reviewing?

# How to test the change?

Describe here in detail how the change can be validated.


Co-authored-by: daniel.schwartznarbonne <[email protected]>

Author: danielsn

examples/ffi/exporter_manager.c

@@ -9,6 +9,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/wait.h>
 #include <unistd.h>
 
 static ddog_CharSlice to_slice_c_char(const char *s) { return (ddog_CharSlice){.ptr = s, .len = strlen(s)}; }

libdd-common/src/entity_id/mod.rs

@@ -99,3 +99,15 @@ pub static DD_EXTERNAL_ENV: LazyLock<Option<&'static str>> = LazyLock::new(|| {
 
     leaked
 });
+
+/// Returns an iterator of entity-related headers (container-id, entity-id, external-env)
+/// as (header_name, header_value) string tuples for any that are available.
+pub fn get_entity_headers() -> impl Iterator<Item = (&'static str, &'static str)> {
+    [
+        get_container_id().map(|v| ("datadog-container-id", v)),
+        get_entity_id().map(|v| ("datadog-entity-id", v)),
+        (*DD_EXTERNAL_ENV).map(|v| ("datadog-external-env", v)),
+    ]
+    .into_iter()
+    .flatten()
+}

libdd-common/src/lib.rs

@@ -7,7 +7,7 @@
 #![cfg_attr(not(test), deny(clippy::unimplemented))]
 
 use anyhow::Context;
-use hyper::{header::HeaderValue, http::uri};
+use hyper::http::uri;
 use serde::de::Error;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::sync::{Mutex, MutexGuard};
@@ -124,7 +124,6 @@ impl<C: hyper_util::client::legacy::connect::Connect + Clone + Send + Sync + 'st
 }
 
 // Used by tag! macro
-use crate::entity_id::DD_EXTERNAL_ENV;
 pub use const_format;
 
 #[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)]
@@ -241,6 +240,19 @@ impl Endpoint {
     /// Default value for the timeout field in milliseconds.
     pub const DEFAULT_TIMEOUT: u64 = 3_000;
 
+    /// Returns an iterator of optional endpoint-specific headers (api-key, test-token)
+    /// as (header_name, header_value) string tuples for any that are available.
+    pub fn get_optional_headers(&self) -> impl Iterator<Item = (&'static str, &str)> {
+        [
+            self.api_key.as_ref().map(|v| ("dd-api-key", v.as_ref())),
+            self.test_token
+                .as_ref()
+                .map(|v| ("x-datadog-test-session-token", v.as_ref())),
+        ]
+        .into_iter()
+        .flatten()
+    }
+
     /// Return a request builder with the following headers:
     /// - User agent
     /// - Api key
@@ -250,32 +262,14 @@ impl Endpoint {
             .uri(self.url.clone())
             .header(hyper::header::USER_AGENT, user_agent);
 
-        // Add the Api key header if available
-        if let Some(api_key) = &self.api_key {
-            builder = builder.header(header::DATADOG_API_KEY, HeaderValue::from_str(api_key)?);
-        }
-
-        // Add the test session token if available
-        if let Some(token) = &self.test_token {
-            builder = builder.header(
-                header::X_DATADOG_TEST_SESSION_TOKEN,
-                HeaderValue::from_str(token)?,
-            );
-        }
-
-        // Add the Container Id header if available
-        if let Some(container_id) = entity_id::get_container_id() {
-            builder = builder.header(header::DATADOG_CONTAINER_ID, container_id);
-        }
-
-        // Add the Entity Id header if available
-        if let Some(entity_id) = entity_id::get_entity_id() {
-            builder = builder.header(header::DATADOG_ENTITY_ID, entity_id);
+        // Add optional endpoint headers (api-key, test-token)
+        for (name, value) in self.get_optional_headers() {
+            builder = builder.header(name, value);
         }
 
-        // Add the External Env header if available
-        if let Some(external_env) = *DD_EXTERNAL_ENV {
-            builder = builder.header(header::DATADOG_EXTERNAL_ENV, external_env);
+        // Add entity-related headers (container-id, entity-id, external-env)
+        for (name, value) in entity_id::get_entity_headers() {
+            builder = builder.header(name, value);
         }
 
         Ok(builder)

libdd-profiling/src/exporter/profile_exporter.rs

@@ -102,7 +102,6 @@ impl ProfileExporter {
         // Pre-build all static headers
         let mut headers = reqwest::header::HeaderMap::new();
 
-        // Always-present headers
         headers.insert(
             "Connection",
             reqwest::header::HeaderValue::from_static("close"),
@@ -123,18 +122,14 @@ impl ProfileExporter {
             ))?,
         );
 
-        // Optional headers (API key, test token)
-        if let Some(api_key) = &endpoint.api_key {
-            headers.insert(
-                "DD-API-KEY",
-                reqwest::header::HeaderValue::from_str(api_key)?,
-            );
+        // Add optional endpoint headers (api-key, test-token)
+        for (name, value) in endpoint.get_optional_headers() {
+            headers.insert(name, reqwest::header::HeaderValue::from_str(value)?);
         }
-        if let Some(test_token) = &endpoint.test_token {
-            headers.insert(
-                "X-Datadog-Test-Session-Token",
-                reqwest::header::HeaderValue::from_str(test_token)?,
-            );
+
+        // Add entity-related headers (container-id, entity-id, external-env)
+        for (name, value) in libdd_common::entity_id::get_entity_headers() {
+            headers.insert(name, reqwest::header::HeaderValue::from_static(value));
         }
 
         // Add Azure App Services tags if available

libdd-profiling/tests/common.rs

@@ -0,0 +1,81 @@
+// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+//! Common test utilities
+
+use std::collections::HashMap;
+
+/// Validates that entity headers (container-id, entity-id, external-env) match
+/// the values provided by libdd_common::entity_id
+///
+/// # Current Limitations
+///
+/// **NOTE:** This test helper has known limitations that should be addressed in a follow-up PR:
+///
+/// 1. **Environment-dependent behavior**: The test changes its behavior dynamically based on the
+///    exact execution environment of the test runner (e.g., whether running in a container, whether
+///    certain environment variables are set).
+///
+/// 2. **Non-deterministic across environments**: What passes on a local machine may fail in CI (or
+///    vice versa) because the underlying entity detection functions return different values in
+///    different environments.
+///
+/// 3. **Incomplete test coverage**: We only exercise the codepaths that happen to be triggered in
+///    the current test environment, not all possible combinations of entity headers being
+///    present/absent.
+///
+/// **Future improvement**: The ideal approach would be to refactor the underlying code
+/// (`libdd_common::entity_id::get_container_id()`, `get_entity_id()`, etc.) to be more testable,
+/// perhaps by making them accept injectable dependencies or configuration. Then we could test all
+/// combinations: container-id [Some/None] × entity-id [Some/None] × external-env [Some/None] to
+/// verify correct header inclusion/exclusion in all 8 cases.
+///
+/// See discussion: https://github.com/DataDog/libdatadog/pull/1493#discussion_r2745712029
+pub fn assert_entity_headers_match(headers: &HashMap<String, String>) {
+    // Check for entity headers and validate their values match what libdd_common provides
+    let expected_container_id = libdd_common::entity_id::get_container_id();
+    let expected_entity_id = libdd_common::entity_id::get_entity_id();
+    let expected_external_env = *libdd_common::entity_id::DD_EXTERNAL_ENV;
+
+    // Validate container ID
+    if let Some(expected) = expected_container_id {
+        assert_eq!(
+            headers.get("datadog-container-id"),
+            Some(&expected.to_string()),
+            "datadog-container-id header should match the value from entity_id::get_container_id()"
+        );
+    } else {
+        assert!(
+            !headers.contains_key("datadog-container-id"),
+            "datadog-container-id header should not be present when entity_id::get_container_id() is None"
+        );
+    }
+
+    // Validate entity ID
+    if let Some(expected) = expected_entity_id {
+        assert_eq!(
+            headers.get("datadog-entity-id"),
+            Some(&expected.to_string()),
+            "datadog-entity-id header should match the value from entity_id::get_entity_id()"
+        );
+    } else {
+        assert!(
+            !headers.contains_key("datadog-entity-id"),
+            "datadog-entity-id header should not be present when entity_id::get_entity_id() is None"
+        );
+    }
+
+    // Validate external env
+    if let Some(expected) = expected_external_env {
+        assert_eq!(
+            headers.get("datadog-external-env"),
+            Some(&expected.to_string()),
+            "datadog-external-env header should match the value from entity_id::DD_EXTERNAL_ENV"
+        );
+    } else {
+        assert!(
+            !headers.contains_key("datadog-external-env"),
+            "datadog-external-env header should not be present when entity_id::DD_EXTERNAL_ENV is None"
+        );
+    }
+}

libdd-profiling/tests/exporter_e2e.rs

@@ -5,6 +5,8 @@
 //!
 //! These tests validate the full export flow across different endpoint types.
 
+mod common;
+
 use libdd_common::test_utils::parse_http_request;
 use libdd_profiling::exporter::config;
 use libdd_profiling::exporter::{File, MimeType, ProfileExporter};
@@ -303,6 +305,9 @@ fn validate_full_export(req: &ReceivedRequest, expected_path: &str) -> anyhow::R
     assert_eq!(req.method, "POST");
     assert_eq!(req.path, expected_path);
 
+    // Check for entity headers and validate their values match what libdd_common provides
+    common::assert_entity_headers_match(&req.headers);
+
     // Parse the request to get multipart parts
     // We need to reconstruct a minimal HTTP request to parse
     let mut http_request_bytes = Vec::new();

libdd-profiling/tests/file_exporter_test.rs

@@ -1,6 +1,8 @@
 // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
+mod common;
+
 use libdd_common::test_utils::{create_temp_file_path, parse_http_request, TempFileGuard};
 use libdd_profiling::exporter::ProfileExporter;
 use libdd_profiling::internal::EncodedProfile;
@@ -337,5 +339,8 @@ mod tests {
             request.headers.get("dd-evp-origin-version").unwrap(),
             profiling_library_version
         );
+
+        // Check for entity headers and validate their values match what libdd_common provides
+        common::assert_entity_headers_match(&request.headers);
     }
 }