ekump/APMSP-2151 create ddsketch ffi crate (#1135)

ekump · web-flow · commit 237c030e5cab · 2025-08-14T18:31:39.000Z
* create FFI layer for ddsketch

* add helper function for CString to unwrap an empty string when unwrap fails

* add build script for FFI examples

* pin the version of the dd-apm-test-agent image so we do not always pull from the main branch
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -148,7 +148,7 @@ jobs:
         env:
           RUSTFLAGS: "${{ matrix.flags }}"
         run: |
-          cargo run --bin release --features profiling,telemetry,data-pipeline,symbolizer,crashtracker,library-config,log --release -- --out $LIBDD_OUTPUT_FOLDER
+          cargo run --bin release --features profiling,telemetry,data-pipeline,symbolizer,crashtracker,library-config,log,ddsketch --release -- --out $LIBDD_OUTPUT_FOLDER
 
       - name: 'Publish libdatadog'
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -39,6 +39,7 @@ members = [
   "data-pipeline",
   "data-pipeline-ffi",
   "ddsketch",
+  "ddsketch-ffi",
   "tinybytes",
   "dogstatsd-client",
   "datadog-log",
diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml
@@ -1,4 +1,4 @@
-root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, cc_utils, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib
+root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, cc_utils, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddsketch-ffi, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib
 third_party_libraries:
 - package_name: addr2line
   package_version: 0.24.2
diff --git a/build-profiling-ffi.sh b/build-profiling-ffi.sh
@@ -153,6 +153,7 @@ FEATURES=(
     "datadog-profiling-ffi/demangler"
     "datadog-library-config-ffi"
     "datadog-log-ffi"
+    "ddsketch-ffi"
 )
 if [[ "$symbolizer" -eq 1 ]]; then
     FEATURES+=("symbolizer")
@@ -237,7 +238,7 @@ echo "Generating $destdir/include/libdatadog headers..."
 rm -r $destdir/include/datadog/
 mkdir $destdir/include/datadog/
 
-CBINDGEN_HEADERS="common.h profiling.h telemetry.h crashtracker.h data-pipeline.h library-config.h log.h"
+CBINDGEN_HEADERS="common.h profiling.h telemetry.h crashtracker.h data-pipeline.h library-config.h log.h ddsketch.h"
 # When optional features are added, don't forget to also include the headers here
 case $ARG_FEATURES in
 esac
diff --git a/builder/Cargo.toml b/builder/Cargo.toml
@@ -15,6 +15,7 @@ data-pipeline = []
 symbolizer = []
 library-config = []
 log = []
+ddsketch = []
 
 [lib]
 bench = false
diff --git a/builder/src/bin/release.rs b/builder/src/bin/release.rs
@@ -70,6 +70,8 @@ pub fn main() {
         f.push("datadog-library-config-ffi".to_string());
         #[cfg(feature = "log")]
         f.push("datadog-log-ffi".to_string());
+        #[cfg(feature = "ddsketch")]
+        f.push("ddsketch-ffi".to_string());
         f
     };
 
diff --git a/builder/src/profiling.rs b/builder/src/profiling.rs
@@ -55,6 +55,8 @@ impl Profiling {
         headers.push("library-config.h");
         #[cfg(feature = "log")]
         headers.push("log.h");
+        #[cfg(feature = "ddsketch")]
+        headers.push("ddsketch.h");
 
         let mut origin_path: PathBuf = [&self.source_include, "dummy.h"].iter().collect();
         let mut target_path: PathBuf = [&self.target_include, "dummy.h"].iter().collect();
diff --git a/datadog-profiling-ffi/Cargo.toml b/datadog-profiling-ffi/Cargo.toml
@@ -29,6 +29,7 @@ crashtracker-receiver = ["crashtracker-ffi", "datadog-crashtracker-ffi/receiver"
 demangler = ["crashtracker-ffi", "datadog-crashtracker-ffi/demangler"]
 datadog-library-config-ffi = ["dep:datadog-library-config-ffi"]
 ddcommon-ffi = ["dep:ddcommon-ffi"]
+ddsketch-ffi = ["dep:ddsketch-ffi"]
 
 [build-dependencies]
 build_common = { path = "../build-common" }
@@ -43,6 +44,7 @@ ddcommon = { path = "../ddcommon" }
 ddcommon-ffi = { path = "../ddcommon-ffi", default-features = false, optional = true }
 ddtelemetry-ffi = { path = "../ddtelemetry-ffi", default-features = false, optional = true, features = ["expanded_builder_macros"] }
 datadog-log-ffi = { path = "../datadog-log-ffi", default-features = false, optional = true }
+ddsketch-ffi = { path = "../ddsketch-ffi", default-features = false, optional = true }
 function_name = "0.3.0"
 futures = { version = "0.3", default-features = false }
 http-body-util = "0.1"
diff --git a/datadog-profiling-ffi/src/lib.rs b/datadog-profiling-ffi/src/lib.rs
@@ -26,6 +26,11 @@ pub use ddtelemetry_ffi::*;
 #[allow(unused_imports)]
 pub use data_pipeline_ffi::*;
 
+// re-export ddsketch ffi
+#[cfg(feature = "ddsketch-ffi")]
+#[allow(unused_imports)]
+pub use ddsketch_ffi::*;
+
 // re-export library-config ffi
 #[cfg(feature = "datadog-library-config-ffi")]
 pub use datadog_library_config_ffi::*;
diff --git a/datadog-trace-utils/src/test_utils/datadog_test_agent.rs b/datadog-trace-utils/src/test_utils/datadog_test_agent.rs
@@ -18,7 +18,7 @@ use testcontainers::{
 };
 
 const TEST_AGENT_IMAGE_NAME: &str = "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent";
-const TEST_AGENT_IMAGE_TAG: &str = "latest";
+const TEST_AGENT_IMAGE_TAG: &str = "v1.31.1";
 const TEST_AGENT_READY_MSG: &str =
     "INFO:ddapm_test_agent.agent:Trace request stall seconds setting set to 0.0.";
 
diff --git a/ddcommon-ffi/src/cstr.rs b/ddcommon-ffi/src/cstr.rs
@@ -60,6 +60,37 @@ impl CString {
         Ok(Self::from_std(std::ffi::CString::new(t)?))
     }
 
+    /// Creates a new `CString` from the given input, or returns an empty `CString`
+    /// if the input contains null bytes.
+    ///
+    /// This method performs an unwrap, but will never panic, as the unwrap is for the fallback to
+    /// an empty string which is guaranteed to not contain null bytes and can always be
+    /// converted to a `CString`. The value of this function is to minimize the occurrences of
+    /// unwrap() in libdatadog.
+    ///
+    /// # Note
+    ///
+    /// This function will always return a valid `CString`. If the input contains null bytes, the
+    /// returned `CString` will be empty.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use ddcommon_ffi::CString;
+    ///
+    /// let good = CString::new_or_empty("hello world");
+    /// assert_eq!(good.as_cstr().into_std().to_str().unwrap(), "hello world");
+    ///
+    /// let bad = CString::new_or_empty("hello\0world");
+    /// assert_eq!(bad.as_cstr().into_std().to_str().unwrap(), "");
+    /// ```
+    pub fn new_or_empty<T: Into<Vec<u8>>>(t: T) -> Self {
+        Self::new(t).unwrap_or_else(|_| {
+            #[allow(clippy::unwrap_used)]
+            Self::new("").unwrap()
+        })
+    }
+
     pub fn as_cstr(&self) -> CStr<'_> {
         CStr {
             ptr: self.ptr,
@@ -118,6 +149,18 @@ mod tests {
         assert_eq!(s.as_cstr().into_std().to_str().unwrap(), "hello");
     }
 
+    #[test]
+    fn test_cstring_new_or_empty() {
+        let good = CString::new_or_empty("hello world");
+        assert_eq!(good.as_cstr().into_std().to_str().unwrap(), "hello world");
+
+        let bad = CString::new_or_empty("hello\0world");
+        assert_eq!(bad.as_cstr().into_std().to_str().unwrap(), "");
+
+        let empty = CString::new_or_empty("");
+        assert_eq!(empty.as_cstr().into_std().to_str().unwrap(), "");
+    }
+
     #[test]
     fn test_raw_cstr() {
         let s: &'static [u8] = b"abc\0";
diff --git a/ddsketch-ffi/Cargo.toml b/ddsketch-ffi/Cargo.toml
@@ -0,0 +1,26 @@
+# Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "ddsketch-ffi"
+edition.workspace = true
+version.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+[lib]
+crate-type = ["lib", "staticlib", "cdylib"]
+bench = false
+
+[features]
+default = ["cbindgen"]
+cbindgen = ["build_common/cbindgen", "ddcommon-ffi/cbindgen"]
+
+[build-dependencies]
+build_common = { path = "../build-common" }
+
+[dependencies]
+datadog-ddsketch = { path = "../ddsketch" }
+ddcommon-ffi = { path = "../ddcommon-ffi", default-features = false }
+
+[dev-dependencies]
diff --git a/ddsketch-ffi/build.rs b/ddsketch-ffi/build.rs
@@ -0,0 +1,11 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+extern crate build_common;
+
+use build_common::generate_and_configure_header;
+
+fn main() {
+    let header_name = "ddsketch.h";
+    generate_and_configure_header(header_name);
+}
diff --git a/ddsketch-ffi/cbindgen.toml b/ddsketch-ffi/cbindgen.toml
@@ -0,0 +1,32 @@
+# Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+# SPDX-License-Identifier: Apache-2.0
+
+language = "C"
+cpp_compat = true
+tab_width = 2
+header = """// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+"""
+include_guard = "DDOG_DDSKETCH_H"
+style = "both"
+pragma_once = true
+no_includes = true
+sys_includes = ["stdbool.h", "stddef.h", "stdint.h"]
+includes = ["common.h"]
+
+[parse]
+parse_deps = true
+include = ["ddcommon-ffi", "datadog-ddsketch"]
+
+[export]
+include = ["ddsketch-ffi"]
+prefix = "ddsketch_"
+renaming_overrides_prefixing = true
+
+
+[fn]
+must_use = "DDOG_CHECK_RETURN"
+
+[enum]
+prefix_with_name = true
+rename_variants = "ScreamingSnakeCase"
diff --git a/ddsketch-ffi/src/lib.rs b/ddsketch-ffi/src/lib.rs
diff --git a/examples/ffi/CMakeLists.txt b/examples/ffi/CMakeLists.txt
diff --git a/examples/ffi/README.md b/examples/ffi/README.md
diff --git a/examples/ffi/build-examples.sh b/examples/ffi/build-examples.sh
diff --git a/examples/ffi/ddsketch.c b/examples/ffi/ddsketch.c
diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		-root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, cc_utils, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib
	`1`	+root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, cc_utils, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddsketch-ffi, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib
`2`	`2`	`third_party_libraries:`
`3`	`3`	`- package_name: addr2line`
`4`	`4`	`package_version: 0.24.2`
Original file line number	Diff line number	Diff line change
`@@ -70,6 +70,8 @@ pub fn main() {`
`70`	`70`	`f.push("datadog-library-config-ffi".to_string());`
`71`	`71`	`#[cfg(feature = "log")]`
`72`	`72`	`f.push("datadog-log-ffi".to_string());`
	`73`	`+ #[cfg(feature = "ddsketch")]`
	`74`	`+ f.push("ddsketch-ffi".to_string());`
`73`	`75`	`f`
`74`	`76`	`};`
`75`	`77`