diff --git a/Cargo.lock b/Cargo.lock index 9e33fac0c..6524f40fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1650,6 +1650,7 @@ dependencies = [ "chrono", "criterion", "datadog-alloc", + "datadog-profiling-otel", "datadog-profiling-protobuf", "ddcommon", "futures", @@ -1694,6 +1695,19 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "datadog-profiling-otel" +version = "21.0.0" +dependencies = [ + "anyhow", + "bolero", + "prost", + "prost-build", + "prost-types", + "protoc-bin-vendored", + "zstd", +] + [[package]] name = "datadog-profiling-protobuf" version = "21.0.0" diff --git a/Cargo.toml b/Cargo.toml index ff267a885..9c5851fe7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "datadog-live-debugger-ffi", "datadog-profiling", "datadog-profiling-ffi", + "datadog-profiling-otel", "datadog-profiling-protobuf", "datadog-profiling-replayer", "datadog-remote-config", diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index e280439d6..866e7736d 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,4 +1,4 @@ -root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, cc_utils, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddsketch-ffi, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib +root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, cc_utils, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-otel, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddsketch-ffi, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib third_party_libraries: - package_name: addr2line package_version: 0.24.2 @@ -22241,6 +22241,13 @@ third_party_libraries: licenses: - license: Apache-2.0 text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" +- package_name: prost-types + package_version: 0.13.5 + repository: https://github.com/tokio-rs/prost + license: Apache-2.0 + licenses: + - license: Apache-2.0 + text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" - package_name: quinn package_version: 0.11.5 repository: https://github.com/quinn-rs/quinn diff --git a/bin_tests/tests/crashtracker_bin_test.rs b/bin_tests/tests/crashtracker_bin_test.rs index 6b26e28eb..d7d178186 100644 --- a/bin_tests/tests/crashtracker_bin_test.rs +++ b/bin_tests/tests/crashtracker_bin_test.rs @@ -145,6 +145,7 @@ fn test_crasht_tracking_validate_callstack() { test_crash_tracking_callstack() } +#[cfg(not(any(all(target_arch = "x86_64", target_env = "musl"), target_os = "macos")))] fn test_crash_tracking_callstack() { let (_, crashtracker_receiver) = setup_crashtracking_crates(BuildProfile::Release); diff --git a/datadog-profiling-ffi/src/profiles/datatypes.rs b/datadog-profiling-ffi/src/profiles/datatypes.rs index ba3fb9f22..0e7514554 100644 --- a/datadog-profiling-ffi/src/profiles/datatypes.rs +++ b/datadog-profiling-ffi/src/profiles/datatypes.rs @@ -754,7 +754,8 @@ pub unsafe extern "C" fn ddog_prof_Profile_serialize( } let end_time = end_time.map(SystemTime::from); - old_profile.serialize_into_compressed_pprof(end_time, None) + // TODO: make this an option instead of hardcoding to otel + old_profile.serialize_into_compressed_otel(end_time, None) })() .context("ddog_prof_Profile_serialize failed") .into() diff --git a/datadog-profiling-otel/Cargo.toml b/datadog-profiling-otel/Cargo.toml new file mode 100644 index 000000000..6e18af1d1 --- /dev/null +++ b/datadog-profiling-otel/Cargo.toml @@ -0,0 +1,25 @@ +# Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "datadog-profiling-otel" +rust-version.workspace = true +edition.workspace = true +version.workspace = true +license.workspace = true + +[lib] +bench = false + +[dependencies] +prost = "0.13" +prost-types = "0.13" +zstd = "0.13" +anyhow = "1.0" + +[build-dependencies] +prost-build = "0.13" +protoc-bin-vendored = "3.0.0" + +[dev-dependencies] +bolero = "0.13" diff --git a/datadog-profiling-otel/README.md b/datadog-profiling-otel/README.md new file mode 100644 index 000000000..afb7798f5 --- /dev/null +++ b/datadog-profiling-otel/README.md @@ -0,0 +1,23 @@ +# datadog-profiling-otel + +This module provides Rust bindings for the OpenTelemetry profiling protobuf definitions, generated using the `prost` library. +This crate implements serialization of data into the otel profiling format; if you're building a profiler you usually don't want to use this crate directly, and instead should use datadog-profiling and ask it to serialize using ottel. + +## Usage + +See the [basic_usage.rs](examples/basic_usage.rs) example for a complete demonstration of how to create OpenTelemetry profile data. + +### Running Examples + +```bash +# Run the basic usage example +cargo run --example basic_usage + +# Run tests +cargo test + +# Build +cargo build +``` + + diff --git a/datadog-profiling-otel/build.rs b/datadog-profiling-otel/build.rs new file mode 100644 index 000000000..d7d83ce64 --- /dev/null +++ b/datadog-profiling-otel/build.rs @@ -0,0 +1,35 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::env; +use std::path::PathBuf; + +fn main() { + // protoc is required to compile proto files. This uses protoc-bin-vendored to provide + // the protoc binary, setting the env var to tell prost_build where to find it. + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); + + // Tell Cargo to rerun this build script if the proto files change + println!("cargo:rerun-if-changed=profiles.proto"); + println!("cargo:rerun-if-changed=opentelemetry/proto/common/v1/common.proto"); + println!("cargo:rerun-if-changed=opentelemetry/proto/resource/v1/resource.proto"); + + // Create the output directory + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + // Configure prost-build + let mut config = prost_build::Config::new(); + config.out_dir(&out_dir); + + // Compile the proto files - include all proto files so imports work correctly + config + .compile_protos( + &[ + "opentelemetry/proto/common/v1/common.proto", + "opentelemetry/proto/resource/v1/resource.proto", + "profiles.proto", + ], + &["."], + ) + .expect("Failed to compile protobuf files"); +} diff --git a/datadog-profiling-otel/examples/basic_usage.rs b/datadog-profiling-otel/examples/basic_usage.rs new file mode 100644 index 000000000..d0ccc8ab5 --- /dev/null +++ b/datadog-profiling-otel/examples/basic_usage.rs @@ -0,0 +1,55 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +// This example demonstrates how to use the generated protobuf code +// from the OpenTelemetry profiles.proto file. + +fn main() { + use datadog_profiling_otel::*; + + let mut profiles_dict = ProfilesDictionary::default(); + + profiles_dict.string_table.push("cpu".to_string()); + profiles_dict.string_table.push("nanoseconds".to_string()); + profiles_dict.string_table.push("main".to_string()); + + let sample_type = ValueType { + type_strindex: 0, // "cpu" + unit_strindex: 1, // "nanoseconds" + aggregation_temporality: AggregationTemporality::Delta.into(), + }; + + let profile = Profile { + sample_type: Some(sample_type), + time_nanos: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as i64, + ..Default::default() + }; + + let mut profiles_data = ProfilesData { + dictionary: Some(profiles_dict), + ..Default::default() + }; + + let mut scope_profiles = ScopeProfiles::default(); + scope_profiles.profiles.push(profile); + + let mut resource_profiles = ResourceProfiles::default(); + resource_profiles.scope_profiles.push(scope_profiles); + + profiles_data.resource_profiles.push(resource_profiles); + + println!("Successfully created OpenTelemetry profile data!"); + println!( + "Profile contains {} resource profiles", + profiles_data.resource_profiles.len() + ); + println!( + "Time: {}", + profiles_data.resource_profiles[0].scope_profiles[0].profiles[0].time_nanos + ); + let serialized = profiles_data.serialize_into_compressed_proto().unwrap(); + println!("Serialized size: {} bytes", serialized.len()); +} diff --git a/datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto b/datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto new file mode 100644 index 000000000..5137e6d55 --- /dev/null +++ b/datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto @@ -0,0 +1,40 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.common.v1; + +// KeyValue is a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValue { + string key = 1; + oneof value { + string string_value = 2; + bool bool_value = 3; + int64 int_value = 4; + double double_value = 5; + bytes bytes_value = 6; + } +} + +// InstrumentationScope is a message representing the instrumentation scope information +// such as the fully qualified name and version. +message InstrumentationScope { + // An empty instrumentation scope name means the name is unknown. + string name = 1; + string version = 2; + repeated KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; +} diff --git a/datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto b/datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto new file mode 100644 index 000000000..06dbb8032 --- /dev/null +++ b/datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto @@ -0,0 +1,29 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.resource.v1; + +import "opentelemetry/proto/common/v1/common.proto"; + +// Resource information. +message Resource { + // Set of attributes that describe the resource. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 2; +} diff --git a/datadog-profiling-otel/profiles.proto b/datadog-profiling-otel/profiles.proto new file mode 100644 index 000000000..060ff86df --- /dev/null +++ b/datadog-profiling-otel/profiles.proto @@ -0,0 +1,503 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file includes work covered by the following copyright and permission notices: +// +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1development; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Development"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.profiles.v1development"; +option java_outer_classname = "ProfilesProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1development"; + +// Relationships Diagram +// +// ┌──────────────────┐ LEGEND +// │ ProfilesData │ ─────┐ +// └──────────────────┘ │ ─────▶ embedded +// │ │ +// │ 1-n │ ─────▷ referenced by index +// ▼ ▼ +// ┌──────────────────┐ ┌────────────────────┐ +// │ ResourceProfiles │ │ ProfilesDictionary │ +// └──────────────────┘ └────────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ ScopeProfiles │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▼ +// ┌──────────────────┐ +// │ Profile │ +// └──────────────────┘ +// │ n-1 +// │ 1-n ┌───────────────────────────────────────┐ +// ▼ │ ▽ +// ┌──────────────────┐ 1-n ┌──────────────┐ ┌──────────┐ +// │ Sample │ ──────▷ │ KeyValue │ │ Link │ +// └──────────────────┘ └──────────────┘ └──────────┘ +// │ 1-n △ △ +// │ 1-n ┌─────────────────┘ │ 1-n +// ▽ │ │ +// ┌──────────────────┐ n-1 ┌──────────────┐ +// │ Location │ ──────▷ │ Mapping │ +// └──────────────────┘ └──────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ Line │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▽ +// ┌──────────────────┐ +// │ Function │ +// └──────────────────┘ +// + +// ProfilesDictionary represents the profiles data shared across the +// entire message being sent. +message ProfilesDictionary { + // Mappings from address ranges to the image/binary/library mapped + // into that address range referenced by locations via Location.mapping_index. + // mapping_table[0] must always be set to a zero value default mapping, + // so that _index fields can use 0 to indicate null/unset. + repeated Mapping mapping_table = 1; + + // Locations referenced by samples via Stack.location_indices. + repeated Location location_table = 2; + + // Functions referenced by locations via Line.function_index. + repeated Function function_table = 3; + + // Links referenced by samples via Sample.link_index. + // link_table[0] must always be set to a zero value default link, + // so that _index fields can use 0 to indicate null/unset. + repeated Link link_table = 4; + + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 5; + + // A common table for attributes referenced by various messages. + repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 6; + + // Represents a mapping between Attribute Keys and Units. + repeated AttributeUnit attribute_units = 7; + + // Stacks referenced by samples via Sample.stack_index. + repeated Stack stack_table = 8; +} + +// ProfilesData represents the profiles data that can be stored in persistent storage, +// OR can be embedded by other protocols that transfer OTLP profiles data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message ProfilesData { + // An array of ResourceProfiles. + // For data coming from an SDK profiler, this array will typically contain one + // element. Host-level profilers will usually create one ResourceProfile per + // container, as well as one additional ResourceProfile grouping all samples + // from non-containerized processes. + // Other resource groupings are possible as well and clarified via + // Resource.attributes and semantic conventions. + repeated ResourceProfiles resource_profiles = 1; + + // One instance of ProfilesDictionary + ProfilesDictionary dictionary = 2; +} + + +// A collection of ScopeProfiles from a Resource. +message ResourceProfiles { + reserved 1000; + + // The resource for the profiles in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeProfiles that originate from a resource. + repeated ScopeProfiles scope_profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_profiles" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Profiles produced by an InstrumentationScope. +message ScopeProfiles { + // The instrumentation scope information for the profiles in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Profiles that originate from an instrumentation scope. + repeated Profile profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the profile data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all profiles in the "profiles" field. + string schema_url = 3; + + // The preferred type and unit of Samples in at least one Profile. + // See Profile.sample_type for possible values. + ValueType default_sample_type = 4; +} + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - The profile is represented as a set of samples, where each sample +// references a stack trace which is a list of locations, each belonging +// to a mapping. +// - There is a N->1 relationship from Stack.location_indices entries to +// locations. For every Stack.location_indices entry there must be a +// unique Location with that index. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that index. + +// Represents a complete profile, including sample types, samples, mappings to +// binaries, stacks, locations, functions, string table, and additional +// metadata. It modifies and annotates pprof Profile with OpenTelemetry +// specific fields. +// +// Note that whilst fields in this message retain the name and field id from pprof in most cases +// for ease of understanding data migration, it is not intended that pprof:Profile and +// OpenTelemetry:Profile encoding be wire compatible. +message Profile { + // The type and unit of all Sample.values in this profile. + // For a cpu or off-cpu profile this might be: + // ["cpu","nanoseconds"] or ["off_cpu","nanoseconds"] + // For a heap profile, this might be: + // ["allocated_objects","count"] or ["allocated_space","bytes"], + ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + + // The following fields 3-13 are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 3; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 4; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 5; + // The number of events between sampled occurrences. + int64 period = 6; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int32 comment_strindices = 7; // Indices into ProfilesDictionary.string_table. + + // A globally unique identifier for a profile. The ID is a 16-byte array. An ID with + // all zeroes is considered invalid. It may be used for deduplication and signal + // correlation purposes. It is acceptable to treat two profiles with different values + // in this field as not equal, even if they represented the same object at an earlier + // time. + // This field is optional; an ID may be assigned to an ID-less profile in a later step. + bytes profile_id = 8; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 9; + + // Specifies format of the original payload. Common values are defined in semantic conventions. [required if original_payload is present] + string original_payload_format = 10; + + // Original payload can be stored in this field. This can be useful for users who want to get the original payload. + // Formats such as JFR are highly extensible and can contain more information than what is defined in this spec. + // Inclusion of original payload should be configurable by the user. Default behavior should be to not include the original payload. + // If the original payload is in pprof format, it SHOULD not be included in this field. + // The field is optional, however if it is present then equivalent converted data should be populated in other fields + // of this message as far as is practicable. + bytes original_payload = 11; + + // References to attributes in attribute_table. [optional] + // It is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated int32 attribute_indices = 12; +} + +// Represents a mapping between Attribute Keys and Units. +message AttributeUnit { + // Index into string table. + int32 attribute_key_strindex = 1; + // Index into string table. + int32 unit_strindex = 2; +} + +// A pointer from a profile Sample to a trace Span. +// Connects a profile sample to a trace span, identified by unique trace and span IDs. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// Specifies the method of aggregating metric values, either DELTA (change since last report) +// or CUMULATIVE (total since a fixed start time). +enum AggregationTemporality { + /* UNSPECIFIED is the default AggregationTemporality, it MUST not be used. */ + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + + /** DELTA is an AggregationTemporality for a profiler which reports + changes since last report time. Successive metrics contain aggregation of + values from continuous and non-overlapping intervals. + + The values for a DELTA metric are based only on the time interval + associated with one measurement cycle. There is no dependency on + previous measurements like is the case for CUMULATIVE metrics. + + For example, consider a system measuring the number of requests that + it receives and reports the sum of these requests every second as a + DELTA metric: + + 1. The system starts receiving at time=t_0. + 2. A request is received, the system measures 1 request. + 3. A request is received, the system measures 1 request. + 4. A request is received, the system measures 1 request. + 5. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+1 with a value of 3. + 6. A request is received, the system measures 1 request. + 7. A request is received, the system measures 1 request. + 8. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0+1 to + t_0+2 with a value of 2. */ + AGGREGATION_TEMPORALITY_DELTA = 1; + + /** CUMULATIVE is an AggregationTemporality for a profiler which + reports changes since a fixed start time. This means that current values + of a CUMULATIVE metric depend on all previous measurements since the + start time. Because of this, the sender is required to retain this state + in some form. If this state is lost or invalidated, the CUMULATIVE metric + values MUST be reset and a new fixed start time following the last + reported measurement time sent MUST be used. + + For example, consider a system measuring the number of requests that + it receives and reports the sum of these requests every second as a + CUMULATIVE metric: + + 1. The system starts receiving at time=t_0. + 2. A request is received, the system measures 1 request. + 3. A request is received, the system measures 1 request. + 4. A request is received, the system measures 1 request. + 5. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+1 with a value of 3. + 6. A request is received, the system measures 1 request. + 7. A request is received, the system measures 1 request. + 8. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+2 with a value of 5. + 9. The system experiences a fault and loses state. + 10. The system recovers and resumes receiving at time=t_1. + 11. A request is received, the system measures 1 request. + 12. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_1 to + t_1+1 with a value of 1. + + Note: Even though, when reporting changes since last report time, using + CUMULATIVE is valid, it is not recommended. */ + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +// ValueType describes the type and units of a value, with an optional aggregation temporality. +message ValueType { + int32 type_strindex = 1; // Index into ProfilesDictionary.string_table. + int32 unit_strindex = 2; // Index into ProfilesDictionary.string_table. + + AggregationTemporality aggregation_temporality = 3; +} + +// Each Sample records values encountered in some program context. The program +// context is typically a stack trace, perhaps augmented with auxiliary +// information like the thread-id, some indicator of a higher level request +// being handled etc. +// +// A Sample MUST have have at least one values or timestamps_unix_nano entry. If +// both fields are populated, they MUST contain the same number of elements, and +// the elements at the same index MUST refer to the same event. +// +// Examples of different ways of representing a sample with the total value of 10: +// +// Report of a stacktrace at 10 timestamps (consumers must assume the value is 1 for each point): +// values: [] +// timestamps_unix_nano: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +// +// Report of a stacktrace with an aggregated value without timestamps: +// values: [10] +// timestamps_unix_nano: [] +// +// Report of a stacktrace at 4 timestamps where each point records a specific value: +// values: [2, 2, 3, 3] +// timestamps_unix_nano: [1, 2, 3, 4] +message Sample { + // Reference to stack in ProfilesDictionary.stack_table. + int32 stack_index = 1; + // The type and unit of each value is defined by Profile.sample_type. + repeated int64 values = 2; + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 3; + + // Reference to link in ProfilesDictionary.link_table. [optional] + // It can be unset / set to 0 if no link exists, as link_table[0] is always a 'null' default value. + int32 link_index = 4; + + // Timestamps associated with Sample represented in nanoseconds. These + // timestamps should fall within the Profile's time range. + repeated uint64 timestamps_unix_nano = 5; +} + +// Describes the mapping of a binary in memory, including its address range, +// file offset, and metadata like build ID +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int32 filename_strindex = 4; // Index into ProfilesDictionary.string_table. + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 5; + // The following fields indicate the resolution of symbolic info. + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; +} + +// A Stack represents a stack trace as a list of locations. The first location +// is the leaf frame. +message Stack { + // References to locations in ProfilesDictionary.location_table. + repeated int32 location_indices = 1; +} + +// Describes function and line table debug information. +message Location { + // Reference to mapping in ProfilesDictionary.mapping_table. + // It can be unset / set to 0 if the mapping is unknown or not applicable for + // this profile type, as mapping_table[0] is always a 'null' default mapping. + int32 mapping_index = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; + + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 5; +} + +// Details a specific line in a source code, linked to a function. +message Line { + // Reference to function in ProfilesDictionary.function_table. + int32 function_index = 1; + // Line number in source code. 0 means unset. + int64 line = 2; + // Column number in source code. 0 means unset. + int64 column = 3; +} + +// Describes a function, including its human-readable name, system name, +// source file, and starting line number in the source. +message Function { + // Function name. Empty string if not available. + int32 name_strindex = 1; + // Function name, as identified by the system. For instance, + // it can be a C++ mangled name. Empty string if not available. + int32 system_name_strindex = 2; + // Source file containing the function. Empty string if not available. + int32 filename_strindex = 3; + // Line number in source file. 0 means unset. + int64 start_line = 4; +} \ No newline at end of file diff --git a/datadog-profiling-otel/src/lib.rs b/datadog-profiling-otel/src/lib.rs new file mode 100644 index 000000000..bba0cee27 --- /dev/null +++ b/datadog-profiling-otel/src/lib.rs @@ -0,0 +1,80 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] +#![cfg_attr(not(test), deny(clippy::unimplemented))] + +pub mod proto; + +pub use proto::*; + +/// Extension trait for ProfilesData to add serialization methods +pub trait ProfilesDataExt { + /// Serializes the profile into a zstd compressed protobuf byte array. + /// This method consumes the ProfilesData and returns the compressed bytes. + fn serialize_into_compressed_proto(self) -> anyhow::Result>; +} + +impl ProfilesDataExt for ProfilesData { + fn serialize_into_compressed_proto(self) -> anyhow::Result> { + // TODO, streaming into zstd is difficult because prost wants a BytesMut which zstd doesn't + // easily supply. + let proto_bytes = prost::Message::encode_to_vec(&self); + let compressed_bytes = zstd::encode_all(&proto_bytes[..], 0)?; + Ok(compressed_bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_proto_generation() { + // Test that we can create basic protobuf messages + let mut profiles_dict = ProfilesDictionary::default(); + profiles_dict.string_table.push("test".to_string()); + + let profiles_data = ProfilesData { + dictionary: Some(profiles_dict), + ..Default::default() + }; + + // Verify the data was set correctly + assert_eq!( + profiles_data.dictionary.as_ref().unwrap().string_table[0], + "test" + ); + } + + #[test] + #[cfg_attr(miri, ignore)] // Skip this test when running under Miri + fn test_serialize_into_compressed_proto() { + // Test that we can serialize and compress a ProfilesData + let mut profiles_dict = ProfilesDictionary::default(); + profiles_dict.string_table.push("test".to_string()); + + let profiles_data = ProfilesData { + dictionary: Some(profiles_dict), + ..Default::default() + }; + + // Serialize and compress + let compressed_bytes = profiles_data.serialize_into_compressed_proto().unwrap(); + + // Verify we got compressed bytes + assert!(!compressed_bytes.is_empty()); + + // Verify we can decompress and deserialize + let decompressed_bytes = zstd::decode_all(&compressed_bytes[..]).unwrap(); + let deserialized: ProfilesData = prost::Message::decode(&decompressed_bytes[..]).unwrap(); + + // Verify the data is correct + assert_eq!( + deserialized.dictionary.as_ref().unwrap().string_table[0], + "test" + ); + } +} diff --git a/datadog-profiling-otel/src/proto.rs b/datadog-profiling-otel/src/proto.rs new file mode 100644 index 000000000..053bd37da --- /dev/null +++ b/datadog-profiling-otel/src/proto.rs @@ -0,0 +1,38 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +// This module is generated by prost-build from profiles.proto +#[allow(clippy::all)] +pub mod opentelemetry { + pub mod proto { + pub mod common { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/opentelemetry.proto.common.v1.rs" + )); + } + } + pub mod resource { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/opentelemetry.proto.resource.v1.rs" + )); + } + } + pub mod profiles { + pub mod v1development { + include!(concat!( + env!("OUT_DIR"), + "/opentelemetry.proto.profiles.v1development.rs" + )); + } + } + } +} + +// Re-export commonly used types +pub use opentelemetry::proto::common::v1::*; +pub use opentelemetry::proto::profiles::v1development::*; +pub use opentelemetry::proto::resource::v1::*; diff --git a/datadog-profiling/Cargo.toml b/datadog-profiling/Cargo.toml index 92c6acd2f..adad70082 100644 --- a/datadog-profiling/Cargo.toml +++ b/datadog-profiling/Cargo.toml @@ -24,6 +24,7 @@ byteorder = { version = "1.5", features = ["std"] } bytes = "1.1" chrono = {version = "0.4", default-features = false, features = ["std", "clock"]} datadog-alloc = { path = "../datadog-alloc" } +datadog-profiling-otel = { path = "../datadog-profiling-otel" } datadog-profiling-protobuf = { path = "../datadog-profiling-protobuf", features = ["prost_impls"] } ddcommon = {path = "../ddcommon" } futures = { version = "0.3", default-features = false } diff --git a/datadog-profiling/src/internal/profile/mod.rs b/datadog-profiling/src/internal/profile/mod.rs index 2b4154cfd..1316a2592 100644 --- a/datadog-profiling/src/internal/profile/mod.rs +++ b/datadog-profiling/src/internal/profile/mod.rs @@ -5,6 +5,7 @@ mod fuzz_tests; pub mod interning_api; +pub mod otel_emitter; use self::api::UpscalingInfo; use super::*; diff --git a/datadog-profiling/src/internal/profile/otel_emitter/function.rs b/datadog-profiling/src/internal/profile/otel_emitter/function.rs new file mode 100644 index 000000000..32b6de338 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/function.rs @@ -0,0 +1,59 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::Function as InternalFunction; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Function { + fn from(internal_function: InternalFunction) -> Self { + Self::from(&internal_function) + } +} + +// For references (existing implementation) +impl From<&InternalFunction> for datadog_profiling_otel::Function { + fn from(internal_function: &InternalFunction) -> Self { + Self { + name_strindex: internal_function.name.to_raw_id() as i32, + system_name_strindex: internal_function.system_name.to_raw_id() as i32, + filename_strindex: internal_function.filename.to_raw_id() as i32, + start_line: 0, // Not available in internal Function + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collections::identifiable::StringId; + + #[test] + fn test_from_internal_function() { + // Test basic conversion + let internal_function = InternalFunction { + name: StringId::from_offset(0), + system_name: StringId::from_offset(1), + filename: StringId::from_offset(2), + }; + + let otel_function = datadog_profiling_otel::Function::from(&internal_function); + assert_eq!(otel_function.name_strindex, 0); + assert_eq!(otel_function.system_name_strindex, 1); + assert_eq!(otel_function.filename_strindex, 2); + assert_eq!(otel_function.start_line, 0); + + // Test with large offsets + let internal_function = InternalFunction { + name: StringId::from_offset(999999), + system_name: StringId::from_offset(888888), + filename: StringId::from_offset(777777), + }; + + let otel_function = datadog_profiling_otel::Function::from(&internal_function); + assert_eq!(otel_function.name_strindex, 999999); + assert_eq!(otel_function.system_name_strindex, 888888); + assert_eq!(otel_function.filename_strindex, 777777); + assert_eq!(otel_function.start_line, 0); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/label.rs b/datadog-profiling/src/internal/profile/otel_emitter/label.rs new file mode 100644 index 000000000..e60f63e0a --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/label.rs @@ -0,0 +1,172 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! All the mess here will go away once https://github.com/open-telemetry/opentelemetry-proto/pull/672/files is merged. + +use crate::collections::identifiable::Id; +use crate::internal::Label as InternalLabel; +use anyhow::{Context, Result}; +use std::collections::HashMap; + +/// Converts a datadog-profiling internal Label to an OpenTelemetry KeyValue +/// +/// # Arguments +/// * `label` - The internal label to convert +/// * `string_table` - A slice of strings where StringIds index into +/// * `key_to_unit_map` - A mutable map from key string index to unit string index for numeric +/// labels +/// +/// # Returns +/// * `Ok(KeyValue)` if the conversion is successful +/// * `Err` with context if the StringIds are out of bounds of the string table +pub fn convert_label_to_key_value( + label: &InternalLabel, + string_table: &[String], + key_to_unit_map: &mut HashMap, +) -> Result { + // Get the key string + let key_id = label.get_key().to_raw_id() as usize; + let key = string_table + .get(key_id) + .with_context(|| { + format!( + "Key string index {} out of bounds (string table has {} elements)", + key_id, + string_table.len() + ) + })? + .to_string(); + + match label.get_value() { + crate::internal::LabelValue::Str(str_id) => { + let str_value_id = str_id.to_raw_id() as usize; + let str_value = string_table + .get(str_value_id) + .with_context(|| { + format!( + "Value string index {} out of bounds (string table has {} elements)", + str_value_id, + string_table.len() + ) + })? + .to_string(); + + Ok(datadog_profiling_otel::KeyValue { + key, + value: Some(datadog_profiling_otel::key_value::Value::StringValue( + str_value, + )), + }) + } + crate::internal::LabelValue::Num { num, num_unit } => { + // Note: OpenTelemetry KeyValue doesn't support units, so we only store the numeric + // value But we track the mapping for building the attribute_units table + let unit_index = num_unit.to_raw_id() as usize; + + // Only add to the map if the unit is not the default empty string (index 0) + if unit_index > 0 { + key_to_unit_map.insert(key_id, unit_index); + } + + Ok(datadog_profiling_otel::KeyValue { + key, + value: Some(datadog_profiling_otel::key_value::Value::IntValue(*num)), + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collections::identifiable::StringId; + + #[test] + fn test_convert_label() { + // Test string label conversion + let string_table = vec![ + "".to_string(), // index 0 + "thread_id".to_string(), // index 1 + "main".to_string(), // index 2 + ]; + + let label = InternalLabel::str( + StringId::from_offset(1), // "thread_id" + StringId::from_offset(2), // "main" + ); + + let mut key_to_unit_map = HashMap::new(); + let key_value = + convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map).unwrap(); + assert_eq!(key_value.key, "thread_id"); + let s = match key_value.value.expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::StringValue(s) => s, + _ => panic!("Expected StringValue"), + }; + assert_eq!(s, "main"); + + // Test numeric label with unit mapping + let label = InternalLabel::num( + StringId::from_offset(1), // "thread_id" (reusing key) + 1024, // 1024 + StringId::from_offset(2), // "main" (reusing as unit) + ); + + let key_value = + convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map).unwrap(); + assert_eq!(key_value.key, "thread_id"); + let n = match key_value.value.expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, + _ => panic!("Expected IntValue"), + }; + assert_eq!(n, 1024); + + // Verify unit mapping was added + assert_eq!(key_to_unit_map.get(&1), Some(&2)); + + // Test numeric label without unit mapping (empty string unit) + let label = InternalLabel::num( + StringId::from_offset(1), // "thread_id" + 42, // 42 + StringId::from_offset(0), // empty string (default) + ); + + let key_value = + convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map).unwrap(); + assert_eq!(key_value.key, "thread_id"); + let n = match key_value.value.expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, + _ => panic!("Expected IntValue"), + }; + assert_eq!(n, 42); + + // Unit mapping should still exist from previous test + assert_eq!(key_to_unit_map.get(&1), Some(&2)); + } + + #[test] + fn test_convert_label_errors() { + // Test out of bounds key + let string_table = vec!["".to_string()]; // Only one string + + let label = InternalLabel::str( + StringId::from_offset(1), // This index doesn't exist + StringId::from_offset(0), // This index exists + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_err()); + + // Test empty string table + let string_table: Vec = vec![]; + + let label = InternalLabel::str( + StringId::from_offset(0), // Even index 0 is out of bounds + StringId::from_offset(0), + ); + + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_err()); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/location.rs b/datadog-profiling/src/internal/profile/otel_emitter/location.rs new file mode 100644 index 000000000..6178debff --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/location.rs @@ -0,0 +1,74 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Location { + fn from(internal_location: internal::Location) -> Self { + Self::from(&internal_location) + } +} + +// For references (existing implementation) +impl From<&internal::Location> for datadog_profiling_otel::Location { + fn from(internal_location: &internal::Location) -> Self { + Self { + mapping_index: internal_location + .mapping_id + .map(|id| id.to_raw_id() as i32) + .unwrap_or(0), // 0 represents no mapping + address: internal_location.address, + line: vec![datadog_profiling_otel::Line { + function_index: internal_location.function_id.to_raw_id() as i32, + line: internal_location.line, + column: 0, // Not available in internal Location + }], + is_folded: false, // Not available in internal Location + attribute_indices: vec![], // Not available in internal Location + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::internal::{FunctionId, MappingId}; + + #[test] + fn test_from_internal_location() { + // Test with mapping + let internal_location = internal::Location { + mapping_id: Some(MappingId::from_offset(1)), + function_id: FunctionId::from_offset(2), + address: 0x1000, + line: 42, + }; + + let otel_location = datadog_profiling_otel::Location::from(&internal_location); + assert_eq!(otel_location.mapping_index, 2); + assert_eq!(otel_location.address, 0x1000); + assert_eq!(otel_location.line.len(), 1); + assert_eq!(otel_location.line[0].function_index, 3); + assert_eq!(otel_location.line[0].line, 42); + assert_eq!(otel_location.line[0].column, 0); + assert!(!otel_location.is_folded); + assert_eq!(otel_location.attribute_indices, vec![] as Vec); + + // Test without mapping + let internal_location = internal::Location { + mapping_id: None, + function_id: FunctionId::from_offset(5), + address: 0x2000, + line: 100, + }; + + let otel_location = datadog_profiling_otel::Location::from(&internal_location); + assert_eq!(otel_location.mapping_index, 0); // 0 represents no mapping + assert_eq!(otel_location.address, 0x2000); + assert_eq!(otel_location.line.len(), 1); + assert_eq!(otel_location.line[0].function_index, 6); + assert_eq!(otel_location.line[0].line, 100); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs b/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs new file mode 100644 index 000000000..351acaba9 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs @@ -0,0 +1,73 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::Mapping as InternalMapping; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Mapping { + fn from(internal_mapping: InternalMapping) -> Self { + Self::from(&internal_mapping) + } +} + +// For references (existing implementation) +impl From<&InternalMapping> for datadog_profiling_otel::Mapping { + fn from(internal_mapping: &InternalMapping) -> Self { + Self { + memory_start: internal_mapping.memory_start, + memory_limit: internal_mapping.memory_limit, + file_offset: internal_mapping.file_offset, + filename_strindex: internal_mapping.filename.to_raw_id() as i32, + attribute_indices: vec![], // Not available in internal Mapping + has_functions: true, // Assume true since we have function information + has_filenames: true, // Assume true since we have filename + has_line_numbers: true, // Assume true since we have line information + has_inline_frames: false, // Not available in internal Mapping + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collections::identifiable::StringId; + + #[test] + fn test_from_internal_mapping() { + // Test basic conversion + let internal_mapping = InternalMapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0x100, + filename: StringId::from_offset(42), + build_id: StringId::from_offset(123), + }; + + let otel_mapping = datadog_profiling_otel::Mapping::from(&internal_mapping); + assert_eq!(otel_mapping.memory_start, 0x1000); + assert_eq!(otel_mapping.memory_limit, 0x2000); + assert_eq!(otel_mapping.file_offset, 0x100); + assert_eq!(otel_mapping.filename_strindex, 42); + assert_eq!(otel_mapping.attribute_indices, vec![] as Vec); + assert!(otel_mapping.has_functions); + assert!(otel_mapping.has_filenames); + assert!(otel_mapping.has_line_numbers); + assert!(!otel_mapping.has_inline_frames); + + // Test with large values + let internal_mapping = InternalMapping { + memory_start: 0x1000000000000000, + memory_limit: 0x2000000000000000, + file_offset: 0x1000000000000000, + filename: StringId::from_offset(999), + build_id: StringId::from_offset(888), + }; + + let otel_mapping = datadog_profiling_otel::Mapping::from(&internal_mapping); + assert_eq!(otel_mapping.memory_start, 0x1000000000000000); + assert_eq!(otel_mapping.memory_limit, 0x2000000000000000); + assert_eq!(otel_mapping.file_offset, 0x1000000000000000); + assert_eq!(otel_mapping.filename_strindex, 999); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/mod.rs b/datadog-profiling/src/internal/profile/otel_emitter/mod.rs new file mode 100644 index 000000000..4929dd31e --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/mod.rs @@ -0,0 +1,20 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! OpenTelemetry emitter for converting datadog-profiling internal types to OpenTelemetry protobuf +//! types +//! +//! This module provides `From` trait implementations for converting internal types to their +//! OpenTelemetry protobuf equivalents. + +pub mod function; +pub mod label; +pub mod location; +pub mod mapping; +pub mod profile; +pub mod stack_trace; + +#[cfg(test)] +mod tests { + include!("profile/tests.rs"); +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs new file mode 100644 index 000000000..134054610 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs @@ -0,0 +1,237 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::{Dedup, Id}; +use crate::internal::profile::otel_emitter::label::convert_label_to_key_value; +use crate::internal::profile::{EncodedProfile, Profile as InternalProfile}; +use crate::iter::{IntoLendingIterator, LendingIterator}; +use anyhow::{Context, Result}; +use datadog_profiling_otel::ProfilesDataExt; +use std::collections::HashMap; + +impl InternalProfile { + /// Converts the profile into OpenTelemetry format + /// + /// * `end_time` - Optional end time of the profile. Passing None will use the current time. + /// * `duration` - Optional duration of the profile. Passing None will try to calculate the + /// duration based on the end time minus the start time, but under anomalous conditions this + /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an + /// earlier time. The duration will be set to zero these cases. + pub fn convert_into_otel( + mut self, + end_time: Option, + duration: Option, + ) -> anyhow::Result { + // Calculate duration using the same logic as encode + let end = end_time.unwrap_or_else(std::time::SystemTime::now); + let start = self.start_time; + let duration_nanos = duration + .unwrap_or_else(|| { + end.duration_since(start).unwrap_or({ + // Let's not throw away the whole profile just because the clocks were wrong. + // todo: log that the clock went backward (or programmer mistake). + std::time::Duration::ZERO + }) + }) + .as_nanos() + .min(i64::MAX as u128) as i64; + + // Create individual OpenTelemetry Profiles for each ValueType + let mut profiles = Vec::with_capacity(self.sample_types.len()); + + for sample_type in self.sample_types.iter() { + // Convert the ValueType to OpenTelemetry format + let otel_sample_type = datadog_profiling_otel::ValueType { + type_strindex: sample_type.r#type.value.to_raw_id() as i32, + unit_strindex: sample_type.unit.value.to_raw_id() as i32, + aggregation_temporality: datadog_profiling_otel::AggregationTemporality::Delta + .into(), + }; + + // Create a Profile for this sample type + let profile = datadog_profiling_otel::Profile { + sample_type: Some(otel_sample_type), + sample: vec![], + time_nanos: self + .start_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as i64, + duration_nanos, // Use calculated duration + period_type: self.period.as_ref().map(|(_, period_type)| { + datadog_profiling_otel::ValueType { + type_strindex: period_type.r#type.value.to_raw_id() as i32, + unit_strindex: period_type.unit.value.to_raw_id() as i32, + aggregation_temporality: + datadog_profiling_otel::AggregationTemporality::Delta.into(), + } + }), + period: self + .period + .map(|(period_value, _)| period_value) + .unwrap_or(0), + comment_strindices: vec![], // We don't have comments + profile_id: vec![], // TODO: Implement when we handle profile IDs + dropped_attributes_count: 0, // We don't drop attributes + original_payload_format: String::new(), // There is no original payload + original_payload: vec![], // There is no original payload + attribute_indices: vec![], // There are currently no attributes at this level + }; + + profiles.push(profile); + } + + // If we have span labels, figure out the corresponding endpoint labels + // Split into two steps to avoid mutating the map we're iterating over + let endpoint_labels: Vec<_> = self + .labels + .iter() + .enumerate() + .filter(|(_, label)| label.get_key() == self.endpoints.local_root_span_id_label) + .filter_map(|(idx, label)| { + self.get_endpoint_for_label(label) + .ok() + .flatten() + .map(|endpoint_label| (idx, endpoint_label)) + }) + .collect(); + + let endpoint_labels_idx: HashMap = endpoint_labels + .into_iter() + .map(|(idx, endpoint_label)| { + let endpoint_idx = self.labels.dedup(endpoint_label); + (idx, endpoint_idx) + }) + .collect(); + + for (sample, timestamp, mut values) in std::mem::take(&mut self.observations).into_iter() { + let stack_index = sample.stacktrace.to_raw_id() as i32; + let label_set = self.get_label_set(sample.labels)?; + let attribute_indicies: Vec<_> = label_set + .iter() + .map(|x| x.to_raw_id() as i32) + .chain( + label_set + .iter() + .find_map(|k| endpoint_labels_idx.get(&(k.to_raw_id() as usize))) + .map(|label| label.to_raw_id() as i32), + ) + .collect(); + let labels = label_set + .iter() + .map(|l| self.get_label(*l).copied()) + .collect::>>()?; + let link_index = 0; // TODO, handle links properly + let timestamps_unix_nano = timestamp.map_or(vec![], |ts| vec![ts.get() as u64]); + self.upscaling_rules.upscale_values(&mut values, &labels)?; + + for (idx, value) in values.iter().enumerate() { + if *value != 0 { + let otel_sample = datadog_profiling_otel::Sample { + stack_index, + attribute_indices: attribute_indicies.clone(), + link_index, + values: vec![*value], + timestamps_unix_nano: timestamps_unix_nano.clone(), + }; + profiles[idx].sample.push(otel_sample); + } + } + } + + // Convert string table using into_lending_iter + // Note: We can't use .map().collect() here because LendingIterator doesn't implement + // the standard Iterator trait. LendingIterator is designed for yielding references + // with lifetimes tied to the iterator itself, so we need to manually iterate and + // convert each string reference to an owned String. + let string_table = { + let mut strings = Vec::with_capacity(self.strings.len()); + let mut iter = self.strings.into_lending_iter(); + while let Some(s) = iter.next() { + strings.push(s.to_string()); + } + strings + }; + + // Convert labels to KeyValues for the attribute table + let mut key_to_unit_map = HashMap::new(); + let mut attribute_table = Vec::with_capacity(self.labels.len()); + + for label in self.labels.iter() { + let key_value = convert_label_to_key_value(label, &string_table, &mut key_to_unit_map) + .with_context(|| { + format!( + "Failed to convert label with key index {}", + label.get_key().to_raw_id() + ) + })?; + attribute_table.push(key_value); + } + + // Build attribute units from the key-to-unit mapping + let attribute_units = key_to_unit_map + .into_iter() + .map( + |(key_index, unit_index)| datadog_profiling_otel::AttributeUnit { + attribute_key_strindex: key_index as i32, + unit_strindex: unit_index as i32, + }, + ) + .collect(); + + // Convert the ProfilesDictionary components + let dictionary = datadog_profiling_otel::ProfilesDictionary { + mapping_table: self.mappings.into_iter().map(From::from).collect(), + location_table: self.locations.into_iter().map(From::from).collect(), + function_table: self.functions.into_iter().map(From::from).collect(), + stack_table: self.stack_traces.into_iter().map(From::from).collect(), + string_table, + attribute_table, + attribute_units, + link_table: vec![], // TODO: Implement when we handle trace links + }; + + // Create a basic ResourceProfiles structure + let resource_profiles = vec![datadog_profiling_otel::ResourceProfiles { + resource: None, // TODO: Implement when we handle resources + scope_profiles: vec![datadog_profiling_otel::ScopeProfiles { + scope: None, // It is legal to leave this unset according to the spec + profiles, // Now contains the individual profiles + schema_url: String::new(), // TODO: Implement when we handle schema URLs + default_sample_type: None, // TODO: Implement when we handle sample types + }], + schema_url: String::new(), // TODO: Implement when we handle schema URLs + }]; + + Ok(datadog_profiling_otel::ProfilesData { + resource_profiles, + dictionary: Some(dictionary), + }) + } + + /// Serializes the profile into OpenTelemetry format and compresses it using zstd. + /// + /// * `end_time` - Optional end time of the profile. Passing None will use the current time. + /// * `duration` - Optional duration of the profile. Passing None will try to calculate the + /// duration based on the end time minus the start time, but under anomalous conditions this + /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an + /// earlier time. The duration will be set to zero these cases. + pub fn serialize_into_compressed_otel( + mut self, + end_time: Option, + duration: Option, + ) -> anyhow::Result { + // Extract values before consuming self + let start = self.start_time; + let endpoints_stats = std::mem::take(&mut self.endpoints.stats); + let otel_profiles_data = self.convert_into_otel(end_time, duration)?; + let buffer = otel_profiles_data.serialize_into_compressed_proto()?; + let end = end_time.unwrap_or_else(std::time::SystemTime::now); + Ok(EncodedProfile { + start, + end, + buffer, + endpoints_stats, + }) + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs new file mode 100644 index 000000000..1d22f6ee2 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs @@ -0,0 +1,406 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::internal; + +// Helper functions for test setup +fn create_basic_function() -> crate::api::Function<'static> { + crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + } +} + +fn create_basic_mapping() -> crate::api::Mapping<'static> { + crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + } +} + +fn setup_profile_with_function_and_location( + sample_types: &[crate::api::ValueType<'static>], +) -> (internal::profile::Profile, crate::api::Location<'static>) { + let mut internal_profile = internal::profile::Profile::new(sample_types, None); + let function = create_basic_function(); + let mapping = create_basic_mapping(); + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + + let _function_id = internal_profile.try_add_function(&function); + let _mapping_id = internal_profile.try_add_mapping(&mapping); + let location_id = internal_profile.try_add_location(&location).unwrap(); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + + (internal_profile, location) +} + +fn create_string_label(key: &'static str, value: &'static str) -> crate::api::Label<'static> { + crate::api::Label { + key, + str: value, + num: 0, + num_unit: "", + } +} + +fn create_numeric_label( + key: &'static str, + value: i64, + unit: &'static str, +) -> crate::api::Label<'static> { + crate::api::Label { + key, + str: "", + num: value, + num_unit: unit, + } +} + +// Common assertion helpers +fn assert_duration_calculation(profiles: &[datadog_profiling_otel::Profile]) { + for profile in profiles { + assert!(profile.duration_nanos > 0); + } +} + +fn assert_profile_has_correct_sample( + profile: &datadog_profiling_otel::Profile, + expected_values: Vec, + expected_stack_index: i32, + expected_attribute_count: usize, +) { + assert_eq!(profile.sample.len(), 1); + let sample = &profile.sample[0]; + assert_eq!(sample.values, expected_values); + assert_eq!(sample.stack_index, expected_stack_index); + assert_eq!(sample.attribute_indices.len(), expected_attribute_count); +} + +fn assert_sample_has_timestamp( + sample: &datadog_profiling_otel::Sample, + expected_timestamp: u64, +) { + assert_eq!(sample.timestamps_unix_nano.len(), 1); + assert_eq!(sample.timestamps_unix_nano[0], expected_timestamp); +} + +fn assert_profiles_data_structure(otel_profiles_data: &datadog_profiling_otel::ProfilesData) { + assert!(otel_profiles_data.dictionary.is_some()); + assert_eq!(otel_profiles_data.resource_profiles.len(), 1); + assert_eq!( + otel_profiles_data.resource_profiles[0].scope_profiles.len(), + 1 + ); +} + +#[test] +fn test_convert_into_otel() { + // Test empty profile + let internal_profile = internal::profile::Profile::new(&[], None); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + assert_profiles_data_structure(&otel_profiles_data); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_eq!(dictionary.mapping_table.len(), 0); + assert_eq!(dictionary.location_table.len(), 0); + assert_eq!(dictionary.function_table.len(), 0); + assert_eq!(dictionary.stack_table.len(), 0); + assert_eq!(dictionary.string_table.len(), 4); + assert_eq!(dictionary.string_table[0], ""); // Empty string + assert_eq!(dictionary.string_table[1], "local root span id"); + assert_eq!(dictionary.string_table[2], "trace endpoint"); + assert_eq!(dictionary.string_table[3], "end_timestamp_ns"); + assert_eq!(dictionary.link_table.len(), 0); + assert_eq!(dictionary.attribute_table.len(), 0); + assert_eq!(dictionary.attribute_units.len(), 0); + + // Test with functions + let mut internal_profile = internal::profile::Profile::new(&[], None); + let function1 = crate::api::Function { + name: "test_function_1", + system_name: "test_system_1", + filename: "test_file_1.rs", + }; + let function2 = crate::api::Function { + name: "test_function_2", + system_name: "test_system_2", + filename: "test_file_2.rs", + }; + let _function1_id = internal_profile.try_add_function(&function1); + let _function2_id = internal_profile.try_add_function(&function2); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_eq!(dictionary.function_table.len(), 2); + assert_eq!(dictionary.string_table.len(), 10); + + // Test with labels + let mut internal_profile = internal::profile::Profile::new(&[], None); + let label1 = create_string_label("thread_id", "main"); + let label2 = create_numeric_label("memory_usage", 1024, "bytes"); + let sample = crate::api::Sample { + locations: vec![], + values: &[42], + labels: vec![label1, label2], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_eq!(dictionary.attribute_table.len(), 2); + assert_eq!(dictionary.attribute_units.len(), 1); + + // Test with sample types + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("allocations", "count"), + ]; + let internal_profile = internal::profile::Profile::new(&sample_types, None); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + assert_eq!(scope_profile.profiles.len(), 2); + assert_duration_calculation(&scope_profile.profiles); +} + +#[test] +fn test_sample_conversion() { + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("memory", "bytes"), + ]; + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); + + // Test basic sample conversion + let sample = crate::api::Sample { + locations: vec![location], + values: &[100, 2048], + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + assert_eq!(scope_profile.profiles.len(), 2); + assert_profile_has_correct_sample(&scope_profile.profiles[0], vec![100], 0, 0); + assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![2048], 0, 0); + + // Test with labels + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); + let sample = crate::api::Sample { + locations: vec![location], + values: &[150], + labels: vec![ + create_string_label("thread_id", "main"), + create_numeric_label("cpu_usage", 75, "percent"), + ], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_profile_has_correct_sample(profile, vec![150], 0, 2); + + // Verify the sample's attribute indices point to correct attributes + let sample = &profile.sample[0]; + let dictionary = &otel_profiles_data.dictionary.as_ref().unwrap(); + + // Check that attribute indices are valid + for &attr_idx in &sample.attribute_indices { + assert!(attr_idx >= 0); + assert!(attr_idx < dictionary.attribute_table.len() as i32); + } + + // Verify the actual attribute content + let attr1 = &dictionary.attribute_table[sample.attribute_indices[0] as usize]; + let attr2 = &dictionary.attribute_table[sample.attribute_indices[1] as usize]; + + // One should be the string label, one should be the numeric label + let (string_attr, numeric_attr) = if attr1.key == "thread_id" { + (attr1, attr2) + } else { + (attr2, attr1) + }; + + // Verify string attribute + assert_eq!(string_attr.key, "thread_id"); + let s = match string_attr.value.as_ref().expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::StringValue(s) => s, + _ => panic!("Expected StringValue"), + }; + assert_eq!(s, "main"); + + // Verify numeric attribute + assert_eq!(numeric_attr.key, "cpu_usage"); + let n = match numeric_attr.value.as_ref().expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, + _ => panic!("Expected IntValue"), + }; + assert_eq!(*n, 75); + + // Verify attribute unit mapping + assert_eq!(dictionary.attribute_units.len(), 1); + let unit = &dictionary.attribute_units[0]; + assert!(unit.attribute_key_strindex > 0); + assert!(unit.unit_strindex > 0); + + // Test with timestamps + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); + let sample = crate::api::Sample { + locations: vec![location], + values: &[200], + labels: vec![], + }; + let timestamp = crate::internal::Timestamp::new(1234567890).unwrap(); + let _ = internal_profile.try_add_sample(sample, Some(timestamp)); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile.sample.len(), 1); + assert_sample_has_timestamp(&profile.sample[0], 1234567890); + + // Test zero value filtering + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); + let sample = crate::api::Sample { + locations: vec![location], + values: &[0, 1024], // 0 nanoseconds, 1024 bytes + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + assert_eq!(scope_profile.profiles[0].sample.len(), 0); // Zero value filtered + assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![1024], 0, 0); + + // Test multiple samples aggregation + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); + let sample1 = crate::api::Sample { + locations: vec![location], + values: &[100], + labels: vec![], + }; + let sample2 = crate::api::Sample { + locations: vec![location], + values: &[200], + labels: vec![], + }; + let sample3 = crate::api::Sample { + locations: vec![location], + values: &[300], + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample1, None); + let _ = internal_profile.try_add_sample(sample2, None); + let _ = internal_profile.try_add_sample(sample3, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile.sample.len(), 1); + assert_eq!(profile.sample[0].values, vec![600]); // 100 + 200 + 300 + + assert_duration_calculation(&scope_profile.profiles); +} + +#[test] +fn test_duration_and_period() { + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + + // Test duration calculation + let internal_profile = internal::profile::Profile::new(&sample_types, None); + let explicit_duration = std::time::Duration::from_secs(5); + let otel_profiles_data = internal_profile + .convert_into_otel(None, Some(explicit_duration)) + .unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile.duration_nanos, 5_000_000_000); + + // Test with explicit end_time + let internal_profile2 = internal::profile::Profile::new(&sample_types, None); + let start_time = internal_profile2.start_time; + let end_time = start_time + std::time::Duration::from_secs(3); + let otel_profiles_data2 = internal_profile2 + .convert_into_otel(Some(end_time), None) + .unwrap(); + let profile2 = &otel_profiles_data2.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile2.duration_nanos, 3_000_000_000); + + // Test with both end_time and duration (duration should take precedence) + let internal_profile3 = internal::profile::Profile::new(&sample_types, None); + let start_time3 = internal_profile3.start_time; + let end_time3 = start_time3 + std::time::Duration::from_secs(10); + let duration3 = std::time::Duration::from_secs(7); + let otel_profiles_data3 = internal_profile3 + .convert_into_otel(Some(end_time3), Some(duration3)) + .unwrap(); + let profile3 = &otel_profiles_data3.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile3.duration_nanos, 7_000_000_000); + + // Test period conversion + let period = crate::api::Period { + r#type: crate::api::ValueType::new("cpu", "cycles"), + value: 1000, + }; + let internal_profile = internal::profile::Profile::new(&sample_types, Some(period)); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert!(profile.period_type.is_some()); + assert_eq!(profile.period, 1000); + + // Test without period + let internal_profile_no_period = internal::profile::Profile::new(&sample_types, None); + let otel_profiles_data_no_period = internal_profile_no_period + .convert_into_otel(None, None) + .unwrap(); + let profile_no_period = + &otel_profiles_data_no_period.resource_profiles[0].scope_profiles[0].profiles[0]; + assert!(profile_no_period.period_type.is_none()); + assert_eq!(profile_no_period.period, 0); +} + +#[test] +#[cfg_attr(miri, ignore)] // Skip this test when running under Miri +fn test_serialize_into_compressed_otel() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); + + // Add a sample + let sample = crate::api::Sample { + locations: vec![location], + values: &[150], + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample, None); + + // Test serialization to compressed OpenTelemetry format + let encoded_profile = internal_profile + .serialize_into_compressed_otel(None, None) + .unwrap(); + + // Verify the encoded profile structure + assert!(encoded_profile.start > std::time::UNIX_EPOCH); + assert!(encoded_profile.end > encoded_profile.start); + assert!(!encoded_profile.buffer.is_empty()); + + // Verify the buffer contains compressed data (should be smaller than uncompressed) + // The compressed buffer should be significantly smaller than a typical uncompressed profile + assert!(encoded_profile.buffer.len() < 10000); // Reasonable upper bound for this small profile + + // Verify endpoints stats are preserved + assert!(encoded_profile.endpoints_stats.is_empty()); // No endpoints added +} + diff --git a/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs b/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs new file mode 100644 index 000000000..5f20dfcb5 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs @@ -0,0 +1,52 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::StackTrace as InternalStackTrace; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Stack { + fn from(internal_stack_trace: InternalStackTrace) -> Self { + Self::from(&internal_stack_trace) + } +} + +// For references (existing implementation) +impl From<&InternalStackTrace> for datadog_profiling_otel::Stack { + fn from(internal_stack_trace: &InternalStackTrace) -> Self { + Self { + location_indices: internal_stack_trace + .locations + .iter() + .map(|location_id| location_id.to_raw_id() as i32) + .collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::internal::LocationId; + + #[test] + fn test_from_internal_stack_trace() { + // Test with locations + let internal_stack_trace = InternalStackTrace { + locations: vec![ + LocationId::from_offset(0), + LocationId::from_offset(1), + LocationId::from_offset(2), + ], + }; + + let otel_stack = datadog_profiling_otel::Stack::from(&internal_stack_trace); + assert_eq!(otel_stack.location_indices, vec![1, 2, 3]); + + // Test with empty locations + let internal_stack_trace = InternalStackTrace { locations: vec![] }; + + let otel_stack = datadog_profiling_otel::Stack::from(&internal_stack_trace); + assert_eq!(otel_stack.location_indices, vec![] as Vec); + } +} diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 1930a739c..1fa6a72d9 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -88,6 +88,7 @@ COPY "datadog-live-debugger-ffi/Cargo.toml" "datadog-live-debugger-ffi/" COPY "datadog-profiling/Cargo.toml" "datadog-profiling/" COPY "datadog-profiling-ffi/Cargo.toml" "datadog-profiling-ffi/" COPY "datadog-profiling-protobuf/Cargo.toml" "datadog-profiling-protobuf/" +COPY "datadog-profiling-otel/Cargo.toml" "datadog-profiling-otel/" COPY "datadog-profiling-replayer/Cargo.toml" "datadog-profiling-replayer/" COPY "datadog-remote-config/Cargo.toml" "datadog-remote-config/" COPY "datadog-sidecar/Cargo.toml" "datadog-sidecar/" @@ -134,6 +135,7 @@ RUN echo \ datadog-profiling-replayer/src/main.rs \ datadog-profiling/benches/interning_strings.rs \ datadog-profiling/benches/main.rs \ + datadog-profiling-otel/src/lib.rs \ tools/sidecar_mockgen/src/bin/sidecar_mockgen.rs \ tools/src/bin/dedup_headers.rs \ datadog-trace-normalization/benches/normalization_utils.rs \ diff --git a/tools/src/lib.rs b/tools/src/lib.rs index c573a0f0d..754c54898 100644 --- a/tools/src/lib.rs +++ b/tools/src/lib.rs @@ -168,9 +168,7 @@ pub mod headers { assert_eq!( matches.len(), expected.len(), - "Expected:\n{:#?}\nActual:\n{:#?}", - expected, - matches + "Expected:\n{expected:#?}\nActual:\n{matches:#?}", ); for (i, m) in matches.iter().enumerate() { assert_eq!(m.str, expected[i]);