From 4b9d3be71d929ab9a6425fd3bb935153cc3249f1 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 11 Aug 2025 09:43:15 -0400 Subject: [PATCH 01/23] [profiling] Support Otel protobuf --- Cargo.lock | 11 + Cargo.toml | 1 + datadog-profiling-otel/Cargo.toml | 22 + datadog-profiling-otel/README.md | 89 ++ datadog-profiling-otel/build.rs | 66 ++ .../examples/basic_usage.rs | 58 ++ .../proto/common/v1/common.proto | 40 + .../proto/resource/v1/resource.proto | 29 + datadog-profiling-otel/profiles.proto | 503 +++++++++++ datadog-profiling-otel/src/lib.rs | 36 + datadog-profiling/Cargo.toml | 1 + datadog-profiling/src/internal/profile/mod.rs | 1 + .../internal/profile/otel_emitter/function.rs | 106 +++ .../internal/profile/otel_emitter/label.rs | 230 +++++ .../internal/profile/otel_emitter/location.rs | 123 +++ .../internal/profile/otel_emitter/mapping.rs | 124 +++ .../src/internal/profile/otel_emitter/mod.rs | 15 + .../internal/profile/otel_emitter/profile.rs | 851 ++++++++++++++++++ .../profile/otel_emitter/stack_trace.rs | 97 ++ 19 files changed, 2403 insertions(+) create mode 100644 datadog-profiling-otel/Cargo.toml create mode 100644 datadog-profiling-otel/README.md create mode 100644 datadog-profiling-otel/build.rs create mode 100644 datadog-profiling-otel/examples/basic_usage.rs create mode 100644 datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto create mode 100644 datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto create mode 100644 datadog-profiling-otel/profiles.proto create mode 100644 datadog-profiling-otel/src/lib.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/function.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/label.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/location.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/mapping.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/mod.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/profile.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs diff --git a/Cargo.lock b/Cargo.lock index 073c14f257..e975496a2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1648,6 +1648,7 @@ dependencies = [ "chrono", "criterion", "datadog-alloc", + "datadog-profiling-otel", "datadog-profiling-protobuf", "ddcommon", "futures", @@ -1691,6 +1692,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "datadog-profiling-otel" +version = "20.0.0" +dependencies = [ + "bolero", + "prost", + "prost-build", + "prost-types", +] + [[package]] name = "datadog-profiling-protobuf" version = "20.0.0" diff --git a/Cargo.toml b/Cargo.toml index f9f9a4080c..8c482ed580 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "datadog-live-debugger-ffi", "datadog-profiling", "datadog-profiling-ffi", + "datadog-profiling-otel", "datadog-profiling-protobuf", "datadog-profiling-replayer", "datadog-remote-config", diff --git a/datadog-profiling-otel/Cargo.toml b/datadog-profiling-otel/Cargo.toml new file mode 100644 index 0000000000..a29c5f6688 --- /dev/null +++ b/datadog-profiling-otel/Cargo.toml @@ -0,0 +1,22 @@ +# Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "datadog-profiling-otel" +rust-version.workspace = true +edition.workspace = true +version.workspace = true +license.workspace = true + +[lib] +bench = false + +[dependencies] +prost = "0.13" +prost-types = "0.13" + +[build-dependencies] +prost-build = "0.13" + +[dev-dependencies] +bolero = "0.13" diff --git a/datadog-profiling-otel/README.md b/datadog-profiling-otel/README.md new file mode 100644 index 0000000000..3f1b6af71c --- /dev/null +++ b/datadog-profiling-otel/README.md @@ -0,0 +1,89 @@ +# datadog-profiling-otel + +This module provides Rust bindings for the OpenTelemetry profiling protobuf definitions, generated using the `prost` library. + +## Usage + +### Basic Setup + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +datadog-profiling-otel = "20.0.0" +``` + +### Creating Profile Data + +```rust +use datadog_profiling_otel::*; + +// Create a profiles dictionary +let mut profiles_dict = ProfilesDictionary::default(); +profiles_dict.string_table.push("cpu".to_string()); +profiles_dict.string_table.push("nanoseconds".to_string()); + +// Create a sample type +let sample_type = ValueType { + type_strindex: 0, // "cpu" + unit_strindex: 1, // "nanoseconds" + aggregation_temporality: AggregationTemporality::Delta.into(), +}; + +// Create a profile +let mut profile = Profile::default(); +profile.sample_type = Some(sample_type); +profile.time_nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as i64; + +// Assemble the complete profiles data +let mut profiles_data = ProfilesData::default(); +profiles_data.dictionary = Some(profiles_dict); + +let mut scope_profiles = ScopeProfiles::default(); +scope_profiles.profiles.push(profile); + +let mut resource_profiles = ResourceProfiles::default(); +resource_profiles.scope_profiles.push(scope_profiles); + +profiles_data.resource_profiles.push(resource_profiles); +``` + +### Running Examples + +```bash +# Run the basic usage example +cargo run --example basic_usage + +# Run tests +cargo test + +# Build +cargo build +``` + +## Module Structure + +The generated code follows the OpenTelemetry protobuf structure: + +- `ProfilesData`: Top-level container for all profile data +- `ResourceProfiles`: Profiles grouped by resource +- `ScopeProfiles`: Profiles grouped by instrumentation scope +- `Profile`: Individual profile with samples and metadata +- `ProfilesDictionary`: Shared data (strings, mappings, locations, etc.) +- `Sample`: Individual measurements with stack traces +- `Location`: Function locations in the call stack +- `Function`: Function information +- `Mapping`: Binary/library mapping information + +## Dependencies + +- `prost`: Protobuf implementation +- `prost-types`: Additional protobuf types +- `prost-build`: Build-time protobuf compilation + +## License + +Apache-2.0 diff --git a/datadog-profiling-otel/build.rs b/datadog-profiling-otel/build.rs new file mode 100644 index 0000000000..53dcd28deb --- /dev/null +++ b/datadog-profiling-otel/build.rs @@ -0,0 +1,66 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::env; +use std::path::PathBuf; + +fn main() { + // Tell Cargo to rerun this build script if the proto files change + println!("cargo:rerun-if-changed=profiles.proto"); + println!("cargo:rerun-if-changed=opentelemetry/proto/common/v1/common.proto"); + println!("cargo:rerun-if-changed=opentelemetry/proto/resource/v1/resource.proto"); + + // Create the output directory + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + // Configure prost-build + let mut config = prost_build::Config::new(); + config.out_dir(&out_dir); + + // Compile the proto files - include all proto files so imports work correctly + config + .compile_protos( + &[ + "opentelemetry/proto/common/v1/common.proto", + "opentelemetry/proto/resource/v1/resource.proto", + "profiles.proto", + ], + &["."], + ) + .expect("Failed to compile protobuf files"); + + // Generate the module file with correct structure + let module_content = r#" +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +// This module is generated by prost-build from profiles.proto +#[allow(clippy::all)] +pub mod opentelemetry { + pub mod proto { + pub mod common { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/opentelemetry.proto.common.v1.rs")); + } + } + pub mod resource { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/opentelemetry.proto.resource.v1.rs")); + } + } + pub mod profiles { + pub mod v1development { + include!(concat!(env!("OUT_DIR"), "/opentelemetry.proto.profiles.v1development.rs")); + } + } + } +} + +// Re-export commonly used types +pub use opentelemetry::proto::profiles::v1development::*; +pub use opentelemetry::proto::common::v1::*; +pub use opentelemetry::proto::resource::v1::*; +"#; + + std::fs::write(out_dir.join("mod.rs"), module_content).expect("Failed to write module file"); +} diff --git a/datadog-profiling-otel/examples/basic_usage.rs b/datadog-profiling-otel/examples/basic_usage.rs new file mode 100644 index 0000000000..a9678b4c87 --- /dev/null +++ b/datadog-profiling-otel/examples/basic_usage.rs @@ -0,0 +1,58 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +// This example demonstrates how to use the generated protobuf code +// from the OpenTelemetry profiles.proto file. + +fn main() { + use datadog_profiling_otel::*; + + // Create a simple profile with some basic data + let mut profiles_dict = ProfilesDictionary::default(); + + // Add some strings to the string table + profiles_dict.string_table.push("cpu".to_string()); + profiles_dict.string_table.push("nanoseconds".to_string()); + profiles_dict.string_table.push("main".to_string()); + + // Create a sample type + let sample_type = ValueType { + type_strindex: 0, // "cpu" + unit_strindex: 1, // "nanoseconds" + aggregation_temporality: AggregationTemporality::Delta.into(), + }; + + // Create a profile + let profile = Profile { + sample_type: Some(sample_type), + time_nanos: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as i64, + ..Default::default() + }; + + // Create profiles data + let mut profiles_data = ProfilesData { + dictionary: Some(profiles_dict), + ..Default::default() + }; + + let mut scope_profiles = ScopeProfiles::default(); + scope_profiles.profiles.push(profile); + + let mut resource_profiles = ResourceProfiles::default(); + resource_profiles.scope_profiles.push(scope_profiles); + + profiles_data.resource_profiles.push(resource_profiles); + + println!("Successfully created OpenTelemetry profile data!"); + println!( + "Profile contains {} resource profiles", + profiles_data.resource_profiles.len() + ); + println!( + "Time: {}", + profiles_data.resource_profiles[0].scope_profiles[0].profiles[0].time_nanos + ); +} diff --git a/datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto b/datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto new file mode 100644 index 0000000000..5137e6d559 --- /dev/null +++ b/datadog-profiling-otel/opentelemetry/proto/common/v1/common.proto @@ -0,0 +1,40 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.common.v1; + +// KeyValue is a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValue { + string key = 1; + oneof value { + string string_value = 2; + bool bool_value = 3; + int64 int_value = 4; + double double_value = 5; + bytes bytes_value = 6; + } +} + +// InstrumentationScope is a message representing the instrumentation scope information +// such as the fully qualified name and version. +message InstrumentationScope { + // An empty instrumentation scope name means the name is unknown. + string name = 1; + string version = 2; + repeated KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; +} diff --git a/datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto b/datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto new file mode 100644 index 0000000000..06dbb80329 --- /dev/null +++ b/datadog-profiling-otel/opentelemetry/proto/resource/v1/resource.proto @@ -0,0 +1,29 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.resource.v1; + +import "opentelemetry/proto/common/v1/common.proto"; + +// Resource information. +message Resource { + // Set of attributes that describe the resource. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 2; +} diff --git a/datadog-profiling-otel/profiles.proto b/datadog-profiling-otel/profiles.proto new file mode 100644 index 0000000000..060ff86df3 --- /dev/null +++ b/datadog-profiling-otel/profiles.proto @@ -0,0 +1,503 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file includes work covered by the following copyright and permission notices: +// +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1development; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Development"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.profiles.v1development"; +option java_outer_classname = "ProfilesProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1development"; + +// Relationships Diagram +// +// ┌──────────────────┐ LEGEND +// │ ProfilesData │ ─────┐ +// └──────────────────┘ │ ─────▶ embedded +// │ │ +// │ 1-n │ ─────▷ referenced by index +// ▼ ▼ +// ┌──────────────────┐ ┌────────────────────┐ +// │ ResourceProfiles │ │ ProfilesDictionary │ +// └──────────────────┘ └────────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ ScopeProfiles │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▼ +// ┌──────────────────┐ +// │ Profile │ +// └──────────────────┘ +// │ n-1 +// │ 1-n ┌───────────────────────────────────────┐ +// ▼ │ ▽ +// ┌──────────────────┐ 1-n ┌──────────────┐ ┌──────────┐ +// │ Sample │ ──────▷ │ KeyValue │ │ Link │ +// └──────────────────┘ └──────────────┘ └──────────┘ +// │ 1-n △ △ +// │ 1-n ┌─────────────────┘ │ 1-n +// ▽ │ │ +// ┌──────────────────┐ n-1 ┌──────────────┐ +// │ Location │ ──────▷ │ Mapping │ +// └──────────────────┘ └──────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ Line │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▽ +// ┌──────────────────┐ +// │ Function │ +// └──────────────────┘ +// + +// ProfilesDictionary represents the profiles data shared across the +// entire message being sent. +message ProfilesDictionary { + // Mappings from address ranges to the image/binary/library mapped + // into that address range referenced by locations via Location.mapping_index. + // mapping_table[0] must always be set to a zero value default mapping, + // so that _index fields can use 0 to indicate null/unset. + repeated Mapping mapping_table = 1; + + // Locations referenced by samples via Stack.location_indices. + repeated Location location_table = 2; + + // Functions referenced by locations via Line.function_index. + repeated Function function_table = 3; + + // Links referenced by samples via Sample.link_index. + // link_table[0] must always be set to a zero value default link, + // so that _index fields can use 0 to indicate null/unset. + repeated Link link_table = 4; + + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 5; + + // A common table for attributes referenced by various messages. + repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 6; + + // Represents a mapping between Attribute Keys and Units. + repeated AttributeUnit attribute_units = 7; + + // Stacks referenced by samples via Sample.stack_index. + repeated Stack stack_table = 8; +} + +// ProfilesData represents the profiles data that can be stored in persistent storage, +// OR can be embedded by other protocols that transfer OTLP profiles data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message ProfilesData { + // An array of ResourceProfiles. + // For data coming from an SDK profiler, this array will typically contain one + // element. Host-level profilers will usually create one ResourceProfile per + // container, as well as one additional ResourceProfile grouping all samples + // from non-containerized processes. + // Other resource groupings are possible as well and clarified via + // Resource.attributes and semantic conventions. + repeated ResourceProfiles resource_profiles = 1; + + // One instance of ProfilesDictionary + ProfilesDictionary dictionary = 2; +} + + +// A collection of ScopeProfiles from a Resource. +message ResourceProfiles { + reserved 1000; + + // The resource for the profiles in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeProfiles that originate from a resource. + repeated ScopeProfiles scope_profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_profiles" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Profiles produced by an InstrumentationScope. +message ScopeProfiles { + // The instrumentation scope information for the profiles in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Profiles that originate from an instrumentation scope. + repeated Profile profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the profile data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all profiles in the "profiles" field. + string schema_url = 3; + + // The preferred type and unit of Samples in at least one Profile. + // See Profile.sample_type for possible values. + ValueType default_sample_type = 4; +} + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - The profile is represented as a set of samples, where each sample +// references a stack trace which is a list of locations, each belonging +// to a mapping. +// - There is a N->1 relationship from Stack.location_indices entries to +// locations. For every Stack.location_indices entry there must be a +// unique Location with that index. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that index. + +// Represents a complete profile, including sample types, samples, mappings to +// binaries, stacks, locations, functions, string table, and additional +// metadata. It modifies and annotates pprof Profile with OpenTelemetry +// specific fields. +// +// Note that whilst fields in this message retain the name and field id from pprof in most cases +// for ease of understanding data migration, it is not intended that pprof:Profile and +// OpenTelemetry:Profile encoding be wire compatible. +message Profile { + // The type and unit of all Sample.values in this profile. + // For a cpu or off-cpu profile this might be: + // ["cpu","nanoseconds"] or ["off_cpu","nanoseconds"] + // For a heap profile, this might be: + // ["allocated_objects","count"] or ["allocated_space","bytes"], + ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + + // The following fields 3-13 are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 3; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 4; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 5; + // The number of events between sampled occurrences. + int64 period = 6; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int32 comment_strindices = 7; // Indices into ProfilesDictionary.string_table. + + // A globally unique identifier for a profile. The ID is a 16-byte array. An ID with + // all zeroes is considered invalid. It may be used for deduplication and signal + // correlation purposes. It is acceptable to treat two profiles with different values + // in this field as not equal, even if they represented the same object at an earlier + // time. + // This field is optional; an ID may be assigned to an ID-less profile in a later step. + bytes profile_id = 8; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 9; + + // Specifies format of the original payload. Common values are defined in semantic conventions. [required if original_payload is present] + string original_payload_format = 10; + + // Original payload can be stored in this field. This can be useful for users who want to get the original payload. + // Formats such as JFR are highly extensible and can contain more information than what is defined in this spec. + // Inclusion of original payload should be configurable by the user. Default behavior should be to not include the original payload. + // If the original payload is in pprof format, it SHOULD not be included in this field. + // The field is optional, however if it is present then equivalent converted data should be populated in other fields + // of this message as far as is practicable. + bytes original_payload = 11; + + // References to attributes in attribute_table. [optional] + // It is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated int32 attribute_indices = 12; +} + +// Represents a mapping between Attribute Keys and Units. +message AttributeUnit { + // Index into string table. + int32 attribute_key_strindex = 1; + // Index into string table. + int32 unit_strindex = 2; +} + +// A pointer from a profile Sample to a trace Span. +// Connects a profile sample to a trace span, identified by unique trace and span IDs. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// Specifies the method of aggregating metric values, either DELTA (change since last report) +// or CUMULATIVE (total since a fixed start time). +enum AggregationTemporality { + /* UNSPECIFIED is the default AggregationTemporality, it MUST not be used. */ + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + + /** DELTA is an AggregationTemporality for a profiler which reports + changes since last report time. Successive metrics contain aggregation of + values from continuous and non-overlapping intervals. + + The values for a DELTA metric are based only on the time interval + associated with one measurement cycle. There is no dependency on + previous measurements like is the case for CUMULATIVE metrics. + + For example, consider a system measuring the number of requests that + it receives and reports the sum of these requests every second as a + DELTA metric: + + 1. The system starts receiving at time=t_0. + 2. A request is received, the system measures 1 request. + 3. A request is received, the system measures 1 request. + 4. A request is received, the system measures 1 request. + 5. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+1 with a value of 3. + 6. A request is received, the system measures 1 request. + 7. A request is received, the system measures 1 request. + 8. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0+1 to + t_0+2 with a value of 2. */ + AGGREGATION_TEMPORALITY_DELTA = 1; + + /** CUMULATIVE is an AggregationTemporality for a profiler which + reports changes since a fixed start time. This means that current values + of a CUMULATIVE metric depend on all previous measurements since the + start time. Because of this, the sender is required to retain this state + in some form. If this state is lost or invalidated, the CUMULATIVE metric + values MUST be reset and a new fixed start time following the last + reported measurement time sent MUST be used. + + For example, consider a system measuring the number of requests that + it receives and reports the sum of these requests every second as a + CUMULATIVE metric: + + 1. The system starts receiving at time=t_0. + 2. A request is received, the system measures 1 request. + 3. A request is received, the system measures 1 request. + 4. A request is received, the system measures 1 request. + 5. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+1 with a value of 3. + 6. A request is received, the system measures 1 request. + 7. A request is received, the system measures 1 request. + 8. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+2 with a value of 5. + 9. The system experiences a fault and loses state. + 10. The system recovers and resumes receiving at time=t_1. + 11. A request is received, the system measures 1 request. + 12. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_1 to + t_1+1 with a value of 1. + + Note: Even though, when reporting changes since last report time, using + CUMULATIVE is valid, it is not recommended. */ + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +// ValueType describes the type and units of a value, with an optional aggregation temporality. +message ValueType { + int32 type_strindex = 1; // Index into ProfilesDictionary.string_table. + int32 unit_strindex = 2; // Index into ProfilesDictionary.string_table. + + AggregationTemporality aggregation_temporality = 3; +} + +// Each Sample records values encountered in some program context. The program +// context is typically a stack trace, perhaps augmented with auxiliary +// information like the thread-id, some indicator of a higher level request +// being handled etc. +// +// A Sample MUST have have at least one values or timestamps_unix_nano entry. If +// both fields are populated, they MUST contain the same number of elements, and +// the elements at the same index MUST refer to the same event. +// +// Examples of different ways of representing a sample with the total value of 10: +// +// Report of a stacktrace at 10 timestamps (consumers must assume the value is 1 for each point): +// values: [] +// timestamps_unix_nano: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +// +// Report of a stacktrace with an aggregated value without timestamps: +// values: [10] +// timestamps_unix_nano: [] +// +// Report of a stacktrace at 4 timestamps where each point records a specific value: +// values: [2, 2, 3, 3] +// timestamps_unix_nano: [1, 2, 3, 4] +message Sample { + // Reference to stack in ProfilesDictionary.stack_table. + int32 stack_index = 1; + // The type and unit of each value is defined by Profile.sample_type. + repeated int64 values = 2; + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 3; + + // Reference to link in ProfilesDictionary.link_table. [optional] + // It can be unset / set to 0 if no link exists, as link_table[0] is always a 'null' default value. + int32 link_index = 4; + + // Timestamps associated with Sample represented in nanoseconds. These + // timestamps should fall within the Profile's time range. + repeated uint64 timestamps_unix_nano = 5; +} + +// Describes the mapping of a binary in memory, including its address range, +// file offset, and metadata like build ID +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int32 filename_strindex = 4; // Index into ProfilesDictionary.string_table. + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 5; + // The following fields indicate the resolution of symbolic info. + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; +} + +// A Stack represents a stack trace as a list of locations. The first location +// is the leaf frame. +message Stack { + // References to locations in ProfilesDictionary.location_table. + repeated int32 location_indices = 1; +} + +// Describes function and line table debug information. +message Location { + // Reference to mapping in ProfilesDictionary.mapping_table. + // It can be unset / set to 0 if the mapping is unknown or not applicable for + // this profile type, as mapping_table[0] is always a 'null' default mapping. + int32 mapping_index = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; + + // References to attributes in ProfilesDictionary.attribute_table. [optional] + repeated int32 attribute_indices = 5; +} + +// Details a specific line in a source code, linked to a function. +message Line { + // Reference to function in ProfilesDictionary.function_table. + int32 function_index = 1; + // Line number in source code. 0 means unset. + int64 line = 2; + // Column number in source code. 0 means unset. + int64 column = 3; +} + +// Describes a function, including its human-readable name, system name, +// source file, and starting line number in the source. +message Function { + // Function name. Empty string if not available. + int32 name_strindex = 1; + // Function name, as identified by the system. For instance, + // it can be a C++ mangled name. Empty string if not available. + int32 system_name_strindex = 2; + // Source file containing the function. Empty string if not available. + int32 filename_strindex = 3; + // Line number in source file. 0 means unset. + int64 start_line = 4; +} \ No newline at end of file diff --git a/datadog-profiling-otel/src/lib.rs b/datadog-profiling-otel/src/lib.rs new file mode 100644 index 0000000000..e8ec73d86e --- /dev/null +++ b/datadog-profiling-otel/src/lib.rs @@ -0,0 +1,36 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] +#![cfg_attr(not(test), deny(clippy::unimplemented))] + +pub mod proto { + include!(concat!(env!("OUT_DIR"), "/mod.rs")); +} + +pub use proto::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_proto_generation() { + // Test that we can create basic protobuf messages + let mut profiles_dict = ProfilesDictionary::default(); + profiles_dict.string_table.push("test".to_string()); + + let profiles_data = ProfilesData { + dictionary: Some(profiles_dict), + ..Default::default() + }; + + // Verify the data was set correctly + assert_eq!( + profiles_data.dictionary.as_ref().unwrap().string_table[0], + "test" + ); + } +} diff --git a/datadog-profiling/Cargo.toml b/datadog-profiling/Cargo.toml index 92c6acd2f2..adad70082f 100644 --- a/datadog-profiling/Cargo.toml +++ b/datadog-profiling/Cargo.toml @@ -24,6 +24,7 @@ byteorder = { version = "1.5", features = ["std"] } bytes = "1.1" chrono = {version = "0.4", default-features = false, features = ["std", "clock"]} datadog-alloc = { path = "../datadog-alloc" } +datadog-profiling-otel = { path = "../datadog-profiling-otel" } datadog-profiling-protobuf = { path = "../datadog-profiling-protobuf", features = ["prost_impls"] } ddcommon = {path = "../ddcommon" } futures = { version = "0.3", default-features = false } diff --git a/datadog-profiling/src/internal/profile/mod.rs b/datadog-profiling/src/internal/profile/mod.rs index e9f87fab36..f258e07768 100644 --- a/datadog-profiling/src/internal/profile/mod.rs +++ b/datadog-profiling/src/internal/profile/mod.rs @@ -5,6 +5,7 @@ mod fuzz_tests; pub mod interning_api; +pub mod otel_emitter; use self::api::UpscalingInfo; use super::*; diff --git a/datadog-profiling/src/internal/profile/otel_emitter/function.rs b/datadog-profiling/src/internal/profile/otel_emitter/function.rs new file mode 100644 index 0000000000..78867d24b5 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/function.rs @@ -0,0 +1,106 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::Function as InternalFunction; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Function { + fn from(internal_function: InternalFunction) -> Self { + Self::from(&internal_function) + } +} + +// For references (existing implementation) +impl From<&InternalFunction> for datadog_profiling_otel::Function { + fn from(internal_function: &InternalFunction) -> Self { + Self { + name_strindex: internal_function.name.to_raw_id() as i32, + system_name_strindex: internal_function.system_name.to_raw_id() as i32, + filename_strindex: internal_function.filename.to_raw_id() as i32, + start_line: 0, // Not available in internal Function + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collections::identifiable::StringId; + + #[test] + fn test_from_internal_function() { + // Create an internal function + let internal_function = InternalFunction { + name: StringId::from_offset(0), + system_name: StringId::from_offset(1), + filename: StringId::from_offset(2), + }; + + // Convert to OpenTelemetry Function + let otel_function = datadog_profiling_otel::Function::from(&internal_function); + + // Verify the conversion - note: StringId doesn't add 1, it's direct conversion + assert_eq!(otel_function.name_strindex, 0); + assert_eq!(otel_function.system_name_strindex, 1); + assert_eq!(otel_function.filename_strindex, 2); + assert_eq!(otel_function.start_line, 0); + } + + #[test] + fn test_from_internal_function_with_large_offsets() { + // Create an internal function with large offsets + let internal_function = InternalFunction { + name: StringId::from_offset(999999), + system_name: StringId::from_offset(888888), + filename: StringId::from_offset(777777), + }; + + // Convert to OpenTelemetry Function + let otel_function = datadog_profiling_otel::Function::from(&internal_function); + + // Verify the conversion + assert_eq!(otel_function.name_strindex, 999999); + assert_eq!(otel_function.system_name_strindex, 888888); + assert_eq!(otel_function.filename_strindex, 777777); + assert_eq!(otel_function.start_line, 0); + } + + #[test] + fn test_into_otel_function() { + // Create an internal function + let internal_function = InternalFunction { + name: StringId::from_offset(100), + system_name: StringId::from_offset(200), + filename: StringId::from_offset(300), + }; + + // Convert using .into() method + let otel_function: datadog_profiling_otel::Function = (&internal_function).into(); + + // Verify the conversion + assert_eq!(otel_function.name_strindex, 100); + assert_eq!(otel_function.system_name_strindex, 200); + assert_eq!(otel_function.filename_strindex, 300); + assert_eq!(otel_function.start_line, 0); + } + + #[test] + fn test_into_otel_function_owned() { + // Create an internal function + let internal_function = InternalFunction { + name: StringId::from_offset(400), + system_name: StringId::from_offset(500), + filename: StringId::from_offset(600), + }; + + // Convert using .into() method with owned value + let otel_function: datadog_profiling_otel::Function = internal_function.into(); + + // Verify the conversion + assert_eq!(otel_function.name_strindex, 400); + assert_eq!(otel_function.system_name_strindex, 500); + assert_eq!(otel_function.filename_strindex, 600); + assert_eq!(otel_function.start_line, 0); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/label.rs b/datadog-profiling/src/internal/profile/otel_emitter/label.rs new file mode 100644 index 0000000000..0824ed8ae1 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/label.rs @@ -0,0 +1,230 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! All the mess here will go away once https://github.com/open-telemetry/opentelemetry-proto/pull/672/files is merged. + +use crate::collections::identifiable::Id; +use crate::internal::Label as InternalLabel; +use anyhow::{Context, Result}; +use std::collections::HashMap; + +/// Converts a datadog-profiling internal Label to an OpenTelemetry KeyValue +/// +/// # Arguments +/// * `label` - The internal label to convert +/// * `string_table` - A slice of strings where StringIds index into +/// * `key_to_unit_map` - A mutable map from key string index to unit string index for numeric +/// labels +/// +/// # Returns +/// * `Ok(KeyValue)` if the conversion is successful +/// * `Err` with context if the StringIds are out of bounds of the string table +pub fn convert_label_to_key_value( + label: &InternalLabel, + string_table: &[String], + key_to_unit_map: &mut HashMap, +) -> Result { + // Get the key string + let key_id = label.get_key().to_raw_id() as usize; + let key = string_table + .get(key_id) + .with_context(|| { + format!( + "Key string index {} out of bounds (string table has {} elements)", + key_id, + string_table.len() + ) + })? + .to_string(); + + match label.get_value() { + crate::internal::LabelValue::Str(str_id) => { + let str_value_id = str_id.to_raw_id() as usize; + let str_value = string_table + .get(str_value_id) + .with_context(|| { + format!( + "Value string index {} out of bounds (string table has {} elements)", + str_value_id, + string_table.len() + ) + })? + .to_string(); + + Ok(datadog_profiling_otel::KeyValue { + key, + value: Some(datadog_profiling_otel::key_value::Value::StringValue( + str_value, + )), + }) + } + crate::internal::LabelValue::Num { num, num_unit } => { + // Note: OpenTelemetry KeyValue doesn't support units, so we only store the numeric + // value But we track the mapping for building the attribute_units table + let key_index = label.get_key().to_raw_id() as usize; + let unit_index = num_unit.to_raw_id() as usize; + + // Only add to the map if the unit is not the default empty string (index 0) + if unit_index > 0 { + key_to_unit_map.insert(key_index, unit_index); + } + + Ok(datadog_profiling_otel::KeyValue { + key, + value: Some(datadog_profiling_otel::key_value::Value::IntValue(*num)), + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collections::identifiable::StringId; + + #[test] + fn test_convert_string_label() { + let string_table = vec![ + "".to_string(), // index 0 + "thread_id".to_string(), // index 1 + "main".to_string(), // index 2 + ]; + + let label = InternalLabel::str( + StringId::from_offset(1), // "thread_id" + StringId::from_offset(2), // "main" + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_ok()); + + let key_value = result.unwrap(); + assert_eq!(key_value.key, "thread_id"); + match key_value.value { + Some(datadog_profiling_otel::key_value::Value::StringValue(s)) => { + assert_eq!(s, "main"); + } + _ => panic!("Expected StringValue"), + } + } + + #[test] + fn test_convert_numeric_label() { + let string_table = vec![ + "".to_string(), // index 0 + "allocation_size".to_string(), // index 1 + "bytes".to_string(), // index 2 + ]; + + let label = InternalLabel::num( + StringId::from_offset(1), // "allocation_size" + 1024, // 1024 bytes + StringId::from_offset(2), // "bytes" + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_ok()); + + let key_value = result.unwrap(); + assert_eq!(key_value.key, "allocation_size"); + match key_value.value { + Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { + assert_eq!(n, 1024); + } + _ => panic!("Expected IntValue"), + } + } + + #[test] + fn test_convert_label_out_of_bounds() { + let string_table = vec!["".to_string()]; // Only one string + + let label = InternalLabel::str( + StringId::from_offset(1), // This index doesn't exist + StringId::from_offset(0), // This index exists + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_err()); + } + + #[test] + fn test_convert_label_empty_string_table() { + let string_table: Vec = vec![]; + + let label = InternalLabel::str( + StringId::from_offset(0), // Even index 0 is out of bounds + StringId::from_offset(0), + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_err()); + } + + #[test] + fn test_convert_numeric_label_with_unit_mapping() { + let string_table = vec![ + "".to_string(), // index 0 + "memory_usage".to_string(), // index 1 + "megabytes".to_string(), // index 2 + ]; + + let label = InternalLabel::num( + StringId::from_offset(1), // "memory_usage" + 512, // 512 MB + StringId::from_offset(2), // "megabytes" + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_ok()); + + // Verify the KeyValue conversion + let key_value = result.unwrap(); + assert_eq!(key_value.key, "memory_usage"); + match key_value.value { + Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { + assert_eq!(n, 512); + } + _ => panic!("Expected IntValue"), + } + + // Verify the unit mapping was added + assert_eq!(key_to_unit_map.get(&1), Some(&2)); // key index 1 maps to unit index 2 + } + + #[test] + fn test_convert_numeric_label_without_unit_mapping() { + let string_table = vec![ + "".to_string(), // index 0 + "counter".to_string(), // index 1 + ]; + + let label = InternalLabel::num( + StringId::from_offset(1), // "counter" + 42, // 42 + StringId::from_offset(0), // empty string (default) + ); + + let mut key_to_unit_map = HashMap::new(); + let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); + assert!(result.is_ok()); + + // Verify the KeyValue conversion + let key_value = result.unwrap(); + assert_eq!(key_value.key, "counter"); + match key_value.value { + Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { + assert_eq!(n, 42); + } + _ => panic!("Expected IntValue"), + } + + // Verify no unit mapping was added for default empty string + assert!(key_to_unit_map.is_empty()); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/location.rs b/datadog-profiling/src/internal/profile/otel_emitter/location.rs new file mode 100644 index 0000000000..27e137bffc --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/location.rs @@ -0,0 +1,123 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::Location as InternalLocation; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Location { + fn from(internal_location: InternalLocation) -> Self { + Self::from(&internal_location) + } +} + +// For references (existing implementation) +impl From<&InternalLocation> for datadog_profiling_otel::Location { + fn from(internal_location: &InternalLocation) -> Self { + Self { + mapping_index: internal_location + .mapping_id + .map(|id| id.to_raw_id() as i32) + .unwrap_or(0), // 0 represents no mapping + address: internal_location.address, + line: vec![datadog_profiling_otel::Line { + function_index: internal_location.function_id.to_raw_id() as i32, + line: internal_location.line, + column: 0, // Not available in internal Location + }], + is_folded: false, // Not available in internal Location + attribute_indices: vec![], // Not available in internal Location + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::internal::{FunctionId, MappingId}; + + #[test] + fn test_from_internal_location() { + // Create an internal location + let internal_location = InternalLocation { + mapping_id: Some(MappingId::from_offset(1)), + function_id: FunctionId::from_offset(2), + address: 0x1000, + line: 42, + }; + + // Convert to OpenTelemetry Location + let otel_location = datadog_profiling_otel::Location::from(&internal_location); + + // Verify the conversion - note: from_offset adds 1 to avoid zero values + assert_eq!(otel_location.mapping_index, 2); + assert_eq!(otel_location.address, 0x1000); + assert_eq!(otel_location.line.len(), 1); + assert_eq!(otel_location.line[0].function_index, 3); + assert_eq!(otel_location.line[0].line, 42); + assert_eq!(otel_location.line[0].column, 0); + assert!(!otel_location.is_folded); + assert_eq!(otel_location.attribute_indices, vec![] as Vec); + } + + #[test] + fn test_from_internal_location_no_mapping() { + // Create an internal location without mapping + let internal_location = InternalLocation { + mapping_id: None, + function_id: FunctionId::from_offset(5), + address: 0x2000, + line: 100, + }; + + // Convert to OpenTelemetry Location + let otel_location = datadog_profiling_otel::Location::from(&internal_location); + + // Verify the conversion + assert_eq!(otel_location.mapping_index, 0); // 0 represents no mapping + assert_eq!(otel_location.address, 0x2000); + assert_eq!(otel_location.line.len(), 1); + assert_eq!(otel_location.line[0].function_index, 6); + assert_eq!(otel_location.line[0].line, 100); + } + + #[test] + fn test_into_otel_location() { + // Create an internal location + let internal_location = InternalLocation { + mapping_id: Some(MappingId::from_offset(10)), + function_id: FunctionId::from_offset(20), + address: 0x3000, + line: 200, + }; + + // Convert using .into() method + let otel_location: datadog_profiling_otel::Location = (&internal_location).into(); + + // Verify the conversion - note: from_offset adds 1 to avoid zero values + assert_eq!(otel_location.mapping_index, 11); + assert_eq!(otel_location.address, 0x3000); + assert_eq!(otel_location.line[0].function_index, 21); + assert_eq!(otel_location.line[0].line, 200); + } + + #[test] + fn test_into_otel_location_owned() { + // Create an internal location + let internal_location = InternalLocation { + mapping_id: Some(MappingId::from_offset(30)), + function_id: FunctionId::from_offset(40), + address: 0x4000, + line: 300, + }; + + // Convert using .into() method with owned value + let otel_location: datadog_profiling_otel::Location = internal_location.into(); + + // Verify the conversion - note: from_offset adds 1 to avoid zero values + assert_eq!(otel_location.mapping_index, 31); + assert_eq!(otel_location.address, 0x4000); + assert_eq!(otel_location.line[0].function_index, 41); + assert_eq!(otel_location.line[0].line, 300); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs b/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs new file mode 100644 index 0000000000..769f5dbdf7 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs @@ -0,0 +1,124 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::Mapping as InternalMapping; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Mapping { + fn from(internal_mapping: InternalMapping) -> Self { + Self::from(&internal_mapping) + } +} + +// For references (existing implementation) +impl From<&InternalMapping> for datadog_profiling_otel::Mapping { + fn from(internal_mapping: &InternalMapping) -> Self { + Self { + memory_start: internal_mapping.memory_start, + memory_limit: internal_mapping.memory_limit, + file_offset: internal_mapping.file_offset, + filename_strindex: internal_mapping.filename.to_raw_id() as i32, + attribute_indices: vec![], // Not available in internal Mapping + has_functions: true, // Assume true since we have function information + has_filenames: true, // Assume true since we have filename + has_line_numbers: true, // Assume true since we have line information + has_inline_frames: false, // Not available in internal Mapping + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collections::identifiable::StringId; + + #[test] + fn test_from_internal_mapping() { + // Create an internal mapping + let internal_mapping = InternalMapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0x100, + filename: StringId::from_offset(42), + build_id: StringId::from_offset(123), + }; + + // Convert to OpenTelemetry Mapping + let otel_mapping = datadog_profiling_otel::Mapping::from(&internal_mapping); + + // Verify the conversion + assert_eq!(otel_mapping.memory_start, 0x1000); + assert_eq!(otel_mapping.memory_limit, 0x2000); + assert_eq!(otel_mapping.file_offset, 0x100); + assert_eq!(otel_mapping.filename_strindex, 42); + assert_eq!(otel_mapping.attribute_indices, vec![] as Vec); + assert!(otel_mapping.has_functions); + assert!(otel_mapping.has_filenames); + assert!(otel_mapping.has_line_numbers); + assert!(!otel_mapping.has_inline_frames); + } + + #[test] + fn test_from_internal_mapping_large_values() { + // Create an internal mapping with large values + let internal_mapping = InternalMapping { + memory_start: 0x1000000000000000, + memory_limit: 0x2000000000000000, + file_offset: 0x1000000000000000, + filename: StringId::from_offset(999), + build_id: StringId::from_offset(888), + }; + + // Convert to OpenTelemetry Mapping + let otel_mapping = datadog_profiling_otel::Mapping::from(&internal_mapping); + + // Verify the conversion + assert_eq!(otel_mapping.memory_start, 0x1000000000000000); + assert_eq!(otel_mapping.memory_limit, 0x2000000000000000); + assert_eq!(otel_mapping.file_offset, 0x1000000000000000); + assert_eq!(otel_mapping.filename_strindex, 999); + } + + #[test] + fn test_into_otel_mapping() { + // Create an internal mapping + let internal_mapping = InternalMapping { + memory_start: 0x3000, + memory_limit: 0x4000, + file_offset: 0x200, + filename: StringId::from_offset(555), + build_id: StringId::from_offset(666), + }; + + // Convert using .into() method + let otel_mapping: datadog_profiling_otel::Mapping = (&internal_mapping).into(); + + // Verify the conversion + assert_eq!(otel_mapping.memory_start, 0x3000); + assert_eq!(otel_mapping.memory_limit, 0x4000); + assert_eq!(otel_mapping.file_offset, 0x200); + assert_eq!(otel_mapping.filename_strindex, 555); + } + + #[test] + fn test_into_otel_mapping_owned() { + // Create an internal mapping + let internal_mapping = InternalMapping { + memory_start: 0x5000, + memory_limit: 0x6000, + file_offset: 0x300, + filename: StringId::from_offset(777), + build_id: StringId::from_offset(888), + }; + + // Convert using .into() method with owned value + let otel_mapping: datadog_profiling_otel::Mapping = internal_mapping.into(); + + // Verify the conversion + assert_eq!(otel_mapping.memory_start, 0x5000); + assert_eq!(otel_mapping.memory_limit, 0x6000); + assert_eq!(otel_mapping.file_offset, 0x300); + assert_eq!(otel_mapping.filename_strindex, 777); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/mod.rs b/datadog-profiling/src/internal/profile/otel_emitter/mod.rs new file mode 100644 index 0000000000..8a91ef08a3 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/mod.rs @@ -0,0 +1,15 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! OpenTelemetry emitter for converting datadog-profiling internal types to OpenTelemetry protobuf +//! types +//! +//! This module provides `From` trait implementations for converting internal types to their +//! OpenTelemetry protobuf equivalents. + +pub mod function; +pub mod label; +pub mod location; +pub mod mapping; +pub mod profile; +pub mod stack_trace; diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs new file mode 100644 index 0000000000..137540d228 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -0,0 +1,851 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::profile::otel_emitter::label::convert_label_to_key_value; +use crate::internal::Profile as InternalProfile; +use crate::iter::{IntoLendingIterator, LendingIterator}; +use anyhow::{Context, Result}; +use std::collections::HashMap; + +impl InternalProfile { + /// Serializes the profile into OpenTelemetry format + /// + /// * `end_time` - Optional end time of the profile. Passing None will use the current time. + /// * `duration` - Optional duration of the profile. Passing None will try to calculate the + /// duration based on the end time minus the start time, but under anomalous conditions this + /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an + /// earlier time. The duration will be set to zero these cases. + pub fn serialize_into_otel( + mut self, + end_time: Option, + duration: Option, + ) -> anyhow::Result { + // Calculate duration using the same logic as encode + let end = end_time.unwrap_or_else(std::time::SystemTime::now); + let start = self.start_time; + let duration_nanos = duration + .unwrap_or_else(|| { + end.duration_since(start).unwrap_or({ + // Let's not throw away the whole profile just because the clocks were wrong. + // todo: log that the clock went backward (or programmer mistake). + std::time::Duration::ZERO + }) + }) + .as_nanos() + .min(i64::MAX as u128) as i64; + + // Create individual OpenTelemetry Profiles for each ValueType + let mut profiles = Vec::with_capacity(self.sample_types.len()); + + for sample_type in self.sample_types.iter() { + // Convert the ValueType to OpenTelemetry format + let otel_sample_type = datadog_profiling_otel::ValueType { + type_strindex: sample_type.r#type.value.to_raw_id() as i32, + unit_strindex: sample_type.unit.value.to_raw_id() as i32, + aggregation_temporality: datadog_profiling_otel::AggregationTemporality::Delta + .into(), + }; + + // Create a Profile for this sample type + let profile = datadog_profiling_otel::Profile { + sample_type: Some(otel_sample_type), + sample: vec![], // TODO: Implement sample conversion + time_nanos: self + .start_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as i64, + duration_nanos, // Use calculated duration + period_type: self.period.as_ref().map(|(_, period_type)| { + datadog_profiling_otel::ValueType { + type_strindex: period_type.r#type.value.to_raw_id() as i32, + unit_strindex: period_type.unit.value.to_raw_id() as i32, + aggregation_temporality: + datadog_profiling_otel::AggregationTemporality::Delta.into(), + } + }), + period: self + .period + .map(|(period_value, _)| period_value) + .unwrap_or(0), + comment_strindices: vec![], // We don't have comments + profile_id: vec![], // TODO: Implement when we handle profile IDs + dropped_attributes_count: 0, // We don't drop attributes + original_payload_format: String::new(), // There is no original payload + original_payload: vec![], // There is no original payload + attribute_indices: vec![], // There are currently no attributes at this level + }; + + profiles.push(profile); + } + + for (sample, timestamp, mut values) in std::mem::take(&mut self.observations).into_iter() { + let stack_index = sample.stacktrace.to_raw_id() as i32; + let label_set = self.get_label_set(sample.labels)?; + let attribute_indicies: Vec<_> = + label_set.iter().map(|x| x.to_raw_id() as i32).collect(); + let labels = label_set + .iter() + .map(|l| self.get_label(*l).copied()) + .collect::>>()?; + let link_index = 0; // TODO, handle links properly + let timestamps_unix_nano = timestamp.map_or(vec![], |ts| vec![ts.get() as u64]); + self.upscaling_rules.upscale_values(&mut values, &labels)?; + + for (idx, value) in values.iter().enumerate() { + if *value != 0 { + let otel_sample = datadog_profiling_otel::Sample { + stack_index, + attribute_indices: attribute_indicies.clone(), + link_index, + values: vec![*value], + timestamps_unix_nano: timestamps_unix_nano.clone(), + }; + profiles[idx].sample.push(otel_sample); + } + } + } + + // Convert string table using into_lending_iter + // Note: We can't use .map().collect() here because LendingIterator doesn't implement + // the standard Iterator trait. LendingIterator is designed for yielding references + // with lifetimes tied to the iterator itself, so we need to manually iterate and + // convert each string reference to an owned String. + let string_table = { + let mut strings = Vec::with_capacity(self.strings.len()); + let mut iter = self.strings.into_lending_iter(); + while let Some(s) = iter.next() { + strings.push(s.to_string()); + } + strings + }; + + // Convert labels to KeyValues for the attribute table + let mut key_to_unit_map = HashMap::new(); + let mut attribute_table = Vec::with_capacity(self.labels.len()); + + for label in self.labels.iter() { + let key_value = convert_label_to_key_value(label, &string_table, &mut key_to_unit_map) + .with_context(|| { + format!( + "Failed to convert label with key index {}", + label.get_key().to_raw_id() + ) + })?; + attribute_table.push(key_value); + } + + // Build attribute units from the key-to-unit mapping + let attribute_units = key_to_unit_map + .into_iter() + .map( + |(key_index, unit_index)| datadog_profiling_otel::AttributeUnit { + attribute_key_strindex: key_index as i32, + unit_strindex: unit_index as i32, + }, + ) + .collect(); + + // Convert the ProfilesDictionary components + let dictionary = datadog_profiling_otel::ProfilesDictionary { + mapping_table: self.mappings.into_iter().map(From::from).collect(), + location_table: self.locations.into_iter().map(From::from).collect(), + function_table: self.functions.into_iter().map(From::from).collect(), + stack_table: self.stack_traces.into_iter().map(From::from).collect(), + string_table, + attribute_table, + attribute_units, + link_table: vec![], // TODO: Implement when we handle trace links + }; + + // Create a basic ResourceProfiles structure + let resource_profiles = vec![datadog_profiling_otel::ResourceProfiles { + resource: None, // TODO: Implement when we handle resources + scope_profiles: vec![datadog_profiling_otel::ScopeProfiles { + scope: None, // TODO: Implement when we handle scopes + profiles, // Now contains the individual profiles + schema_url: String::new(), // TODO: Implement when we handle schema URLs + default_sample_type: None, // TODO: Implement when we handle sample types + }], + schema_url: String::new(), // TODO: Implement when we handle schema URLs + }]; + + Ok(datadog_profiling_otel::ProfilesData { + resource_profiles, + dictionary: Some(dictionary), + }) + } +} + +#[cfg(test)] +mod tests { + use crate::internal::profile::Profile as InternalProfile; + + #[test] + fn test_from_internal_profile_empty() { + // Create an empty internal profile + let internal_profile = InternalProfile::new(&[], None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + assert!(otel_profiles_data.dictionary.is_some()); + let dictionary = otel_profiles_data.dictionary.unwrap(); + + assert_eq!(dictionary.mapping_table.len(), 0); + assert_eq!(dictionary.location_table.len(), 0); + assert_eq!(dictionary.function_table.len(), 0); + assert_eq!(dictionary.stack_table.len(), 0); + assert_eq!(dictionary.string_table.len(), 4); // Default strings: "", "local root span id", "trace endpoint", "end_timestamp_ns" + assert_eq!(dictionary.link_table.len(), 0); + assert_eq!(dictionary.attribute_table.len(), 0); + assert_eq!(dictionary.attribute_units.len(), 0); + + assert_eq!(otel_profiles_data.resource_profiles.len(), 1); + + // Check duration calculation - only if profiles exist + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + if !scope_profile.profiles.is_empty() { + let profile = &scope_profile.profiles[0]; + // When no duration is provided, it should calculate from current time - start time + // Since we're testing with None, None, the duration should be > 0 (current time - start + // time) + assert!(profile.duration_nanos > 0); + } + } + + #[test] + fn test_from_internal_profile_with_data() { + // Create an internal profile with some data + let mut internal_profile = InternalProfile::new(&[], None); + + // Add some functions using the API Function type + let function1 = crate::api::Function { + name: "test_function_1", + system_name: "test_system_1", + filename: "test_file_1.rs", + }; + let function2 = crate::api::Function { + name: "test_function_2", + system_name: "test_system_2", + filename: "test_file_2.rs", + }; + + let _function1_id = internal_profile.add_function(&function1); + let _function2_id = internal_profile.add_function(&function2); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + assert!(otel_profiles_data.dictionary.is_some()); + let dictionary = otel_profiles_data.dictionary.unwrap(); + + assert_eq!(dictionary.function_table.len(), 2); + assert_eq!(dictionary.string_table.len(), 10); // 4 default strings + 6 strings from the 2 functions + + // Verify the first function conversion - using actual observed values + let otel_function1 = &dictionary.function_table[0]; + assert_eq!(otel_function1.name_strindex, 4); + assert_eq!(otel_function1.system_name_strindex, 5); + assert_eq!(otel_function1.filename_strindex, 6); + + // Verify the second function conversion - using actual observed values + let otel_function2 = &dictionary.function_table[1]; + assert_eq!(otel_function2.name_strindex, 7); + assert_eq!(otel_function2.system_name_strindex, 8); + assert_eq!(otel_function2.filename_strindex, 9); + + // Check duration calculation - only if profiles exist + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + if !scope_profile.profiles.is_empty() { + let profile = &scope_profile.profiles[0]; + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + } + + #[test] + fn test_from_internal_profile_with_labels() { + // Create an internal profile with some data + let mut internal_profile = InternalProfile::new(&[], None); + + // Add some labels using the API + let label1 = crate::api::Label { + key: "thread_id", + str: "main", + num: 0, + num_unit: "", + }; + let label2 = crate::api::Label { + key: "memory_usage", + str: "", + num: 1024, + num_unit: "bytes", + }; + + // Add a sample with these labels + let sample = crate::api::Sample { + locations: vec![], + values: &[42], + labels: vec![label1, label2], + }; + + let _ = internal_profile.add_sample(sample, None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + assert!(otel_profiles_data.dictionary.is_some()); + let dictionary = otel_profiles_data.dictionary.unwrap(); + + // Should have 2 labels converted to attributes + assert_eq!(dictionary.attribute_table.len(), 2); + + // Should have 1 attribute unit (for the numeric label with unit) + assert_eq!(dictionary.attribute_units.len(), 1); + + // Verify the first attribute (string label) + let attr1 = &dictionary.attribute_table[0]; + assert_eq!(attr1.key, "thread_id"); + match &attr1.value { + Some(datadog_profiling_otel::key_value::Value::StringValue(s)) => { + assert_eq!(s, "main"); + } + _ => panic!("Expected StringValue"), + } + + // Verify the second attribute (numeric label) + let attr2 = &dictionary.attribute_table[1]; + assert_eq!(attr2.key, "memory_usage"); + match &attr2.value { + Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { + assert_eq!(*n, 1024); + } + _ => panic!("Expected IntValue"), + } + + // Verify the attribute unit mapping + let unit = &dictionary.attribute_units[0]; + // The key should map to the memory_usage string index + // and the unit should map to the "bytes" string index + assert!(unit.attribute_key_strindex > 0); + assert!(unit.unit_strindex > 0); + + // Check duration calculation - only if profiles exist + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + if !scope_profile.profiles.is_empty() { + let profile = &scope_profile.profiles[0]; + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + } + + #[test] + fn test_from_internal_profile_with_sample_types() { + // Create an internal profile with specific sample types + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("allocations", "count"), + ]; + let internal_profile = InternalProfile::new(&sample_types, None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify that individual profiles are created for each sample type + assert_eq!(otel_profiles_data.resource_profiles.len(), 1); + let resource_profile = &otel_profiles_data.resource_profiles[0]; + assert_eq!(resource_profile.scope_profiles.len(), 1); + let scope_profile = &resource_profile.scope_profiles[0]; + + // Should have 2 profiles (one for each sample type) + assert_eq!(scope_profile.profiles.len(), 2); + + // Verify the first profile (cpu profile) + let cpu_profile = &scope_profile.profiles[0]; + assert!(cpu_profile.sample_type.is_some()); + let cpu_sample_type = cpu_profile.sample_type.as_ref().unwrap(); + assert_eq!(cpu_sample_type.type_strindex, 4); // "cpu" string index + assert_eq!(cpu_sample_type.unit_strindex, 5); // "nanoseconds" string index + + // Verify the second profile (allocations profile) + let allocations_profile = &scope_profile.profiles[1]; + assert!(allocations_profile.sample_type.is_some()); + let allocations_sample_type = allocations_profile.sample_type.as_ref().unwrap(); + assert_eq!(allocations_sample_type.type_strindex, 6); // "allocations" string index + assert_eq!(allocations_sample_type.unit_strindex, 7); // "count" string index + + // Check duration calculation for both profiles + for profile in &scope_profile.profiles { + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + } + + #[test] + fn test_sample_conversion_basic() { + // Create an internal profile with sample types + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("memory", "bytes"), + ]; + let mut internal_profile = InternalProfile::new(&sample_types, None); + + // Add a function to create a location + let function = crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + }; + let _function_id = internal_profile.add_function(&function); + + // Add a mapping + let mapping = crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + }; + let _mapping_id = internal_profile.add_mapping(&mapping); + + // Add a location + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + let location_id = internal_profile.add_location(&location).unwrap(); + + // Add a stack trace + let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + + // Add a sample with values + let sample = crate::api::Sample { + locations: vec![location], + values: &[100, 2048], // 100 nanoseconds, 2048 bytes + labels: vec![], + }; + let _ = internal_profile.add_sample(sample, None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + assert!(otel_profiles_data.dictionary.is_some()); + let _dictionary = otel_profiles_data.dictionary.unwrap(); + + // Should have 2 profiles (one for each sample type) + assert_eq!( + otel_profiles_data.resource_profiles[0].scope_profiles[0] + .profiles + .len(), + 2 + ); + + // Verify the first profile (cpu profile) has the correct sample + let cpu_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(cpu_profile.sample.len(), 1); + let cpu_sample = &cpu_profile.sample[0]; + assert_eq!(cpu_sample.values, vec![100]); + assert_eq!(cpu_sample.stack_index, 0); // First stack trace + assert_eq!(cpu_sample.attribute_indices.len(), 0); // No labels + + // Verify the second profile (memory profile) has the correct sample + let memory_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[1]; + assert_eq!(memory_profile.sample.len(), 1); + let memory_sample = &memory_profile.sample[0]; + assert_eq!(memory_sample.values, vec![2048]); + assert_eq!(memory_sample.stack_index, 0); // First stack trace + assert_eq!(memory_sample.attribute_indices.len(), 0); // No labels + + // Check duration calculation for both profiles + for profile in &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles { + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + } + + #[test] + fn test_sample_conversion_with_labels() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let mut internal_profile = InternalProfile::new(&sample_types, None); + + // Add a function and location + let function = crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + }; + let _function_id = internal_profile.add_function(&function); + + // Add a mapping + let mapping = crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + }; + let _mapping_id = internal_profile.add_mapping(&mapping); + + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + let location_id = internal_profile.add_location(&location).unwrap(); + + let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + + // Add a sample with labels + let sample = crate::api::Sample { + locations: vec![location], + values: &[150], + labels: vec![ + crate::api::Label { + key: "thread_id", + str: "main", + num: 0, + num_unit: "", + }, + crate::api::Label { + key: "cpu_usage", + str: "", + num: 75, + num_unit: "percent", + }, + ], + }; + let _ = internal_profile.add_sample(sample, None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + let _dictionary = otel_profiles_data.dictionary.unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + + // Should have 1 sample + assert_eq!(profile.sample.len(), 1); + let sample = &profile.sample[0]; + + // Verify the sample has the correct values + assert_eq!(sample.values, vec![150]); + assert_eq!(sample.stack_index, 0); + + // Verify the sample has the correct attribute indices + assert_eq!(sample.attribute_indices.len(), 2); + // The attribute indices should correspond to the labels in the attribute table + assert!(sample.attribute_indices[0] >= 0); + + // Check duration calculation + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + assert!(sample.attribute_indices[1] >= 0); + + // Verify the attributes were converted correctly + assert_eq!(_dictionary.attribute_table.len(), 2); + assert_eq!(_dictionary.attribute_units.len(), 1); // One numeric label with unit + } + + #[test] + fn test_sample_conversion_with_timestamps() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let mut internal_profile = InternalProfile::new(&sample_types, None); + + // Add a function and location + let function = crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + }; + let _function_id = internal_profile.add_function(&function); + + // Add a mapping + let mapping = crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + }; + let _mapping_id = internal_profile.add_mapping(&mapping); + + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + let location_id = internal_profile.add_location(&location).unwrap(); + + let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + + // Add a sample with timestamp + let sample = crate::api::Sample { + locations: vec![location], + values: &[200], + labels: vec![], + }; + let timestamp = crate::internal::Timestamp::new(1234567890).unwrap(); + let _ = internal_profile.add_sample(sample, Some(timestamp)); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + + // Should have 1 sample + assert_eq!(profile.sample.len(), 1); + let sample = &profile.sample[0]; + + // Verify the sample has the correct timestamp + assert_eq!(sample.timestamps_unix_nano.len(), 1); + assert_eq!(sample.timestamps_unix_nano[0], 1234567890); + + // Check duration calculation + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + + #[test] + fn test_sample_conversion_zero_values_filtered() { + // Create an internal profile with sample types + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("memory", "bytes"), + ]; + let mut internal_profile = InternalProfile::new(&sample_types, None); + + // Add a function and location + let function = crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + }; + let _function_id = internal_profile.add_function(&function); + + // Add a mapping + let mapping = crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + }; + let _mapping_id = internal_profile.add_mapping(&mapping); + + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + let location_id = internal_profile.add_location(&location).unwrap(); + + let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + + // Add a sample with one zero value and one non-zero value + let sample = crate::api::Sample { + locations: vec![location], + values: &[0, 1024], // 0 nanoseconds, 1024 bytes + labels: vec![], + }; + let _ = internal_profile.add_sample(sample, None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + let profile0 = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + let profile1 = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[1]; + + // First profile (cpu) should have no samples since value is 0 + assert_eq!(profile0.sample.len(), 0); + + // Second profile (memory) should have 1 sample since value is non-zero + assert_eq!(profile1.sample.len(), 1); + let memory_sample = &profile1.sample[0]; + assert_eq!(memory_sample.values, vec![1024]); + + // Check duration calculation for both profiles + for profile in &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles { + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + } + + #[test] + fn test_sample_conversion_multiple_samples() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let mut internal_profile = InternalProfile::new(&sample_types, None); + + // Add a function and location + let function = crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + }; + let _function_id = internal_profile.add_function(&function); + + // Add a mapping + let mapping = crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + }; + let _mapping_id = internal_profile.add_mapping(&mapping); + + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + let location_id = internal_profile.add_location(&location).unwrap(); + + let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + + // Add multiple samples + let sample1 = crate::api::Sample { + locations: vec![location], + values: &[100], + labels: vec![], + }; + let sample2 = crate::api::Sample { + locations: vec![location], + values: &[200], + labels: vec![], + }; + let sample3 = crate::api::Sample { + locations: vec![location], + values: &[300], + labels: vec![], + }; + + let _ = internal_profile.add_sample(sample1, None); + let _ = internal_profile.add_sample(sample2, None); + let _ = internal_profile.add_sample(sample3, None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + // Verify the conversion + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + + // Should have 1 aggregated sample (samples with same stack trace and labels get aggregated) + assert_eq!(profile.sample.len(), 1); + + // Verify the aggregated sample has the summed value + let sample = &profile.sample[0]; + assert_eq!(sample.values, vec![600]); // 100 + 200 + 300 + + // Verify all samples have the same stack index + for sample in &profile.sample { + assert_eq!(sample.stack_index, 0); + } + + // Check duration calculation + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + + #[test] + fn test_duration_calculation() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let internal_profile = InternalProfile::new(&sample_types, None); + + // Test with explicit duration + let explicit_duration = std::time::Duration::from_secs(5); + let otel_profiles_data = internal_profile + .serialize_into_otel(None, Some(explicit_duration)) + .unwrap(); + + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + // Should use the explicit duration (5 seconds = 5_000_000_000 nanoseconds) + assert_eq!(profile.duration_nanos, 5_000_000_000); + + // Test with explicit end_time + let internal_profile2 = InternalProfile::new(&sample_types, None); + let start_time = internal_profile2.start_time; + let end_time = start_time + std::time::Duration::from_secs(3); + let otel_profiles_data2 = internal_profile2 + .serialize_into_otel(Some(end_time), None) + .unwrap(); + + let profile2 = &otel_profiles_data2.resource_profiles[0].scope_profiles[0].profiles[0]; + // Should calculate duration from end_time - start_time (3 seconds = 3_000_000_000 + // nanoseconds) + assert_eq!(profile2.duration_nanos, 3_000_000_000); + + // Test with both end_time and duration (duration should take precedence) + let internal_profile3 = InternalProfile::new(&sample_types, None); + let start_time3 = internal_profile3.start_time; + let end_time3 = start_time3 + std::time::Duration::from_secs(10); + let duration3 = std::time::Duration::from_secs(7); + let otel_profiles_data3 = internal_profile3 + .serialize_into_otel(Some(end_time3), Some(duration3)) + .unwrap(); + + let profile3 = &otel_profiles_data3.resource_profiles[0].scope_profiles[0].profiles[0]; + // Should use the explicit duration (7 seconds = 7_000_000_000 nanoseconds) + assert_eq!(profile3.duration_nanos, 7_000_000_000); + } + + #[test] + fn test_period_conversion() { + // Create an internal profile with sample types and period + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let period = crate::api::Period { + r#type: crate::api::ValueType::new("cpu", "cycles"), + value: 1000, + }; + let internal_profile = InternalProfile::new(&sample_types, Some(period)); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + + // Should have period type information + assert!(profile.period_type.is_some()); + let period_type = profile.period_type.as_ref().unwrap(); + + // The period type should be converted from the internal profile's period + // Note: The exact string indices depend on the string table, but we can verify they're + // valid + assert!(period_type.type_strindex >= 0); + assert!(period_type.unit_strindex >= 0); + + // Should have the correct period value + assert_eq!(profile.period, 1000); + + // Test without period + let internal_profile_no_period = InternalProfile::new(&sample_types, None); + let otel_profiles_data_no_period = internal_profile_no_period + .serialize_into_otel(None, None) + .unwrap(); + + let profile_no_period = + &otel_profiles_data_no_period.resource_profiles[0].scope_profiles[0].profiles[0]; + + // Should have no period type when no period is set + assert!(profile_no_period.period_type.is_none()); + // Should have period value of 0 when no period is set + assert_eq!(profile_no_period.period, 0); + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs b/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs new file mode 100644 index 0000000000..ec7c3e6d93 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs @@ -0,0 +1,97 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::Id; +use crate::internal::StackTrace as InternalStackTrace; + +// For owned values - forward to reference version +impl From for datadog_profiling_otel::Stack { + fn from(internal_stack_trace: InternalStackTrace) -> Self { + Self::from(&internal_stack_trace) + } +} + +// For references (existing implementation) +impl From<&InternalStackTrace> for datadog_profiling_otel::Stack { + fn from(internal_stack_trace: &InternalStackTrace) -> Self { + Self { + location_indices: internal_stack_trace + .locations + .iter() + .map(|location_id| location_id.to_raw_id() as i32) + .collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::internal::LocationId; + + #[test] + fn test_from_internal_stack_trace() { + // Create an internal stack trace + let internal_stack_trace = InternalStackTrace { + locations: vec![ + LocationId::from_offset(0), + LocationId::from_offset(1), + LocationId::from_offset(2), + ], + }; + + // Convert to OpenTelemetry Stack + let otel_stack = datadog_profiling_otel::Stack::from(&internal_stack_trace); + + // Verify the conversion - note: from_offset adds 1 to avoid zero values + assert_eq!(otel_stack.location_indices, vec![1, 2, 3]); + } + + #[test] + fn test_from_empty_stack_trace() { + // Create an internal stack trace with no locations + let internal_stack_trace = InternalStackTrace { locations: vec![] }; + + // Convert to OpenTelemetry Stack + let otel_stack = datadog_profiling_otel::Stack::from(&internal_stack_trace); + + // Verify the conversion + assert_eq!(otel_stack.location_indices, vec![] as Vec); + } + + #[test] + fn test_into_otel_stack() { + // Create an internal stack trace + let internal_stack_trace = InternalStackTrace { + locations: vec![ + LocationId::from_offset(10), + LocationId::from_offset(20), + LocationId::from_offset(30), + ], + }; + + // Convert using .into() method + let otel_stack: datadog_profiling_otel::Stack = (&internal_stack_trace).into(); + + // Verify the conversion - note: from_offset adds 1 to avoid zero values + assert_eq!(otel_stack.location_indices, vec![11, 21, 31]); + } + + #[test] + fn test_into_otel_stack_owned() { + // Create an internal stack trace + let internal_stack_trace = InternalStackTrace { + locations: vec![ + LocationId::from_offset(40), + LocationId::from_offset(50), + LocationId::from_offset(60), + ], + }; + + // Convert using .into() method with owned value + let otel_stack: datadog_profiling_otel::Stack = internal_stack_trace.into(); + + // Verify the conversion - note: from_offset adds 1 to avoid zero values + assert_eq!(otel_stack.location_indices, vec![41, 51, 61]); + } +} From ab970244400790ed001b5b932994f05d219ab95b Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 09:46:01 -0400 Subject: [PATCH 02/23] vendor protoc --- Cargo.lock | 1 + datadog-profiling-otel/Cargo.toml | 1 + datadog-profiling-otel/build.rs | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index e975496a2f..03ea168004 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,6 +1700,7 @@ dependencies = [ "prost", "prost-build", "prost-types", + "protoc-bin-vendored", ] [[package]] diff --git a/datadog-profiling-otel/Cargo.toml b/datadog-profiling-otel/Cargo.toml index a29c5f6688..ecca91a5f9 100644 --- a/datadog-profiling-otel/Cargo.toml +++ b/datadog-profiling-otel/Cargo.toml @@ -17,6 +17,7 @@ prost-types = "0.13" [build-dependencies] prost-build = "0.13" +protoc-bin-vendored = "3.0.0" [dev-dependencies] bolero = "0.13" diff --git a/datadog-profiling-otel/build.rs b/datadog-profiling-otel/build.rs index 53dcd28deb..0f710eb799 100644 --- a/datadog-profiling-otel/build.rs +++ b/datadog-profiling-otel/build.rs @@ -5,6 +5,10 @@ use std::env; use std::path::PathBuf; fn main() { + // protoc is required to compile proto files. This uses protoc-bin-vendored to provide + // the protoc binary, setting the env var to tell prost_build where to find it. + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); + // Tell Cargo to rerun this build script if the proto files change println!("cargo:rerun-if-changed=profiles.proto"); println!("cargo:rerun-if-changed=opentelemetry/proto/common/v1/common.proto"); From 93b17ae5d72141366a1837fd71a7c7fe0d0e4a5c Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 14:37:04 -0400 Subject: [PATCH 03/23] serialize --- Cargo.lock | 2 ++ LICENSE-3rdparty.yml | 9 ++++++- datadog-profiling-otel/Cargo.toml | 2 ++ datadog-profiling-otel/src/lib.rs | 45 +++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 03ea168004..4c6a919e41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1696,11 +1696,13 @@ dependencies = [ name = "datadog-profiling-otel" version = "20.0.0" dependencies = [ + "anyhow", "bolero", "prost", "prost-build", "prost-types", "protoc-bin-vendored", + "zstd", ] [[package]] diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 61a346d4d6..5fc4e2f272 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,4 +1,4 @@ -root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, cc_utils, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib +root_name: builder, build_common, tools, datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, tinybytes, spawn_worker, cc_utils, datadog-library-config, datadog-library-config-ffi, datadog-live-debugger, datadog-live-debugger-ffi, datadog-profiling, datadog-profiling-otel, datadog-profiling-protobuf, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-protobuf, datadog-trace-utils, datadog-trace-normalization, dogstatsd-client, datadog-log-ffi, datadog-log, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, datadog-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib third_party_libraries: - package_name: addr2line package_version: 0.24.2 @@ -22132,6 +22132,13 @@ third_party_libraries: licenses: - license: Apache-2.0 text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" +- package_name: prost-types + package_version: 0.13.5 + repository: https://github.com/tokio-rs/prost + license: Apache-2.0 + licenses: + - license: Apache-2.0 + text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" - package_name: quinn package_version: 0.11.5 repository: https://github.com/quinn-rs/quinn diff --git a/datadog-profiling-otel/Cargo.toml b/datadog-profiling-otel/Cargo.toml index ecca91a5f9..6e18af1d14 100644 --- a/datadog-profiling-otel/Cargo.toml +++ b/datadog-profiling-otel/Cargo.toml @@ -14,6 +14,8 @@ bench = false [dependencies] prost = "0.13" prost-types = "0.13" +zstd = "0.13" +anyhow = "1.0" [build-dependencies] prost-build = "0.13" diff --git a/datadog-profiling-otel/src/lib.rs b/datadog-profiling-otel/src/lib.rs index e8ec73d86e..5237fc4121 100644 --- a/datadog-profiling-otel/src/lib.rs +++ b/datadog-profiling-otel/src/lib.rs @@ -12,6 +12,23 @@ pub mod proto { pub use proto::*; +/// Extension trait for ProfilesData to add serialization methods +pub trait ProfilesDataExt { + /// Serializes the profile into a zstd compressed protobuf byte array. + /// This method consumes the ProfilesData and returns the compressed bytes. + fn serialize_into_compressed_proto(self) -> anyhow::Result>; +} + +impl ProfilesDataExt for ProfilesData { + fn serialize_into_compressed_proto(self) -> anyhow::Result> { + // TODO, streaming into zstd is difficult because prost wants a BytesMut which zstd doesn't + // easily supply. + let proto_bytes = prost::Message::encode_to_vec(&self); + let compressed_bytes = zstd::encode_all(&proto_bytes[..], 0)?; + Ok(compressed_bytes) + } +} + #[cfg(test)] mod tests { use super::*; @@ -33,4 +50,32 @@ mod tests { "test" ); } + + #[test] + fn test_serialize_into_compressed_proto() { + // Test that we can serialize and compress a ProfilesData + let mut profiles_dict = ProfilesDictionary::default(); + profiles_dict.string_table.push("test".to_string()); + + let profiles_data = ProfilesData { + dictionary: Some(profiles_dict), + ..Default::default() + }; + + // Serialize and compress + let compressed_bytes = profiles_data.serialize_into_compressed_proto().unwrap(); + + // Verify we got compressed bytes + assert!(!compressed_bytes.is_empty()); + + // Verify we can decompress and deserialize + let decompressed_bytes = zstd::decode_all(&compressed_bytes[..]).unwrap(); + let deserialized: ProfilesData = prost::Message::decode(&decompressed_bytes[..]).unwrap(); + + // Verify the data is correct + assert_eq!( + deserialized.dictionary.as_ref().unwrap().string_table[0], + "test" + ); + } } From c6106840372f6846747618e1e4eb18bc9f215d25 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 14:38:46 -0400 Subject: [PATCH 04/23] dockerfile --- tools/docker/Dockerfile.build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 80527260bc..1a76373d16 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -87,6 +87,7 @@ COPY "datadog-live-debugger-ffi/Cargo.toml" "datadog-live-debugger-ffi/" COPY "datadog-profiling/Cargo.toml" "datadog-profiling/" COPY "datadog-profiling-ffi/Cargo.toml" "datadog-profiling-ffi/" COPY "datadog-profiling-protobuf/Cargo.toml" "datadog-profiling-protobuf/" +COPY "datadog-profiling-otel/Cargo.toml" "datadog-profiling-otel/" COPY "datadog-profiling-replayer/Cargo.toml" "datadog-profiling-replayer/" COPY "datadog-remote-config/Cargo.toml" "datadog-remote-config/" COPY "datadog-sidecar/Cargo.toml" "datadog-sidecar/" @@ -132,6 +133,7 @@ RUN echo \ datadog-profiling-replayer/src/main.rs \ datadog-profiling/benches/interning_strings.rs \ datadog-profiling/benches/main.rs \ + datadog-profiling-otel/src/lib.rs \ tools/sidecar_mockgen/src/bin/sidecar_mockgen.rs \ tools/src/bin/dedup_headers.rs \ datadog-trace-normalization/benches/normalization_utils.rs \ From 234c4669cda88a77976e9864abd67c6cbc05a434 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 14:59:57 -0400 Subject: [PATCH 05/23] emit otel --- .../src/profiles/datatypes.rs | 3 +- .../internal/profile/otel_emitter/profile.rs | 121 +++++++++++++++--- 2 files changed, 106 insertions(+), 18 deletions(-) diff --git a/datadog-profiling-ffi/src/profiles/datatypes.rs b/datadog-profiling-ffi/src/profiles/datatypes.rs index add6136621..156af324b0 100644 --- a/datadog-profiling-ffi/src/profiles/datatypes.rs +++ b/datadog-profiling-ffi/src/profiles/datatypes.rs @@ -754,7 +754,8 @@ pub unsafe extern "C" fn ddog_prof_Profile_serialize( } let end_time = end_time.map(SystemTime::from); - old_profile.serialize_into_compressed_pprof(end_time, None) + // TODO: make this an option instead of hardcoding to otel + old_profile.serialize_into_compressed_otel(end_time, None) })() .context("ddog_prof_Profile_serialize failed") .into() diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index 137540d228..dcae33dc31 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -3,20 +3,21 @@ use crate::collections::identifiable::Id; use crate::internal::profile::otel_emitter::label::convert_label_to_key_value; -use crate::internal::Profile as InternalProfile; +use crate::internal::profile::{EncodedProfile, Profile as InternalProfile}; use crate::iter::{IntoLendingIterator, LendingIterator}; use anyhow::{Context, Result}; +use datadog_profiling_otel::ProfilesDataExt; use std::collections::HashMap; impl InternalProfile { - /// Serializes the profile into OpenTelemetry format + /// Converts the profile into OpenTelemetry format /// /// * `end_time` - Optional end time of the profile. Passing None will use the current time. /// * `duration` - Optional duration of the profile. Passing None will try to calculate the /// duration based on the end time minus the start time, but under anomalous conditions this /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an /// earlier time. The duration will be set to zero these cases. - pub fn serialize_into_otel( + pub fn convert_into_otel( mut self, end_time: Option, duration: Option, @@ -176,6 +177,32 @@ impl InternalProfile { dictionary: Some(dictionary), }) } + + /// Serializes the profile into OpenTelemetry format and compresses it using zstd. + /// + /// * `end_time` - Optional end time of the profile. Passing None will use the current time. + /// * `duration` - Optional duration of the profile. Passing None will try to calculate the + /// duration based on the end time minus the start time, but under anomalous conditions this + /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an + /// earlier time. The duration will be set to zero these cases. + pub fn serialize_into_compressed_otel( + mut self, + end_time: Option, + duration: Option, + ) -> anyhow::Result { + // Extract values before consuming self + let start = self.start_time; + let endpoints_stats = std::mem::take(&mut self.endpoints.stats); + let otel_profiles_data = self.convert_into_otel(end_time, duration)?; + let buffer = otel_profiles_data.serialize_into_compressed_proto()?; + let end = end_time.unwrap_or_else(std::time::SystemTime::now); + Ok(EncodedProfile { + start, + end, + buffer, + endpoints_stats, + }) + } } #[cfg(test)] @@ -188,7 +215,7 @@ mod tests { let internal_profile = InternalProfile::new(&[], None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion assert!(otel_profiles_data.dictionary.is_some()); @@ -237,7 +264,7 @@ mod tests { let _function2_id = internal_profile.add_function(&function2); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion assert!(otel_profiles_data.dictionary.is_some()); @@ -296,7 +323,7 @@ mod tests { let _ = internal_profile.add_sample(sample, None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion assert!(otel_profiles_data.dictionary.is_some()); @@ -354,7 +381,7 @@ mod tests { let internal_profile = InternalProfile::new(&sample_types, None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify that individual profiles are created for each sample type assert_eq!(otel_profiles_data.resource_profiles.len(), 1); @@ -434,7 +461,7 @@ mod tests { let _ = internal_profile.add_sample(sample, None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion assert!(otel_profiles_data.dictionary.is_some()); @@ -527,7 +554,7 @@ mod tests { let _ = internal_profile.add_sample(sample, None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion let _dictionary = otel_profiles_data.dictionary.unwrap(); @@ -600,7 +627,7 @@ mod tests { let _ = internal_profile.add_sample(sample, Some(timestamp)); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -664,7 +691,7 @@ mod tests { let _ = internal_profile.add_sample(sample, None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion let profile0 = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -741,7 +768,7 @@ mod tests { let _ = internal_profile.add_sample(sample3, None); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -772,7 +799,7 @@ mod tests { // Test with explicit duration let explicit_duration = std::time::Duration::from_secs(5); let otel_profiles_data = internal_profile - .serialize_into_otel(None, Some(explicit_duration)) + .convert_into_otel(None, Some(explicit_duration)) .unwrap(); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -784,7 +811,7 @@ mod tests { let start_time = internal_profile2.start_time; let end_time = start_time + std::time::Duration::from_secs(3); let otel_profiles_data2 = internal_profile2 - .serialize_into_otel(Some(end_time), None) + .convert_into_otel(Some(end_time), None) .unwrap(); let profile2 = &otel_profiles_data2.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -798,7 +825,7 @@ mod tests { let end_time3 = start_time3 + std::time::Duration::from_secs(10); let duration3 = std::time::Duration::from_secs(7); let otel_profiles_data3 = internal_profile3 - .serialize_into_otel(Some(end_time3), Some(duration3)) + .convert_into_otel(Some(end_time3), Some(duration3)) .unwrap(); let profile3 = &otel_profiles_data3.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -817,7 +844,7 @@ mod tests { let internal_profile = InternalProfile::new(&sample_types, Some(period)); // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.serialize_into_otel(None, None).unwrap(); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; @@ -837,7 +864,7 @@ mod tests { // Test without period let internal_profile_no_period = InternalProfile::new(&sample_types, None); let otel_profiles_data_no_period = internal_profile_no_period - .serialize_into_otel(None, None) + .convert_into_otel(None, None) .unwrap(); let profile_no_period = @@ -848,4 +875,64 @@ mod tests { // Should have period value of 0 when no period is set assert_eq!(profile_no_period.period, 0); } + + #[test] + fn test_serialize_into_compressed_otel() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let mut internal_profile = InternalProfile::new(&sample_types, None); + + // Add a function and location + let function = crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + }; + let _function_id = internal_profile.add_function(&function); + + // Add a mapping + let mapping = crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + }; + let _mapping_id = internal_profile.add_mapping(&mapping); + + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + let location_id = internal_profile.add_location(&location).unwrap(); + + let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + + // Add a sample + let sample = crate::api::Sample { + locations: vec![location], + values: &[150], + labels: vec![], + }; + let _ = internal_profile.add_sample(sample, None); + + // Test serialization to compressed OpenTelemetry format + let encoded_profile = internal_profile + .serialize_into_compressed_otel(None, None) + .unwrap(); + + // Verify the encoded profile structure + assert!(encoded_profile.start > std::time::UNIX_EPOCH); + assert!(encoded_profile.end > encoded_profile.start); + assert!(!encoded_profile.buffer.is_empty()); + + // Verify the buffer contains compressed data (should be smaller than uncompressed) + // The compressed buffer should be significantly smaller than a typical uncompressed profile + assert!(encoded_profile.buffer.len() < 10000); // Reasonable upper bound for this small profile + + // Verify endpoints stats are preserved + assert!(encoded_profile.endpoints_stats.is_empty()); // No endpoints added + } } From b08a0dfe4af3151e4b762173441a1e87c2aa4dad Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 17:20:27 -0400 Subject: [PATCH 06/23] miri ignore --- datadog-profiling-otel/src/lib.rs | 1 + datadog-trace-protobuf/src/remoteconfig.rs | 30 ++++++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/datadog-profiling-otel/src/lib.rs b/datadog-profiling-otel/src/lib.rs index 5237fc4121..84e1829ef6 100644 --- a/datadog-profiling-otel/src/lib.rs +++ b/datadog-profiling-otel/src/lib.rs @@ -52,6 +52,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] // Skip this test when running under Miri fn test_serialize_into_compressed_proto() { // Test that we can serialize and compress a ProfilesData let mut profiles_dict = ProfilesDictionary::default(); diff --git a/datadog-trace-protobuf/src/remoteconfig.rs b/datadog-trace-protobuf/src/remoteconfig.rs index a8a4b66524..fdc89ee871 100644 --- a/datadog-trace-protobuf/src/remoteconfig.rs +++ b/datadog-trace-protobuf/src/remoteconfig.rs @@ -3,7 +3,8 @@ use serde::{Deserialize, Serialize}; // This file is @generated by prost-build. -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct File { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -11,7 +12,8 @@ pub struct File { #[serde(with = "serde_bytes")] pub raw: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Client { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, @@ -33,7 +35,8 @@ pub struct Client { #[prost(bytes = "vec", tag = "11")] pub capabilities: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientTracer { #[prost(string, tag = "1")] pub runtime_id: ::prost::alloc::string::String, @@ -52,7 +55,8 @@ pub struct ClientTracer { #[prost(string, repeated, tag = "7")] pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientAgent { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -65,7 +69,8 @@ pub struct ClientAgent { #[prost(string, repeated, tag = "5")] pub cws_workloads: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ConfigState { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -78,7 +83,8 @@ pub struct ConfigState { #[prost(string, tag = "5")] pub apply_error: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientState { #[prost(uint64, tag = "1")] pub root_version: u64, @@ -93,14 +99,16 @@ pub struct ClientState { #[prost(bytes = "vec", tag = "6")] pub backend_client_state: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TargetFileHash { #[prost(string, tag = "1")] pub algorithm: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub hash: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TargetFileMeta { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -109,14 +117,16 @@ pub struct TargetFileMeta { #[prost(message, repeated, tag = "3")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsRequest { #[prost(message, optional, tag = "1")] pub client: ::core::option::Option, #[prost(message, repeated, tag = "2")] pub cached_target_files: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsResponse { #[prost(bytes = "vec", repeated, tag = "1")] #[serde(with = "crate::serde")] From e4c872450a41cf1d63338ef5551076248f7e2972 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 17:21:11 -0400 Subject: [PATCH 07/23] don't checkout that file --- datadog-trace-protobuf/src/remoteconfig.rs | 30 ++++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/datadog-trace-protobuf/src/remoteconfig.rs b/datadog-trace-protobuf/src/remoteconfig.rs index fdc89ee871..a8a4b66524 100644 --- a/datadog-trace-protobuf/src/remoteconfig.rs +++ b/datadog-trace-protobuf/src/remoteconfig.rs @@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize}; // This file is @generated by prost-build. -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct File { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -12,8 +11,7 @@ pub struct File { #[serde(with = "serde_bytes")] pub raw: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct Client { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, @@ -35,8 +33,7 @@ pub struct Client { #[prost(bytes = "vec", tag = "11")] pub capabilities: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientTracer { #[prost(string, tag = "1")] pub runtime_id: ::prost::alloc::string::String, @@ -55,8 +52,7 @@ pub struct ClientTracer { #[prost(string, repeated, tag = "7")] pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientAgent { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -69,8 +65,7 @@ pub struct ClientAgent { #[prost(string, repeated, tag = "5")] pub cws_workloads: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ConfigState { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -83,8 +78,7 @@ pub struct ConfigState { #[prost(string, tag = "5")] pub apply_error: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientState { #[prost(uint64, tag = "1")] pub root_version: u64, @@ -99,16 +93,14 @@ pub struct ClientState { #[prost(bytes = "vec", tag = "6")] pub backend_client_state: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct TargetFileHash { #[prost(string, tag = "1")] pub algorithm: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub hash: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct TargetFileMeta { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -117,16 +109,14 @@ pub struct TargetFileMeta { #[prost(message, repeated, tag = "3")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsRequest { #[prost(message, optional, tag = "1")] pub client: ::core::option::Option, #[prost(message, repeated, tag = "2")] pub cached_target_files: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsResponse { #[prost(bytes = "vec", repeated, tag = "1")] #[serde(with = "crate::serde")] From 8f693814468412969e97bd0925db2543b94572a5 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 14 Aug 2025 17:30:44 -0400 Subject: [PATCH 08/23] more miri ignore --- datadog-profiling/src/internal/profile/otel_emitter/profile.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index dcae33dc31..b297019bf9 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -877,6 +877,7 @@ mod tests { } #[test] + #[cfg_attr(miri, ignore)] // Skip this test when running under Miri fn test_serialize_into_compressed_otel() { // Create an internal profile with sample types let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; From 4edfdef3937e5d37a525e093467164c51b462636 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 15:37:24 -0400 Subject: [PATCH 09/23] PR Comment: Move prelude to file --- datadog-profiling-otel/build.rs | 35 ---------- datadog-profiling-otel/src/lib.rs | 4 +- datadog-profiling-otel/src/proto.rs | 38 ++++++++++ .../internal/profile/otel_emitter/profile.rs | 70 +++++++++---------- datadog-trace-protobuf/src/remoteconfig.rs | 30 +++++--- 5 files changed, 94 insertions(+), 83 deletions(-) create mode 100644 datadog-profiling-otel/src/proto.rs diff --git a/datadog-profiling-otel/build.rs b/datadog-profiling-otel/build.rs index 0f710eb799..d7d83ce645 100644 --- a/datadog-profiling-otel/build.rs +++ b/datadog-profiling-otel/build.rs @@ -32,39 +32,4 @@ fn main() { &["."], ) .expect("Failed to compile protobuf files"); - - // Generate the module file with correct structure - let module_content = r#" -// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -// This module is generated by prost-build from profiles.proto -#[allow(clippy::all)] -pub mod opentelemetry { - pub mod proto { - pub mod common { - pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/opentelemetry.proto.common.v1.rs")); - } - } - pub mod resource { - pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/opentelemetry.proto.resource.v1.rs")); - } - } - pub mod profiles { - pub mod v1development { - include!(concat!(env!("OUT_DIR"), "/opentelemetry.proto.profiles.v1development.rs")); - } - } - } -} - -// Re-export commonly used types -pub use opentelemetry::proto::profiles::v1development::*; -pub use opentelemetry::proto::common::v1::*; -pub use opentelemetry::proto::resource::v1::*; -"#; - - std::fs::write(out_dir.join("mod.rs"), module_content).expect("Failed to write module file"); } diff --git a/datadog-profiling-otel/src/lib.rs b/datadog-profiling-otel/src/lib.rs index 84e1829ef6..bba0cee27d 100644 --- a/datadog-profiling-otel/src/lib.rs +++ b/datadog-profiling-otel/src/lib.rs @@ -6,9 +6,7 @@ #![cfg_attr(not(test), deny(clippy::expect_used))] #![cfg_attr(not(test), deny(clippy::unimplemented))] -pub mod proto { - include!(concat!(env!("OUT_DIR"), "/mod.rs")); -} +pub mod proto; pub use proto::*; diff --git a/datadog-profiling-otel/src/proto.rs b/datadog-profiling-otel/src/proto.rs new file mode 100644 index 0000000000..053bd37da2 --- /dev/null +++ b/datadog-profiling-otel/src/proto.rs @@ -0,0 +1,38 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +// This module is generated by prost-build from profiles.proto +#[allow(clippy::all)] +pub mod opentelemetry { + pub mod proto { + pub mod common { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/opentelemetry.proto.common.v1.rs" + )); + } + } + pub mod resource { + pub mod v1 { + include!(concat!( + env!("OUT_DIR"), + "/opentelemetry.proto.resource.v1.rs" + )); + } + } + pub mod profiles { + pub mod v1development { + include!(concat!( + env!("OUT_DIR"), + "/opentelemetry.proto.profiles.v1development.rs" + )); + } + } + } +} + +// Re-export commonly used types +pub use opentelemetry::proto::common::v1::*; +pub use opentelemetry::proto::profiles::v1development::*; +pub use opentelemetry::proto::resource::v1::*; diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index b297019bf9..525514ec16 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -260,8 +260,8 @@ mod tests { filename: "test_file_2.rs", }; - let _function1_id = internal_profile.add_function(&function1); - let _function2_id = internal_profile.add_function(&function2); + let _function1_id = internal_profile.try_add_function(&function1); + let _function2_id = internal_profile.try_add_function(&function2); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -320,7 +320,7 @@ mod tests { labels: vec![label1, label2], }; - let _ = internal_profile.add_sample(sample, None); + let _ = internal_profile.try_add_sample(sample, None); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -428,7 +428,7 @@ mod tests { system_name: "test_system", filename: "test_file.rs", }; - let _function_id = internal_profile.add_function(&function); + let _function_id = internal_profile.try_add_function(&function); // Add a mapping let mapping = crate::api::Mapping { @@ -438,7 +438,7 @@ mod tests { filename: "test_binary", build_id: "test_build_id", }; - let _mapping_id = internal_profile.add_mapping(&mapping); + let _mapping_id = internal_profile.try_add_mapping(&mapping); // Add a location let location = crate::api::Location { @@ -447,10 +447,10 @@ mod tests { address: 0x1000, line: 42, }; - let location_id = internal_profile.add_location(&location).unwrap(); + let location_id = internal_profile.try_add_location(&location).unwrap(); // Add a stack trace - let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); // Add a sample with values let sample = crate::api::Sample { @@ -458,7 +458,7 @@ mod tests { values: &[100, 2048], // 100 nanoseconds, 2048 bytes labels: vec![], }; - let _ = internal_profile.add_sample(sample, None); + let _ = internal_profile.try_add_sample(sample, None); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -510,7 +510,7 @@ mod tests { system_name: "test_system", filename: "test_file.rs", }; - let _function_id = internal_profile.add_function(&function); + let _function_id = internal_profile.try_add_function(&function); // Add a mapping let mapping = crate::api::Mapping { @@ -520,7 +520,7 @@ mod tests { filename: "test_binary", build_id: "test_build_id", }; - let _mapping_id = internal_profile.add_mapping(&mapping); + let _mapping_id = internal_profile.try_add_mapping(&mapping); let location = crate::api::Location { mapping, @@ -528,9 +528,9 @@ mod tests { address: 0x1000, line: 42, }; - let location_id = internal_profile.add_location(&location).unwrap(); + let location_id = internal_profile.try_add_location(&location).unwrap(); - let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); // Add a sample with labels let sample = crate::api::Sample { @@ -551,7 +551,7 @@ mod tests { }, ], }; - let _ = internal_profile.add_sample(sample, None); + let _ = internal_profile.try_add_sample(sample, None); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -595,7 +595,7 @@ mod tests { system_name: "test_system", filename: "test_file.rs", }; - let _function_id = internal_profile.add_function(&function); + let _function_id = internal_profile.try_add_function(&function); // Add a mapping let mapping = crate::api::Mapping { @@ -605,7 +605,7 @@ mod tests { filename: "test_binary", build_id: "test_build_id", }; - let _mapping_id = internal_profile.add_mapping(&mapping); + let _mapping_id = internal_profile.try_add_mapping(&mapping); let location = crate::api::Location { mapping, @@ -613,9 +613,9 @@ mod tests { address: 0x1000, line: 42, }; - let location_id = internal_profile.add_location(&location).unwrap(); + let location_id = internal_profile.try_add_location(&location).unwrap(); - let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); // Add a sample with timestamp let sample = crate::api::Sample { @@ -624,7 +624,7 @@ mod tests { labels: vec![], }; let timestamp = crate::internal::Timestamp::new(1234567890).unwrap(); - let _ = internal_profile.add_sample(sample, Some(timestamp)); + let _ = internal_profile.try_add_sample(sample, Some(timestamp)); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -660,7 +660,7 @@ mod tests { system_name: "test_system", filename: "test_file.rs", }; - let _function_id = internal_profile.add_function(&function); + let _function_id = internal_profile.try_add_function(&function); // Add a mapping let mapping = crate::api::Mapping { @@ -670,7 +670,7 @@ mod tests { filename: "test_binary", build_id: "test_build_id", }; - let _mapping_id = internal_profile.add_mapping(&mapping); + let _mapping_id = internal_profile.try_add_mapping(&mapping); let location = crate::api::Location { mapping, @@ -678,9 +678,9 @@ mod tests { address: 0x1000, line: 42, }; - let location_id = internal_profile.add_location(&location).unwrap(); + let location_id = internal_profile.try_add_location(&location).unwrap(); - let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); // Add a sample with one zero value and one non-zero value let sample = crate::api::Sample { @@ -688,7 +688,7 @@ mod tests { values: &[0, 1024], // 0 nanoseconds, 1024 bytes labels: vec![], }; - let _ = internal_profile.add_sample(sample, None); + let _ = internal_profile.try_add_sample(sample, None); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -724,7 +724,7 @@ mod tests { system_name: "test_system", filename: "test_file.rs", }; - let _function_id = internal_profile.add_function(&function); + let _function_id = internal_profile.try_add_function(&function); // Add a mapping let mapping = crate::api::Mapping { @@ -734,7 +734,7 @@ mod tests { filename: "test_binary", build_id: "test_build_id", }; - let _mapping_id = internal_profile.add_mapping(&mapping); + let _mapping_id = internal_profile.try_add_mapping(&mapping); let location = crate::api::Location { mapping, @@ -742,9 +742,9 @@ mod tests { address: 0x1000, line: 42, }; - let location_id = internal_profile.add_location(&location).unwrap(); + let location_id = internal_profile.try_add_location(&location).unwrap(); - let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); // Add multiple samples let sample1 = crate::api::Sample { @@ -763,9 +763,9 @@ mod tests { labels: vec![], }; - let _ = internal_profile.add_sample(sample1, None); - let _ = internal_profile.add_sample(sample2, None); - let _ = internal_profile.add_sample(sample3, None); + let _ = internal_profile.try_add_sample(sample1, None); + let _ = internal_profile.try_add_sample(sample2, None); + let _ = internal_profile.try_add_sample(sample3, None); // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); @@ -889,7 +889,7 @@ mod tests { system_name: "test_system", filename: "test_file.rs", }; - let _function_id = internal_profile.add_function(&function); + let _function_id = internal_profile.try_add_function(&function); // Add a mapping let mapping = crate::api::Mapping { @@ -899,7 +899,7 @@ mod tests { filename: "test_binary", build_id: "test_build_id", }; - let _mapping_id = internal_profile.add_mapping(&mapping); + let _mapping_id = internal_profile.try_add_mapping(&mapping); let location = crate::api::Location { mapping, @@ -907,9 +907,9 @@ mod tests { address: 0x1000, line: 42, }; - let location_id = internal_profile.add_location(&location).unwrap(); + let location_id = internal_profile.try_add_location(&location).unwrap(); - let _stack_trace_id = internal_profile.add_stacktrace(vec![location_id]); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); // Add a sample let sample = crate::api::Sample { @@ -917,7 +917,7 @@ mod tests { values: &[150], labels: vec![], }; - let _ = internal_profile.add_sample(sample, None); + let _ = internal_profile.try_add_sample(sample, None); // Test serialization to compressed OpenTelemetry format let encoded_profile = internal_profile diff --git a/datadog-trace-protobuf/src/remoteconfig.rs b/datadog-trace-protobuf/src/remoteconfig.rs index a8a4b66524..fdc89ee871 100644 --- a/datadog-trace-protobuf/src/remoteconfig.rs +++ b/datadog-trace-protobuf/src/remoteconfig.rs @@ -3,7 +3,8 @@ use serde::{Deserialize, Serialize}; // This file is @generated by prost-build. -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct File { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -11,7 +12,8 @@ pub struct File { #[serde(with = "serde_bytes")] pub raw: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Client { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, @@ -33,7 +35,8 @@ pub struct Client { #[prost(bytes = "vec", tag = "11")] pub capabilities: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientTracer { #[prost(string, tag = "1")] pub runtime_id: ::prost::alloc::string::String, @@ -52,7 +55,8 @@ pub struct ClientTracer { #[prost(string, repeated, tag = "7")] pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientAgent { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -65,7 +69,8 @@ pub struct ClientAgent { #[prost(string, repeated, tag = "5")] pub cws_workloads: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ConfigState { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -78,7 +83,8 @@ pub struct ConfigState { #[prost(string, tag = "5")] pub apply_error: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientState { #[prost(uint64, tag = "1")] pub root_version: u64, @@ -93,14 +99,16 @@ pub struct ClientState { #[prost(bytes = "vec", tag = "6")] pub backend_client_state: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TargetFileHash { #[prost(string, tag = "1")] pub algorithm: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub hash: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TargetFileMeta { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -109,14 +117,16 @@ pub struct TargetFileMeta { #[prost(message, repeated, tag = "3")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsRequest { #[prost(message, optional, tag = "1")] pub client: ::core::option::Option, #[prost(message, repeated, tag = "2")] pub cached_target_files: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsResponse { #[prost(bytes = "vec", repeated, tag = "1")] #[serde(with = "crate::serde")] From 5c1b2af9f17539ed87106074cba9846226004bbe Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 15:50:00 -0400 Subject: [PATCH 10/23] simplify tests --- bin_tests/tests/crashtracker_bin_test.rs | 1 + .../internal/profile/otel_emitter/profile.rs | 278 +++++------------- tools/src/lib.rs | 4 +- 3 files changed, 80 insertions(+), 203 deletions(-) diff --git a/bin_tests/tests/crashtracker_bin_test.rs b/bin_tests/tests/crashtracker_bin_test.rs index 0665bd271f..d7006c777b 100644 --- a/bin_tests/tests/crashtracker_bin_test.rs +++ b/bin_tests/tests/crashtracker_bin_test.rs @@ -139,6 +139,7 @@ fn test_crasht_tracking_validate_callstack() { test_crash_tracking_callstack() } +#[cfg(not(any(all(target_arch = "x86_64", target_env = "musl"), target_os = "macos")))] fn test_crash_tracking_callstack() { let (_, crashtracker_receiver) = setup_crashtracking_crates(BuildProfile::Release); diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index 525514ec16..b2faf54f6f 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -209,6 +209,68 @@ impl InternalProfile { mod tests { use crate::internal::profile::Profile as InternalProfile; + // Helper functions for test setup + fn create_basic_function() -> crate::api::Function<'static> { + crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + } + } + + fn create_basic_mapping() -> crate::api::Mapping<'static> { + crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + } + } + + fn setup_profile_with_function_and_location( + sample_types: &[crate::api::ValueType<'static>], + ) -> (InternalProfile, crate::api::Location<'static>) { + let mut internal_profile = InternalProfile::new(sample_types, None); + let function = create_basic_function(); + let mapping = create_basic_mapping(); + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + + let _function_id = internal_profile.try_add_function(&function); + let _mapping_id = internal_profile.try_add_mapping(&mapping); + let location_id = internal_profile.try_add_location(&location).unwrap(); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + + (internal_profile, location) + } + + fn create_string_label(key: &'static str, value: &'static str) -> crate::api::Label<'static> { + crate::api::Label { + key, + str: value, + num: 0, + num_unit: "", + } + } + + fn create_numeric_label( + key: &'static str, + value: i64, + unit: &'static str, + ) -> crate::api::Label<'static> { + crate::api::Label { + key, + str: "", + num: value, + num_unit: unit, + } + } + #[test] fn test_from_internal_profile_empty() { // Create an empty internal profile @@ -300,18 +362,8 @@ mod tests { let mut internal_profile = InternalProfile::new(&[], None); // Add some labels using the API - let label1 = crate::api::Label { - key: "thread_id", - str: "main", - num: 0, - num_unit: "", - }; - let label2 = crate::api::Label { - key: "memory_usage", - str: "", - num: 1024, - num_unit: "bytes", - }; + let label1 = create_string_label("thread_id", "main"); + let label2 = create_numeric_label("memory_usage", 1024, "bytes"); // Add a sample with these labels let sample = crate::api::Sample { @@ -420,37 +472,8 @@ mod tests { crate::api::ValueType::new("cpu", "nanoseconds"), crate::api::ValueType::new("memory", "bytes"), ]; - let mut internal_profile = InternalProfile::new(&sample_types, None); - - // Add a function to create a location - let function = crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - }; - let _function_id = internal_profile.try_add_function(&function); - - // Add a mapping - let mapping = crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - }; - let _mapping_id = internal_profile.try_add_mapping(&mapping); - - // Add a location - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - let location_id = internal_profile.try_add_location(&location).unwrap(); - - // Add a stack trace - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); // Add a sample with values let sample = crate::api::Sample { @@ -502,53 +525,16 @@ mod tests { fn test_sample_conversion_with_labels() { // Create an internal profile with sample types let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let mut internal_profile = InternalProfile::new(&sample_types, None); - - // Add a function and location - let function = crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - }; - let _function_id = internal_profile.try_add_function(&function); - - // Add a mapping - let mapping = crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - }; - let _mapping_id = internal_profile.try_add_mapping(&mapping); - - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - let location_id = internal_profile.try_add_location(&location).unwrap(); - - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); // Add a sample with labels let sample = crate::api::Sample { locations: vec![location], values: &[150], labels: vec![ - crate::api::Label { - key: "thread_id", - str: "main", - num: 0, - num_unit: "", - }, - crate::api::Label { - key: "cpu_usage", - str: "", - num: 75, - num_unit: "percent", - }, + create_string_label("thread_id", "main"), + create_numeric_label("cpu_usage", 75, "percent"), ], }; let _ = internal_profile.try_add_sample(sample, None); @@ -587,35 +573,8 @@ mod tests { fn test_sample_conversion_with_timestamps() { // Create an internal profile with sample types let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let mut internal_profile = InternalProfile::new(&sample_types, None); - - // Add a function and location - let function = crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - }; - let _function_id = internal_profile.try_add_function(&function); - - // Add a mapping - let mapping = crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - }; - let _mapping_id = internal_profile.try_add_mapping(&mapping); - - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - let location_id = internal_profile.try_add_location(&location).unwrap(); - - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); // Add a sample with timestamp let sample = crate::api::Sample { @@ -652,35 +611,8 @@ mod tests { crate::api::ValueType::new("cpu", "nanoseconds"), crate::api::ValueType::new("memory", "bytes"), ]; - let mut internal_profile = InternalProfile::new(&sample_types, None); - - // Add a function and location - let function = crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - }; - let _function_id = internal_profile.try_add_function(&function); - - // Add a mapping - let mapping = crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - }; - let _mapping_id = internal_profile.try_add_mapping(&mapping); - - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - let location_id = internal_profile.try_add_location(&location).unwrap(); - - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); // Add a sample with one zero value and one non-zero value let sample = crate::api::Sample { @@ -716,35 +648,8 @@ mod tests { fn test_sample_conversion_multiple_samples() { // Create an internal profile with sample types let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let mut internal_profile = InternalProfile::new(&sample_types, None); - - // Add a function and location - let function = crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - }; - let _function_id = internal_profile.try_add_function(&function); - - // Add a mapping - let mapping = crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - }; - let _mapping_id = internal_profile.try_add_mapping(&mapping); - - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - let location_id = internal_profile.try_add_location(&location).unwrap(); - - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); // Add multiple samples let sample1 = crate::api::Sample { @@ -881,35 +786,8 @@ mod tests { fn test_serialize_into_compressed_otel() { // Create an internal profile with sample types let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let mut internal_profile = InternalProfile::new(&sample_types, None); - - // Add a function and location - let function = crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - }; - let _function_id = internal_profile.try_add_function(&function); - - // Add a mapping - let mapping = crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - }; - let _mapping_id = internal_profile.try_add_mapping(&mapping); - - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - let location_id = internal_profile.try_add_location(&location).unwrap(); - - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); // Add a sample let sample = crate::api::Sample { diff --git a/tools/src/lib.rs b/tools/src/lib.rs index c573a0f0d7..754c548987 100644 --- a/tools/src/lib.rs +++ b/tools/src/lib.rs @@ -168,9 +168,7 @@ pub mod headers { assert_eq!( matches.len(), expected.len(), - "Expected:\n{:#?}\nActual:\n{:#?}", - expected, - matches + "Expected:\n{expected:#?}\nActual:\n{matches:#?}", ); for (i, m) in matches.iter().enumerate() { assert_eq!(m.str, expected[i]); From 29f6c6729035d3892c0037691ef6764a5114c8cd Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 15:56:11 -0400 Subject: [PATCH 11/23] simplify the tests a bit more --- .../internal/profile/otel_emitter/profile.rs | 158 +++++++++--------- 1 file changed, 77 insertions(+), 81 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index b2faf54f6f..df199cb054 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -271,18 +271,15 @@ mod tests { } } - #[test] - fn test_from_internal_profile_empty() { - // Create an empty internal profile - let internal_profile = InternalProfile::new(&[], None); - - // Convert to OpenTelemetry ProfilesData - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert!(otel_profiles_data.dictionary.is_some()); - let dictionary = otel_profiles_data.dictionary.unwrap(); + // Common assertion helpers + fn assert_duration_calculation(profiles: &[datadog_profiling_otel::Profile]) { + for profile in profiles { + // When no duration is provided, it should calculate from current time - start time + assert!(profile.duration_nanos > 0); + } + } + fn assert_basic_dictionary_structure(dictionary: &datadog_profiling_otel::ProfilesDictionary) { assert_eq!(dictionary.mapping_table.len(), 0); assert_eq!(dictionary.location_table.len(), 0); assert_eq!(dictionary.function_table.len(), 0); @@ -291,17 +288,49 @@ mod tests { assert_eq!(dictionary.link_table.len(), 0); assert_eq!(dictionary.attribute_table.len(), 0); assert_eq!(dictionary.attribute_units.len(), 0); + } + + fn assert_profile_has_correct_sample( + profile: &datadog_profiling_otel::Profile, + expected_values: Vec, + expected_stack_index: i32, + expected_attribute_count: usize, + ) { + assert_eq!(profile.sample.len(), 1); + let sample = &profile.sample[0]; + assert_eq!(sample.values, expected_values); + assert_eq!(sample.stack_index, expected_stack_index); + assert_eq!(sample.attribute_indices.len(), expected_attribute_count); + } + fn assert_sample_has_timestamp(sample: &datadog_profiling_otel::Sample, expected_timestamp: u64) { + assert_eq!(sample.timestamps_unix_nano.len(), 1); + assert_eq!(sample.timestamps_unix_nano[0], expected_timestamp); + } + + fn assert_profiles_data_structure(otel_profiles_data: &datadog_profiling_otel::ProfilesData) { + assert!(otel_profiles_data.dictionary.is_some()); assert_eq!(otel_profiles_data.resource_profiles.len(), 1); + assert_eq!(otel_profiles_data.resource_profiles[0].scope_profiles.len(), 1); + } + + #[test] + fn test_from_internal_profile_empty() { + // Create an empty internal profile + let internal_profile = InternalProfile::new(&[], None); + + // Convert to OpenTelemetry ProfilesData + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + + // Verify the conversion + assert_profiles_data_structure(&otel_profiles_data); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_basic_dictionary_structure(&dictionary); // Check duration calculation - only if profiles exist let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; if !scope_profile.profiles.is_empty() { - let profile = &scope_profile.profiles[0]; - // When no duration is provided, it should calculate from current time - start time - // Since we're testing with None, None, the duration should be > 0 (current time - start - // time) - assert!(profile.duration_nanos > 0); + assert_duration_calculation(&scope_profile.profiles); } } @@ -329,7 +358,7 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion - assert!(otel_profiles_data.dictionary.is_some()); + assert_profiles_data_structure(&otel_profiles_data); let dictionary = otel_profiles_data.dictionary.unwrap(); assert_eq!(dictionary.function_table.len(), 2); @@ -350,9 +379,7 @@ mod tests { // Check duration calculation - only if profiles exist let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; if !scope_profile.profiles.is_empty() { - let profile = &scope_profile.profiles[0]; - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); + assert_duration_calculation(&scope_profile.profiles); } } @@ -378,7 +405,7 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion - assert!(otel_profiles_data.dictionary.is_some()); + assert_profiles_data_structure(&otel_profiles_data); let dictionary = otel_profiles_data.dictionary.unwrap(); // Should have 2 labels converted to attributes @@ -417,9 +444,7 @@ mod tests { // Check duration calculation - only if profiles exist let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; if !scope_profile.profiles.is_empty() { - let profile = &scope_profile.profiles[0]; - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); + assert_duration_calculation(&scope_profile.profiles); } } @@ -436,10 +461,8 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify that individual profiles are created for each sample type - assert_eq!(otel_profiles_data.resource_profiles.len(), 1); - let resource_profile = &otel_profiles_data.resource_profiles[0]; - assert_eq!(resource_profile.scope_profiles.len(), 1); - let scope_profile = &resource_profile.scope_profiles[0]; + assert_profiles_data_structure(&otel_profiles_data); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; // Should have 2 profiles (one for each sample type) assert_eq!(scope_profile.profiles.len(), 2); @@ -459,10 +482,7 @@ mod tests { assert_eq!(allocations_sample_type.unit_strindex, 7); // "count" string index // Check duration calculation for both profiles - for profile in &scope_profile.profiles { - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); - } + assert_duration_calculation(&scope_profile.profiles); } #[test] @@ -487,38 +507,21 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion - assert!(otel_profiles_data.dictionary.is_some()); + assert_profiles_data_structure(&otel_profiles_data); let _dictionary = otel_profiles_data.dictionary.unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; // Should have 2 profiles (one for each sample type) - assert_eq!( - otel_profiles_data.resource_profiles[0].scope_profiles[0] - .profiles - .len(), - 2 - ); + assert_eq!(scope_profile.profiles.len(), 2); // Verify the first profile (cpu profile) has the correct sample - let cpu_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_eq!(cpu_profile.sample.len(), 1); - let cpu_sample = &cpu_profile.sample[0]; - assert_eq!(cpu_sample.values, vec![100]); - assert_eq!(cpu_sample.stack_index, 0); // First stack trace - assert_eq!(cpu_sample.attribute_indices.len(), 0); // No labels + assert_profile_has_correct_sample(&scope_profile.profiles[0], vec![100], 0, 0); // Verify the second profile (memory profile) has the correct sample - let memory_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[1]; - assert_eq!(memory_profile.sample.len(), 1); - let memory_sample = &memory_profile.sample[0]; - assert_eq!(memory_sample.values, vec![2048]); - assert_eq!(memory_sample.stack_index, 0); // First stack trace - assert_eq!(memory_sample.attribute_indices.len(), 0); // No labels + assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![2048], 0, 0); // Check duration calculation for both profiles - for profile in &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles { - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); - } + assert_duration_calculation(&scope_profile.profiles); } #[test] @@ -543,26 +546,21 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion + assert_profiles_data_structure(&otel_profiles_data); let _dictionary = otel_profiles_data.dictionary.unwrap(); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - // Should have 1 sample - assert_eq!(profile.sample.len(), 1); - let sample = &profile.sample[0]; - - // Verify the sample has the correct values - assert_eq!(sample.values, vec![150]); - assert_eq!(sample.stack_index, 0); + // Should have 1 sample with correct values and attributes + assert_profile_has_correct_sample(profile, vec![150], 0, 2); // Verify the sample has the correct attribute indices - assert_eq!(sample.attribute_indices.len(), 2); + let sample = &profile.sample[0]; // The attribute indices should correspond to the labels in the attribute table assert!(sample.attribute_indices[0] >= 0); + assert!(sample.attribute_indices[1] >= 0); // Check duration calculation - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); - assert!(sample.attribute_indices[1] >= 0); + assert_duration_calculation(&[profile.clone()]); // Verify the attributes were converted correctly assert_eq!(_dictionary.attribute_table.len(), 2); @@ -589,6 +587,7 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion + assert_profiles_data_structure(&otel_profiles_data); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; // Should have 1 sample @@ -596,12 +595,10 @@ mod tests { let sample = &profile.sample[0]; // Verify the sample has the correct timestamp - assert_eq!(sample.timestamps_unix_nano.len(), 1); - assert_eq!(sample.timestamps_unix_nano[0], 1234567890); + assert_sample_has_timestamp(sample, 1234567890); // Check duration calculation - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); + assert_duration_calculation(&[profile.clone()]); } #[test] @@ -626,22 +623,19 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion - let profile0 = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - let profile1 = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[1]; + assert_profiles_data_structure(&otel_profiles_data); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + let profile0 = &scope_profile.profiles[0]; + let profile1 = &scope_profile.profiles[1]; // First profile (cpu) should have no samples since value is 0 assert_eq!(profile0.sample.len(), 0); // Second profile (memory) should have 1 sample since value is non-zero - assert_eq!(profile1.sample.len(), 1); - let memory_sample = &profile1.sample[0]; - assert_eq!(memory_sample.values, vec![1024]); + assert_profile_has_correct_sample(profile1, vec![1024], 0, 0); // Check duration calculation for both profiles - for profile in &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles { - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); - } + assert_duration_calculation(&scope_profile.profiles); } #[test] @@ -676,6 +670,7 @@ mod tests { let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); // Verify the conversion + assert_profiles_data_structure(&otel_profiles_data); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; // Should have 1 aggregated sample (samples with same stack trace and labels get aggregated) @@ -691,8 +686,7 @@ mod tests { } // Check duration calculation - // When no duration is provided, it should calculate from current time - start time - assert!(profile.duration_nanos > 0); + assert_duration_calculation(&[profile.clone()]); } #[test] @@ -751,6 +745,7 @@ mod tests { // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + assert_profiles_data_structure(&otel_profiles_data); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; // Should have period type information @@ -772,6 +767,7 @@ mod tests { .convert_into_otel(None, None) .unwrap(); + assert_profiles_data_structure(&otel_profiles_data_no_period); let profile_no_period = &otel_profiles_data_no_period.resource_profiles[0].scope_profiles[0].profiles[0]; From 4004cb5f8ff300dea8d247b175de24a6be59b5c7 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 16:02:16 -0400 Subject: [PATCH 12/23] remove unneeded comments --- datadog-profiling-otel/README.md | 70 +------------------ .../examples/basic_usage.rs | 5 -- .../internal/profile/otel_emitter/profile.rs | 10 ++- 3 files changed, 10 insertions(+), 75 deletions(-) diff --git a/datadog-profiling-otel/README.md b/datadog-profiling-otel/README.md index 3f1b6af71c..afb7798f5d 100644 --- a/datadog-profiling-otel/README.md +++ b/datadog-profiling-otel/README.md @@ -1,55 +1,11 @@ # datadog-profiling-otel This module provides Rust bindings for the OpenTelemetry profiling protobuf definitions, generated using the `prost` library. +This crate implements serialization of data into the otel profiling format; if you're building a profiler you usually don't want to use this crate directly, and instead should use datadog-profiling and ask it to serialize using ottel. ## Usage -### Basic Setup - -Add this to your `Cargo.toml`: - -```toml -[dependencies] -datadog-profiling-otel = "20.0.0" -``` - -### Creating Profile Data - -```rust -use datadog_profiling_otel::*; - -// Create a profiles dictionary -let mut profiles_dict = ProfilesDictionary::default(); -profiles_dict.string_table.push("cpu".to_string()); -profiles_dict.string_table.push("nanoseconds".to_string()); - -// Create a sample type -let sample_type = ValueType { - type_strindex: 0, // "cpu" - unit_strindex: 1, // "nanoseconds" - aggregation_temporality: AggregationTemporality::Delta.into(), -}; - -// Create a profile -let mut profile = Profile::default(); -profile.sample_type = Some(sample_type); -profile.time_nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() as i64; - -// Assemble the complete profiles data -let mut profiles_data = ProfilesData::default(); -profiles_data.dictionary = Some(profiles_dict); - -let mut scope_profiles = ScopeProfiles::default(); -scope_profiles.profiles.push(profile); - -let mut resource_profiles = ResourceProfiles::default(); -resource_profiles.scope_profiles.push(scope_profiles); - -profiles_data.resource_profiles.push(resource_profiles); -``` +See the [basic_usage.rs](examples/basic_usage.rs) example for a complete demonstration of how to create OpenTelemetry profile data. ### Running Examples @@ -64,26 +20,4 @@ cargo test cargo build ``` -## Module Structure - -The generated code follows the OpenTelemetry protobuf structure: - -- `ProfilesData`: Top-level container for all profile data -- `ResourceProfiles`: Profiles grouped by resource -- `ScopeProfiles`: Profiles grouped by instrumentation scope -- `Profile`: Individual profile with samples and metadata -- `ProfilesDictionary`: Shared data (strings, mappings, locations, etc.) -- `Sample`: Individual measurements with stack traces -- `Location`: Function locations in the call stack -- `Function`: Function information -- `Mapping`: Binary/library mapping information - -## Dependencies - -- `prost`: Protobuf implementation -- `prost-types`: Additional protobuf types -- `prost-build`: Build-time protobuf compilation - -## License -Apache-2.0 diff --git a/datadog-profiling-otel/examples/basic_usage.rs b/datadog-profiling-otel/examples/basic_usage.rs index a9678b4c87..4303e493a6 100644 --- a/datadog-profiling-otel/examples/basic_usage.rs +++ b/datadog-profiling-otel/examples/basic_usage.rs @@ -7,22 +7,18 @@ fn main() { use datadog_profiling_otel::*; - // Create a simple profile with some basic data let mut profiles_dict = ProfilesDictionary::default(); - // Add some strings to the string table profiles_dict.string_table.push("cpu".to_string()); profiles_dict.string_table.push("nanoseconds".to_string()); profiles_dict.string_table.push("main".to_string()); - // Create a sample type let sample_type = ValueType { type_strindex: 0, // "cpu" unit_strindex: 1, // "nanoseconds" aggregation_temporality: AggregationTemporality::Delta.into(), }; - // Create a profile let profile = Profile { sample_type: Some(sample_type), time_nanos: std::time::SystemTime::now() @@ -32,7 +28,6 @@ fn main() { ..Default::default() }; - // Create profiles data let mut profiles_data = ProfilesData { dictionary: Some(profiles_dict), ..Default::default() diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index df199cb054..fe91023e3b 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -303,7 +303,10 @@ mod tests { assert_eq!(sample.attribute_indices.len(), expected_attribute_count); } - fn assert_sample_has_timestamp(sample: &datadog_profiling_otel::Sample, expected_timestamp: u64) { + fn assert_sample_has_timestamp( + sample: &datadog_profiling_otel::Sample, + expected_timestamp: u64, + ) { assert_eq!(sample.timestamps_unix_nano.len(), 1); assert_eq!(sample.timestamps_unix_nano[0], expected_timestamp); } @@ -311,7 +314,10 @@ mod tests { fn assert_profiles_data_structure(otel_profiles_data: &datadog_profiling_otel::ProfilesData) { assert!(otel_profiles_data.dictionary.is_some()); assert_eq!(otel_profiles_data.resource_profiles.len(), 1); - assert_eq!(otel_profiles_data.resource_profiles[0].scope_profiles.len(), 1); + assert_eq!( + otel_profiles_data.resource_profiles[0].scope_profiles.len(), + 1 + ); } #[test] From f3682ee220255130848a28a9d0fbc004502b0ca7 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 16:08:30 -0400 Subject: [PATCH 13/23] PR comment: don't rename types --- .../internal/profile/otel_emitter/location.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/location.rs b/datadog-profiling/src/internal/profile/otel_emitter/location.rs index 27e137bffc..75cd3cdf32 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/location.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/location.rs @@ -2,18 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 use crate::collections::identifiable::Id; -use crate::internal::Location as InternalLocation; +use crate::internal; // For owned values - forward to reference version -impl From for datadog_profiling_otel::Location { - fn from(internal_location: InternalLocation) -> Self { +impl From for datadog_profiling_otel::Location { + fn from(internal_location: internal::Location) -> Self { Self::from(&internal_location) } } // For references (existing implementation) -impl From<&InternalLocation> for datadog_profiling_otel::Location { - fn from(internal_location: &InternalLocation) -> Self { +impl From<&internal::Location> for datadog_profiling_otel::Location { + fn from(internal_location: &internal::Location) -> Self { Self { mapping_index: internal_location .mapping_id @@ -39,7 +39,7 @@ mod tests { #[test] fn test_from_internal_location() { // Create an internal location - let internal_location = InternalLocation { + let internal_location = internal::Location { mapping_id: Some(MappingId::from_offset(1)), function_id: FunctionId::from_offset(2), address: 0x1000, @@ -63,7 +63,7 @@ mod tests { #[test] fn test_from_internal_location_no_mapping() { // Create an internal location without mapping - let internal_location = InternalLocation { + let internal_location = internal::Location { mapping_id: None, function_id: FunctionId::from_offset(5), address: 0x2000, @@ -84,7 +84,7 @@ mod tests { #[test] fn test_into_otel_location() { // Create an internal location - let internal_location = InternalLocation { + let internal_location = internal::Location { mapping_id: Some(MappingId::from_offset(10)), function_id: FunctionId::from_offset(20), address: 0x3000, @@ -104,7 +104,7 @@ mod tests { #[test] fn test_into_otel_location_owned() { // Create an internal location - let internal_location = InternalLocation { + let internal_location = internal::Location { mapping_id: Some(MappingId::from_offset(30)), function_id: FunctionId::from_offset(40), address: 0x4000, From b63529ef3df0a6ce887d58e97866a892c31599c1 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 16:17:52 -0400 Subject: [PATCH 14/23] cleanup tests --- .../internal/profile/otel_emitter/location.rs | 53 +----------------- .../internal/profile/otel_emitter/mapping.rs | 55 +------------------ .../profile/otel_emitter/stack_trace.rs | 49 +---------------- 3 files changed, 6 insertions(+), 151 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/location.rs b/datadog-profiling/src/internal/profile/otel_emitter/location.rs index 75cd3cdf32..6178debff6 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/location.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/location.rs @@ -38,7 +38,7 @@ mod tests { #[test] fn test_from_internal_location() { - // Create an internal location + // Test with mapping let internal_location = internal::Location { mapping_id: Some(MappingId::from_offset(1)), function_id: FunctionId::from_offset(2), @@ -46,10 +46,7 @@ mod tests { line: 42, }; - // Convert to OpenTelemetry Location let otel_location = datadog_profiling_otel::Location::from(&internal_location); - - // Verify the conversion - note: from_offset adds 1 to avoid zero values assert_eq!(otel_location.mapping_index, 2); assert_eq!(otel_location.address, 0x1000); assert_eq!(otel_location.line.len(), 1); @@ -58,11 +55,8 @@ mod tests { assert_eq!(otel_location.line[0].column, 0); assert!(!otel_location.is_folded); assert_eq!(otel_location.attribute_indices, vec![] as Vec); - } - #[test] - fn test_from_internal_location_no_mapping() { - // Create an internal location without mapping + // Test without mapping let internal_location = internal::Location { mapping_id: None, function_id: FunctionId::from_offset(5), @@ -70,54 +64,11 @@ mod tests { line: 100, }; - // Convert to OpenTelemetry Location let otel_location = datadog_profiling_otel::Location::from(&internal_location); - - // Verify the conversion assert_eq!(otel_location.mapping_index, 0); // 0 represents no mapping assert_eq!(otel_location.address, 0x2000); assert_eq!(otel_location.line.len(), 1); assert_eq!(otel_location.line[0].function_index, 6); assert_eq!(otel_location.line[0].line, 100); } - - #[test] - fn test_into_otel_location() { - // Create an internal location - let internal_location = internal::Location { - mapping_id: Some(MappingId::from_offset(10)), - function_id: FunctionId::from_offset(20), - address: 0x3000, - line: 200, - }; - - // Convert using .into() method - let otel_location: datadog_profiling_otel::Location = (&internal_location).into(); - - // Verify the conversion - note: from_offset adds 1 to avoid zero values - assert_eq!(otel_location.mapping_index, 11); - assert_eq!(otel_location.address, 0x3000); - assert_eq!(otel_location.line[0].function_index, 21); - assert_eq!(otel_location.line[0].line, 200); - } - - #[test] - fn test_into_otel_location_owned() { - // Create an internal location - let internal_location = internal::Location { - mapping_id: Some(MappingId::from_offset(30)), - function_id: FunctionId::from_offset(40), - address: 0x4000, - line: 300, - }; - - // Convert using .into() method with owned value - let otel_location: datadog_profiling_otel::Location = internal_location.into(); - - // Verify the conversion - note: from_offset adds 1 to avoid zero values - assert_eq!(otel_location.mapping_index, 31); - assert_eq!(otel_location.address, 0x4000); - assert_eq!(otel_location.line[0].function_index, 41); - assert_eq!(otel_location.line[0].line, 300); - } } diff --git a/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs b/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs index 769f5dbdf7..351acaba9f 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/mapping.rs @@ -35,7 +35,7 @@ mod tests { #[test] fn test_from_internal_mapping() { - // Create an internal mapping + // Test basic conversion let internal_mapping = InternalMapping { memory_start: 0x1000, memory_limit: 0x2000, @@ -44,10 +44,7 @@ mod tests { build_id: StringId::from_offset(123), }; - // Convert to OpenTelemetry Mapping let otel_mapping = datadog_profiling_otel::Mapping::from(&internal_mapping); - - // Verify the conversion assert_eq!(otel_mapping.memory_start, 0x1000); assert_eq!(otel_mapping.memory_limit, 0x2000); assert_eq!(otel_mapping.file_offset, 0x100); @@ -57,11 +54,8 @@ mod tests { assert!(otel_mapping.has_filenames); assert!(otel_mapping.has_line_numbers); assert!(!otel_mapping.has_inline_frames); - } - #[test] - fn test_from_internal_mapping_large_values() { - // Create an internal mapping with large values + // Test with large values let internal_mapping = InternalMapping { memory_start: 0x1000000000000000, memory_limit: 0x2000000000000000, @@ -70,55 +64,10 @@ mod tests { build_id: StringId::from_offset(888), }; - // Convert to OpenTelemetry Mapping let otel_mapping = datadog_profiling_otel::Mapping::from(&internal_mapping); - - // Verify the conversion assert_eq!(otel_mapping.memory_start, 0x1000000000000000); assert_eq!(otel_mapping.memory_limit, 0x2000000000000000); assert_eq!(otel_mapping.file_offset, 0x1000000000000000); assert_eq!(otel_mapping.filename_strindex, 999); } - - #[test] - fn test_into_otel_mapping() { - // Create an internal mapping - let internal_mapping = InternalMapping { - memory_start: 0x3000, - memory_limit: 0x4000, - file_offset: 0x200, - filename: StringId::from_offset(555), - build_id: StringId::from_offset(666), - }; - - // Convert using .into() method - let otel_mapping: datadog_profiling_otel::Mapping = (&internal_mapping).into(); - - // Verify the conversion - assert_eq!(otel_mapping.memory_start, 0x3000); - assert_eq!(otel_mapping.memory_limit, 0x4000); - assert_eq!(otel_mapping.file_offset, 0x200); - assert_eq!(otel_mapping.filename_strindex, 555); - } - - #[test] - fn test_into_otel_mapping_owned() { - // Create an internal mapping - let internal_mapping = InternalMapping { - memory_start: 0x5000, - memory_limit: 0x6000, - file_offset: 0x300, - filename: StringId::from_offset(777), - build_id: StringId::from_offset(888), - }; - - // Convert using .into() method with owned value - let otel_mapping: datadog_profiling_otel::Mapping = internal_mapping.into(); - - // Verify the conversion - assert_eq!(otel_mapping.memory_start, 0x5000); - assert_eq!(otel_mapping.memory_limit, 0x6000); - assert_eq!(otel_mapping.file_offset, 0x300); - assert_eq!(otel_mapping.filename_strindex, 777); - } } diff --git a/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs b/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs index ec7c3e6d93..5f20dfcb53 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/stack_trace.rs @@ -31,7 +31,7 @@ mod tests { #[test] fn test_from_internal_stack_trace() { - // Create an internal stack trace + // Test with locations let internal_stack_trace = InternalStackTrace { locations: vec![ LocationId::from_offset(0), @@ -40,58 +40,13 @@ mod tests { ], }; - // Convert to OpenTelemetry Stack let otel_stack = datadog_profiling_otel::Stack::from(&internal_stack_trace); - - // Verify the conversion - note: from_offset adds 1 to avoid zero values assert_eq!(otel_stack.location_indices, vec![1, 2, 3]); - } - #[test] - fn test_from_empty_stack_trace() { - // Create an internal stack trace with no locations + // Test with empty locations let internal_stack_trace = InternalStackTrace { locations: vec![] }; - // Convert to OpenTelemetry Stack let otel_stack = datadog_profiling_otel::Stack::from(&internal_stack_trace); - - // Verify the conversion assert_eq!(otel_stack.location_indices, vec![] as Vec); } - - #[test] - fn test_into_otel_stack() { - // Create an internal stack trace - let internal_stack_trace = InternalStackTrace { - locations: vec![ - LocationId::from_offset(10), - LocationId::from_offset(20), - LocationId::from_offset(30), - ], - }; - - // Convert using .into() method - let otel_stack: datadog_profiling_otel::Stack = (&internal_stack_trace).into(); - - // Verify the conversion - note: from_offset adds 1 to avoid zero values - assert_eq!(otel_stack.location_indices, vec![11, 21, 31]); - } - - #[test] - fn test_into_otel_stack_owned() { - // Create an internal stack trace - let internal_stack_trace = InternalStackTrace { - locations: vec![ - LocationId::from_offset(40), - LocationId::from_offset(50), - LocationId::from_offset(60), - ], - }; - - // Convert using .into() method with owned value - let otel_stack: datadog_profiling_otel::Stack = internal_stack_trace.into(); - - // Verify the conversion - note: from_offset adds 1 to avoid zero values - assert_eq!(otel_stack.location_indices, vec![41, 51, 61]); - } } From 819f5717552dc6b6289b28b8c756c2d26efe3fe9 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Wed, 10 Sep 2025 16:19:17 -0400 Subject: [PATCH 15/23] serialize the example --- datadog-profiling-otel/examples/basic_usage.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datadog-profiling-otel/examples/basic_usage.rs b/datadog-profiling-otel/examples/basic_usage.rs index 4303e493a6..d0ccc8ab5f 100644 --- a/datadog-profiling-otel/examples/basic_usage.rs +++ b/datadog-profiling-otel/examples/basic_usage.rs @@ -50,4 +50,6 @@ fn main() { "Time: {}", profiles_data.resource_profiles[0].scope_profiles[0].profiles[0].time_nanos ); + let serialized = profiles_data.serialize_into_compressed_proto().unwrap(); + println!("Serialized size: {} bytes", serialized.len()); } From dcef39a4895d7351ca6184e6d812c803149ac111 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 11 Sep 2025 15:38:48 -0400 Subject: [PATCH 16/23] simplify tests --- .../internal/profile/otel_emitter/function.rs | 51 +------ .../internal/profile/otel_emitter/label.rs | 143 ++++++------------ .../internal/profile/otel_emitter/profile.rs | 2 +- 3 files changed, 46 insertions(+), 150 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/function.rs b/datadog-profiling/src/internal/profile/otel_emitter/function.rs index 78867d24b5..32b6de338b 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/function.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/function.rs @@ -30,77 +30,30 @@ mod tests { #[test] fn test_from_internal_function() { - // Create an internal function + // Test basic conversion let internal_function = InternalFunction { name: StringId::from_offset(0), system_name: StringId::from_offset(1), filename: StringId::from_offset(2), }; - // Convert to OpenTelemetry Function let otel_function = datadog_profiling_otel::Function::from(&internal_function); - - // Verify the conversion - note: StringId doesn't add 1, it's direct conversion assert_eq!(otel_function.name_strindex, 0); assert_eq!(otel_function.system_name_strindex, 1); assert_eq!(otel_function.filename_strindex, 2); assert_eq!(otel_function.start_line, 0); - } - #[test] - fn test_from_internal_function_with_large_offsets() { - // Create an internal function with large offsets + // Test with large offsets let internal_function = InternalFunction { name: StringId::from_offset(999999), system_name: StringId::from_offset(888888), filename: StringId::from_offset(777777), }; - // Convert to OpenTelemetry Function let otel_function = datadog_profiling_otel::Function::from(&internal_function); - - // Verify the conversion assert_eq!(otel_function.name_strindex, 999999); assert_eq!(otel_function.system_name_strindex, 888888); assert_eq!(otel_function.filename_strindex, 777777); assert_eq!(otel_function.start_line, 0); } - - #[test] - fn test_into_otel_function() { - // Create an internal function - let internal_function = InternalFunction { - name: StringId::from_offset(100), - system_name: StringId::from_offset(200), - filename: StringId::from_offset(300), - }; - - // Convert using .into() method - let otel_function: datadog_profiling_otel::Function = (&internal_function).into(); - - // Verify the conversion - assert_eq!(otel_function.name_strindex, 100); - assert_eq!(otel_function.system_name_strindex, 200); - assert_eq!(otel_function.filename_strindex, 300); - assert_eq!(otel_function.start_line, 0); - } - - #[test] - fn test_into_otel_function_owned() { - // Create an internal function - let internal_function = InternalFunction { - name: StringId::from_offset(400), - system_name: StringId::from_offset(500), - filename: StringId::from_offset(600), - }; - - // Convert using .into() method with owned value - let otel_function: datadog_profiling_otel::Function = internal_function.into(); - - // Verify the conversion - assert_eq!(otel_function.name_strindex, 400); - assert_eq!(otel_function.system_name_strindex, 500); - assert_eq!(otel_function.filename_strindex, 600); - assert_eq!(otel_function.start_line, 0); - } } diff --git a/datadog-profiling/src/internal/profile/otel_emitter/label.rs b/datadog-profiling/src/internal/profile/otel_emitter/label.rs index 0824ed8ae1..e7c46ab9a6 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/label.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/label.rs @@ -83,7 +83,8 @@ mod tests { use crate::collections::identifiable::StringId; #[test] - fn test_convert_string_label() { + fn test_convert_label() { + // Test string label conversion let string_table = vec![ "".to_string(), // index 0 "thread_id".to_string(), // index 1 @@ -96,49 +97,57 @@ mod tests { ); let mut key_to_unit_map = HashMap::new(); - let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); - assert!(result.is_ok()); - - let key_value = result.unwrap(); + let key_value = + convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map).unwrap(); assert_eq!(key_value.key, "thread_id"); - match key_value.value { - Some(datadog_profiling_otel::key_value::Value::StringValue(s)) => { - assert_eq!(s, "main"); - } + let s = match key_value.value.expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::StringValue(s) => s, _ => panic!("Expected StringValue"), - } - } - - #[test] - fn test_convert_numeric_label() { - let string_table = vec![ - "".to_string(), // index 0 - "allocation_size".to_string(), // index 1 - "bytes".to_string(), // index 2 - ]; + }; + assert_eq!(s, "main"); + // Test numeric label with unit mapping let label = InternalLabel::num( - StringId::from_offset(1), // "allocation_size" - 1024, // 1024 bytes - StringId::from_offset(2), // "bytes" + StringId::from_offset(1), // "thread_id" (reusing key) + 1024, // 1024 + StringId::from_offset(2), // "main" (reusing as unit) ); - let mut key_to_unit_map = HashMap::new(); - let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); - assert!(result.is_ok()); + let key_value = + convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map).unwrap(); + assert_eq!(key_value.key, "thread_id"); + let n = match key_value.value.expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, + _ => panic!("Expected IntValue"), + }; + assert_eq!(n, 1024); - let key_value = result.unwrap(); - assert_eq!(key_value.key, "allocation_size"); - match key_value.value { - Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { - assert_eq!(n, 1024); - } + // Verify unit mapping was added + assert_eq!(key_to_unit_map.get(&1), Some(&2)); + + // Test numeric label without unit mapping (empty string unit) + let label = InternalLabel::num( + StringId::from_offset(1), // "thread_id" + 42, // 42 + StringId::from_offset(0), // empty string (default) + ); + + let key_value = + convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map).unwrap(); + assert_eq!(key_value.key, "thread_id"); + let n = match key_value.value.expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, _ => panic!("Expected IntValue"), - } + }; + assert_eq!(n, 42); + + // Unit mapping should still exist from previous test + assert_eq!(key_to_unit_map.get(&1), Some(&2)); } #[test] - fn test_convert_label_out_of_bounds() { + fn test_convert_label_errors() { + // Test out of bounds key let string_table = vec!["".to_string()]; // Only one string let label = InternalLabel::str( @@ -149,10 +158,8 @@ mod tests { let mut key_to_unit_map = HashMap::new(); let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); assert!(result.is_err()); - } - #[test] - fn test_convert_label_empty_string_table() { + // Test empty string table let string_table: Vec = vec![]; let label = InternalLabel::str( @@ -160,71 +167,7 @@ mod tests { StringId::from_offset(0), ); - let mut key_to_unit_map = HashMap::new(); let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); assert!(result.is_err()); } - - #[test] - fn test_convert_numeric_label_with_unit_mapping() { - let string_table = vec![ - "".to_string(), // index 0 - "memory_usage".to_string(), // index 1 - "megabytes".to_string(), // index 2 - ]; - - let label = InternalLabel::num( - StringId::from_offset(1), // "memory_usage" - 512, // 512 MB - StringId::from_offset(2), // "megabytes" - ); - - let mut key_to_unit_map = HashMap::new(); - let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); - assert!(result.is_ok()); - - // Verify the KeyValue conversion - let key_value = result.unwrap(); - assert_eq!(key_value.key, "memory_usage"); - match key_value.value { - Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { - assert_eq!(n, 512); - } - _ => panic!("Expected IntValue"), - } - - // Verify the unit mapping was added - assert_eq!(key_to_unit_map.get(&1), Some(&2)); // key index 1 maps to unit index 2 - } - - #[test] - fn test_convert_numeric_label_without_unit_mapping() { - let string_table = vec![ - "".to_string(), // index 0 - "counter".to_string(), // index 1 - ]; - - let label = InternalLabel::num( - StringId::from_offset(1), // "counter" - 42, // 42 - StringId::from_offset(0), // empty string (default) - ); - - let mut key_to_unit_map = HashMap::new(); - let result = convert_label_to_key_value(&label, &string_table, &mut key_to_unit_map); - assert!(result.is_ok()); - - // Verify the KeyValue conversion - let key_value = result.unwrap(); - assert_eq!(key_value.key, "counter"); - match key_value.value { - Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { - assert_eq!(n, 42); - } - _ => panic!("Expected IntValue"), - } - - // Verify no unit mapping was added for default empty string - assert!(key_to_unit_map.is_empty()); - } } diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index fe91023e3b..c24fe70f5f 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -51,7 +51,7 @@ impl InternalProfile { // Create a Profile for this sample type let profile = datadog_profiling_otel::Profile { sample_type: Some(otel_sample_type), - sample: vec![], // TODO: Implement sample conversion + sample: vec![], time_nanos: self .start_time .duration_since(std::time::UNIX_EPOCH) From 3e1238c67d8cf58e540cc9616f81353c529d8bf8 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 11 Sep 2025 15:54:34 -0400 Subject: [PATCH 17/23] improve profile.rs tests --- .../internal/profile/otel_emitter/profile.rs | 357 ++++-------------- 1 file changed, 73 insertions(+), 284 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index c24fe70f5f..b18b038a6e 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -51,7 +51,7 @@ impl InternalProfile { // Create a Profile for this sample type let profile = datadog_profiling_otel::Profile { sample_type: Some(otel_sample_type), - sample: vec![], + sample: vec![], time_nanos: self .start_time .duration_since(std::time::UNIX_EPOCH) @@ -274,22 +274,10 @@ mod tests { // Common assertion helpers fn assert_duration_calculation(profiles: &[datadog_profiling_otel::Profile]) { for profile in profiles { - // When no duration is provided, it should calculate from current time - start time assert!(profile.duration_nanos > 0); } } - fn assert_basic_dictionary_structure(dictionary: &datadog_profiling_otel::ProfilesDictionary) { - assert_eq!(dictionary.mapping_table.len(), 0); - assert_eq!(dictionary.location_table.len(), 0); - assert_eq!(dictionary.function_table.len(), 0); - assert_eq!(dictionary.stack_table.len(), 0); - assert_eq!(dictionary.string_table.len(), 4); // Default strings: "", "local root span id", "trace endpoint", "end_timestamp_ns" - assert_eq!(dictionary.link_table.len(), 0); - assert_eq!(dictionary.attribute_table.len(), 0); - assert_eq!(dictionary.attribute_units.len(), 0); - } - fn assert_profile_has_correct_sample( profile: &datadog_profiling_otel::Profile, expected_values: Vec, @@ -321,31 +309,23 @@ mod tests { } #[test] - fn test_from_internal_profile_empty() { - // Create an empty internal profile + fn test_convert_into_otel() { + // Test empty profile let internal_profile = InternalProfile::new(&[], None); - - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion assert_profiles_data_structure(&otel_profiles_data); let dictionary = otel_profiles_data.dictionary.unwrap(); - assert_basic_dictionary_structure(&dictionary); - - // Check duration calculation - only if profiles exist - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - if !scope_profile.profiles.is_empty() { - assert_duration_calculation(&scope_profile.profiles); - } - } + assert_eq!(dictionary.mapping_table.len(), 0); + assert_eq!(dictionary.location_table.len(), 0); + assert_eq!(dictionary.function_table.len(), 0); + assert_eq!(dictionary.stack_table.len(), 0); + assert_eq!(dictionary.string_table.len(), 4); + assert_eq!(dictionary.link_table.len(), 0); + assert_eq!(dictionary.attribute_table.len(), 0); + assert_eq!(dictionary.attribute_units.len(), 0); - #[test] - fn test_from_internal_profile_with_data() { - // Create an internal profile with some data + // Test with functions let mut internal_profile = InternalProfile::new(&[], None); - - // Add some functions using the API Function type let function1 = crate::api::Function { name: "test_function_1", system_name: "test_system_1", @@ -356,144 +336,44 @@ mod tests { system_name: "test_system_2", filename: "test_file_2.rs", }; - let _function1_id = internal_profile.try_add_function(&function1); let _function2_id = internal_profile.try_add_function(&function2); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); let dictionary = otel_profiles_data.dictionary.unwrap(); - assert_eq!(dictionary.function_table.len(), 2); - assert_eq!(dictionary.string_table.len(), 10); // 4 default strings + 6 strings from the 2 functions - - // Verify the first function conversion - using actual observed values - let otel_function1 = &dictionary.function_table[0]; - assert_eq!(otel_function1.name_strindex, 4); - assert_eq!(otel_function1.system_name_strindex, 5); - assert_eq!(otel_function1.filename_strindex, 6); - - // Verify the second function conversion - using actual observed values - let otel_function2 = &dictionary.function_table[1]; - assert_eq!(otel_function2.name_strindex, 7); - assert_eq!(otel_function2.system_name_strindex, 8); - assert_eq!(otel_function2.filename_strindex, 9); + assert_eq!(dictionary.string_table.len(), 10); - // Check duration calculation - only if profiles exist - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - if !scope_profile.profiles.is_empty() { - assert_duration_calculation(&scope_profile.profiles); - } - } - - #[test] - fn test_from_internal_profile_with_labels() { - // Create an internal profile with some data + // Test with labels let mut internal_profile = InternalProfile::new(&[], None); - - // Add some labels using the API let label1 = create_string_label("thread_id", "main"); let label2 = create_numeric_label("memory_usage", 1024, "bytes"); - - // Add a sample with these labels let sample = crate::api::Sample { locations: vec![], values: &[42], labels: vec![label1, label2], }; - let _ = internal_profile.try_add_sample(sample, None); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); let dictionary = otel_profiles_data.dictionary.unwrap(); - - // Should have 2 labels converted to attributes assert_eq!(dictionary.attribute_table.len(), 2); - - // Should have 1 attribute unit (for the numeric label with unit) assert_eq!(dictionary.attribute_units.len(), 1); - // Verify the first attribute (string label) - let attr1 = &dictionary.attribute_table[0]; - assert_eq!(attr1.key, "thread_id"); - match &attr1.value { - Some(datadog_profiling_otel::key_value::Value::StringValue(s)) => { - assert_eq!(s, "main"); - } - _ => panic!("Expected StringValue"), - } - - // Verify the second attribute (numeric label) - let attr2 = &dictionary.attribute_table[1]; - assert_eq!(attr2.key, "memory_usage"); - match &attr2.value { - Some(datadog_profiling_otel::key_value::Value::IntValue(n)) => { - assert_eq!(*n, 1024); - } - _ => panic!("Expected IntValue"), - } - - // Verify the attribute unit mapping - let unit = &dictionary.attribute_units[0]; - // The key should map to the memory_usage string index - // and the unit should map to the "bytes" string index - assert!(unit.attribute_key_strindex > 0); - assert!(unit.unit_strindex > 0); - - // Check duration calculation - only if profiles exist - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - if !scope_profile.profiles.is_empty() { - assert_duration_calculation(&scope_profile.profiles); - } - } - - #[test] - fn test_from_internal_profile_with_sample_types() { - // Create an internal profile with specific sample types + // Test with sample types let sample_types = [ crate::api::ValueType::new("cpu", "nanoseconds"), crate::api::ValueType::new("allocations", "count"), ]; let internal_profile = InternalProfile::new(&sample_types, None); - - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify that individual profiles are created for each sample type - assert_profiles_data_structure(&otel_profiles_data); let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - - // Should have 2 profiles (one for each sample type) assert_eq!(scope_profile.profiles.len(), 2); - - // Verify the first profile (cpu profile) - let cpu_profile = &scope_profile.profiles[0]; - assert!(cpu_profile.sample_type.is_some()); - let cpu_sample_type = cpu_profile.sample_type.as_ref().unwrap(); - assert_eq!(cpu_sample_type.type_strindex, 4); // "cpu" string index - assert_eq!(cpu_sample_type.unit_strindex, 5); // "nanoseconds" string index - - // Verify the second profile (allocations profile) - let allocations_profile = &scope_profile.profiles[1]; - assert!(allocations_profile.sample_type.is_some()); - let allocations_sample_type = allocations_profile.sample_type.as_ref().unwrap(); - assert_eq!(allocations_sample_type.type_strindex, 6); // "allocations" string index - assert_eq!(allocations_sample_type.unit_strindex, 7); // "count" string index - - // Check duration calculation for both profiles assert_duration_calculation(&scope_profile.profiles); } #[test] - fn test_sample_conversion_basic() { - // Create an internal profile with sample types + fn test_sample_conversion() { let sample_types = [ crate::api::ValueType::new("cpu", "nanoseconds"), crate::api::ValueType::new("memory", "bytes"), @@ -501,43 +381,23 @@ mod tests { let (mut internal_profile, location) = setup_profile_with_function_and_location(&sample_types); - // Add a sample with values + // Test basic sample conversion let sample = crate::api::Sample { locations: vec![location], - values: &[100, 2048], // 100 nanoseconds, 2048 bytes + values: &[100, 2048], labels: vec![], }; let _ = internal_profile.try_add_sample(sample, None); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); - let _dictionary = otel_profiles_data.dictionary.unwrap(); - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - // Should have 2 profiles (one for each sample type) assert_eq!(scope_profile.profiles.len(), 2); - - // Verify the first profile (cpu profile) has the correct sample assert_profile_has_correct_sample(&scope_profile.profiles[0], vec![100], 0, 0); - - // Verify the second profile (memory profile) has the correct sample assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![2048], 0, 0); - // Check duration calculation for both profiles - assert_duration_calculation(&scope_profile.profiles); - } - - #[test] - fn test_sample_conversion_with_labels() { - // Create an internal profile with sample types - let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + // Test with labels let (mut internal_profile, location) = - setup_profile_with_function_and_location(&sample_types); - - // Add a sample with labels + setup_profile_with_function_and_location(&[sample_types[0]]); let sample = crate::api::Sample { locations: vec![location], values: &[150], @@ -548,39 +408,56 @@ mod tests { }; let _ = internal_profile.try_add_sample(sample, None); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); - let _dictionary = otel_profiles_data.dictionary.unwrap(); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - - // Should have 1 sample with correct values and attributes assert_profile_has_correct_sample(profile, vec![150], 0, 2); - // Verify the sample has the correct attribute indices + // Verify the sample's attribute indices point to correct attributes let sample = &profile.sample[0]; - // The attribute indices should correspond to the labels in the attribute table - assert!(sample.attribute_indices[0] >= 0); - assert!(sample.attribute_indices[1] >= 0); + let dictionary = &otel_profiles_data.dictionary.as_ref().unwrap(); - // Check duration calculation - assert_duration_calculation(&[profile.clone()]); + // Check that attribute indices are valid + for &attr_idx in &sample.attribute_indices { + assert!(attr_idx >= 0); + assert!(attr_idx < dictionary.attribute_table.len() as i32); + } - // Verify the attributes were converted correctly - assert_eq!(_dictionary.attribute_table.len(), 2); - assert_eq!(_dictionary.attribute_units.len(), 1); // One numeric label with unit - } + // Verify the actual attribute content + let attr1 = &dictionary.attribute_table[sample.attribute_indices[0] as usize]; + let attr2 = &dictionary.attribute_table[sample.attribute_indices[1] as usize]; - #[test] - fn test_sample_conversion_with_timestamps() { - // Create an internal profile with sample types - let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&sample_types); + // One should be the string label, one should be the numeric label + let (string_attr, numeric_attr) = if attr1.key == "thread_id" { + (attr1, attr2) + } else { + (attr2, attr1) + }; + + // Verify string attribute + assert_eq!(string_attr.key, "thread_id"); + let s = match string_attr.value.as_ref().expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::StringValue(s) => s, + _ => panic!("Expected StringValue"), + }; + assert_eq!(s, "main"); + + // Verify numeric attribute + assert_eq!(numeric_attr.key, "cpu_usage"); + let n = match numeric_attr.value.as_ref().expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, + _ => panic!("Expected IntValue"), + }; + assert_eq!(*n, 75); + + // Verify attribute unit mapping + assert_eq!(dictionary.attribute_units.len(), 1); + let unit = &dictionary.attribute_units[0]; + assert!(unit.attribute_key_strindex > 0); + assert!(unit.unit_strindex > 0); - // Add a sample with timestamp + // Test with timestamps + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); let sample = crate::api::Sample { locations: vec![location], values: &[200], @@ -589,35 +466,14 @@ mod tests { let timestamp = crate::internal::Timestamp::new(1234567890).unwrap(); let _ = internal_profile.try_add_sample(sample, Some(timestamp)); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - - // Should have 1 sample assert_eq!(profile.sample.len(), 1); - let sample = &profile.sample[0]; - - // Verify the sample has the correct timestamp - assert_sample_has_timestamp(sample, 1234567890); + assert_sample_has_timestamp(&profile.sample[0], 1234567890); - // Check duration calculation - assert_duration_calculation(&[profile.clone()]); - } - - #[test] - fn test_sample_conversion_zero_values_filtered() { - // Create an internal profile with sample types - let sample_types = [ - crate::api::ValueType::new("cpu", "nanoseconds"), - crate::api::ValueType::new("memory", "bytes"), - ]; + // Test zero value filtering let (mut internal_profile, location) = setup_profile_with_function_and_location(&sample_types); - - // Add a sample with one zero value and one non-zero value let sample = crate::api::Sample { locations: vec![location], values: &[0, 1024], // 0 nanoseconds, 1024 bytes @@ -625,33 +481,14 @@ mod tests { }; let _ = internal_profile.try_add_sample(sample, None); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - let profile0 = &scope_profile.profiles[0]; - let profile1 = &scope_profile.profiles[1]; - - // First profile (cpu) should have no samples since value is 0 - assert_eq!(profile0.sample.len(), 0); - - // Second profile (memory) should have 1 sample since value is non-zero - assert_profile_has_correct_sample(profile1, vec![1024], 0, 0); - - // Check duration calculation for both profiles - assert_duration_calculation(&scope_profile.profiles); - } + assert_eq!(scope_profile.profiles[0].sample.len(), 0); // Zero value filtered + assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![1024], 0, 0); - #[test] - fn test_sample_conversion_multiple_samples() { - // Create an internal profile with sample types - let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + // Test multiple samples aggregation let (mut internal_profile, location) = - setup_profile_with_function_and_location(&sample_types); - - // Add multiple samples + setup_profile_with_function_and_location(&[sample_types[0]]); let sample1 = crate::api::Sample { locations: vec![location], values: &[100], @@ -667,48 +504,29 @@ mod tests { values: &[300], labels: vec![], }; - let _ = internal_profile.try_add_sample(sample1, None); let _ = internal_profile.try_add_sample(sample2, None); let _ = internal_profile.try_add_sample(sample3, None); - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - // Verify the conversion - assert_profiles_data_structure(&otel_profiles_data); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - - // Should have 1 aggregated sample (samples with same stack trace and labels get aggregated) assert_eq!(profile.sample.len(), 1); + assert_eq!(profile.sample[0].values, vec![600]); // 100 + 200 + 300 - // Verify the aggregated sample has the summed value - let sample = &profile.sample[0]; - assert_eq!(sample.values, vec![600]); // 100 + 200 + 300 - - // Verify all samples have the same stack index - for sample in &profile.sample { - assert_eq!(sample.stack_index, 0); - } - - // Check duration calculation - assert_duration_calculation(&[profile.clone()]); + assert_duration_calculation(&scope_profile.profiles); } #[test] - fn test_duration_calculation() { - // Create an internal profile with sample types + fn test_duration_and_period() { let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let internal_profile = InternalProfile::new(&sample_types, None); - // Test with explicit duration + // Test duration calculation + let internal_profile = InternalProfile::new(&sample_types, None); let explicit_duration = std::time::Duration::from_secs(5); let otel_profiles_data = internal_profile .convert_into_otel(None, Some(explicit_duration)) .unwrap(); - let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - // Should use the explicit duration (5 seconds = 5_000_000_000 nanoseconds) assert_eq!(profile.duration_nanos, 5_000_000_000); // Test with explicit end_time @@ -718,10 +536,7 @@ mod tests { let otel_profiles_data2 = internal_profile2 .convert_into_otel(Some(end_time), None) .unwrap(); - let profile2 = &otel_profiles_data2.resource_profiles[0].scope_profiles[0].profiles[0]; - // Should calculate duration from end_time - start_time (3 seconds = 3_000_000_000 - // nanoseconds) assert_eq!(profile2.duration_nanos, 3_000_000_000); // Test with both end_time and duration (duration should take precedence) @@ -732,39 +547,18 @@ mod tests { let otel_profiles_data3 = internal_profile3 .convert_into_otel(Some(end_time3), Some(duration3)) .unwrap(); - let profile3 = &otel_profiles_data3.resource_profiles[0].scope_profiles[0].profiles[0]; - // Should use the explicit duration (7 seconds = 7_000_000_000 nanoseconds) assert_eq!(profile3.duration_nanos, 7_000_000_000); - } - #[test] - fn test_period_conversion() { - // Create an internal profile with sample types and period - let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + // Test period conversion let period = crate::api::Period { r#type: crate::api::ValueType::new("cpu", "cycles"), value: 1000, }; let internal_profile = InternalProfile::new(&sample_types, Some(period)); - - // Convert to OpenTelemetry ProfilesData let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - - assert_profiles_data_structure(&otel_profiles_data); let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - - // Should have period type information assert!(profile.period_type.is_some()); - let period_type = profile.period_type.as_ref().unwrap(); - - // The period type should be converted from the internal profile's period - // Note: The exact string indices depend on the string table, but we can verify they're - // valid - assert!(period_type.type_strindex >= 0); - assert!(period_type.unit_strindex >= 0); - - // Should have the correct period value assert_eq!(profile.period, 1000); // Test without period @@ -772,14 +566,9 @@ mod tests { let otel_profiles_data_no_period = internal_profile_no_period .convert_into_otel(None, None) .unwrap(); - - assert_profiles_data_structure(&otel_profiles_data_no_period); let profile_no_period = &otel_profiles_data_no_period.resource_profiles[0].scope_profiles[0].profiles[0]; - - // Should have no period type when no period is set assert!(profile_no_period.period_type.is_none()); - // Should have period value of 0 when no period is set assert_eq!(profile_no_period.period, 0); } From d3e8ff2daf3eafa1fde6ed5ea5e9c7747f15d81f Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 11 Sep 2025 16:02:05 -0400 Subject: [PATCH 18/23] assert default expected strings --- .../src/internal/profile/otel_emitter/profile.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index b18b038a6e..f0a29a9214 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -320,6 +320,10 @@ mod tests { assert_eq!(dictionary.function_table.len(), 0); assert_eq!(dictionary.stack_table.len(), 0); assert_eq!(dictionary.string_table.len(), 4); + assert_eq!(dictionary.string_table[0], ""); // Empty string + assert_eq!(dictionary.string_table[1], "local root span id"); + assert_eq!(dictionary.string_table[2], "trace endpoint"); + assert_eq!(dictionary.string_table[3], "end_timestamp_ns"); assert_eq!(dictionary.link_table.len(), 0); assert_eq!(dictionary.attribute_table.len(), 0); assert_eq!(dictionary.attribute_units.len(), 0); From 8cc1624d088eb91a8001da37112b75a0671f0c9e Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 11 Sep 2025 16:13:18 -0400 Subject: [PATCH 19/23] PR comments --- datadog-profiling/src/internal/profile/otel_emitter/label.rs | 3 +-- datadog-profiling/src/internal/profile/otel_emitter/profile.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/label.rs b/datadog-profiling/src/internal/profile/otel_emitter/label.rs index e7c46ab9a6..e60f63e0a0 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/label.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/label.rs @@ -61,12 +61,11 @@ pub fn convert_label_to_key_value( crate::internal::LabelValue::Num { num, num_unit } => { // Note: OpenTelemetry KeyValue doesn't support units, so we only store the numeric // value But we track the mapping for building the attribute_units table - let key_index = label.get_key().to_raw_id() as usize; let unit_index = num_unit.to_raw_id() as usize; // Only add to the map if the unit is not the default empty string (index 0) if unit_index > 0 { - key_to_unit_map.insert(key_index, unit_index); + key_to_unit_map.insert(key_id, unit_index); } Ok(datadog_profiling_otel::KeyValue { diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index f0a29a9214..c24c18b815 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -164,7 +164,7 @@ impl InternalProfile { let resource_profiles = vec![datadog_profiling_otel::ResourceProfiles { resource: None, // TODO: Implement when we handle resources scope_profiles: vec![datadog_profiling_otel::ScopeProfiles { - scope: None, // TODO: Implement when we handle scopes + scope: None, // It is legal to leave this unset according to the spec profiles, // Now contains the individual profiles schema_url: String::new(), // TODO: Implement when we handle schema URLs default_sample_type: None, // TODO: Implement when we handle sample types From ec50f57eec558994b81be4a7b438b93eab662c2b Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Thu, 11 Sep 2025 16:14:11 -0400 Subject: [PATCH 20/23] don't reformat outside file --- datadog-trace-protobuf/src/remoteconfig.rs | 30 ++++++++-------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/datadog-trace-protobuf/src/remoteconfig.rs b/datadog-trace-protobuf/src/remoteconfig.rs index fdc89ee871..a8a4b66524 100644 --- a/datadog-trace-protobuf/src/remoteconfig.rs +++ b/datadog-trace-protobuf/src/remoteconfig.rs @@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize}; // This file is @generated by prost-build. -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct File { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -12,8 +11,7 @@ pub struct File { #[serde(with = "serde_bytes")] pub raw: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct Client { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, @@ -35,8 +33,7 @@ pub struct Client { #[prost(bytes = "vec", tag = "11")] pub capabilities: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientTracer { #[prost(string, tag = "1")] pub runtime_id: ::prost::alloc::string::String, @@ -55,8 +52,7 @@ pub struct ClientTracer { #[prost(string, repeated, tag = "7")] pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientAgent { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -69,8 +65,7 @@ pub struct ClientAgent { #[prost(string, repeated, tag = "5")] pub cws_workloads: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ConfigState { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -83,8 +78,7 @@ pub struct ConfigState { #[prost(string, tag = "5")] pub apply_error: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientState { #[prost(uint64, tag = "1")] pub root_version: u64, @@ -99,16 +93,14 @@ pub struct ClientState { #[prost(bytes = "vec", tag = "6")] pub backend_client_state: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct TargetFileHash { #[prost(string, tag = "1")] pub algorithm: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub hash: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct TargetFileMeta { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -117,16 +109,14 @@ pub struct TargetFileMeta { #[prost(message, repeated, tag = "3")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsRequest { #[prost(message, optional, tag = "1")] pub client: ::core::option::Option, #[prost(message, repeated, tag = "2")] pub cached_target_files: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsResponse { #[prost(bytes = "vec", repeated, tag = "1")] #[serde(with = "crate::serde")] From 1a3440f16aa4446782a05f7fa3d841409160f1f5 Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 15 Sep 2025 17:26:54 -0400 Subject: [PATCH 21/23] endpoint labels --- Cargo.lock | 2 +- .../internal/profile/otel_emitter/profile.rs | 32 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bb4c6ca0f..6524f40faa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1697,7 +1697,7 @@ dependencies = [ [[package]] name = "datadog-profiling-otel" -version = "20.0.0" +version = "21.0.0" dependencies = [ "anyhow", "bolero", diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index c24c18b815..9467e9fc19 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -1,7 +1,7 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::collections::identifiable::Id; +use crate::collections::identifiable::{Dedup, Id}; use crate::internal::profile::otel_emitter::label::convert_label_to_key_value; use crate::internal::profile::{EncodedProfile, Profile as InternalProfile}; use crate::iter::{IntoLendingIterator, LendingIterator}; @@ -81,11 +81,37 @@ impl InternalProfile { profiles.push(profile); } + // If we have span labels, figure out the corresponding endpoint labels + // Do this into a temporary map to avoid mutating the map we're iterating over + let mut endpoint_labels = HashMap::new(); + for (idx, label) in self.labels.iter().enumerate() { + if label.get_key() == self.endpoints.local_root_span_id_label { + if let Some(endpoint_label) = self.get_endpoint_for_label(label)? { + endpoint_labels.insert(idx, endpoint_label); + } + } + } + + // Put the values from the temporary map back into the original labels map + let mut endpoint_labels_idx = HashMap::new(); + for (idx, label) in endpoint_labels { + let endpoint_idx = self.labels.dedup(label); + endpoint_labels_idx.insert(idx, endpoint_idx); + } + for (sample, timestamp, mut values) in std::mem::take(&mut self.observations).into_iter() { let stack_index = sample.stacktrace.to_raw_id() as i32; let label_set = self.get_label_set(sample.labels)?; - let attribute_indicies: Vec<_> = - label_set.iter().map(|x| x.to_raw_id() as i32).collect(); + let attribute_indicies: Vec<_> = label_set + .iter() + .map(|x| x.to_raw_id() as i32) + .chain( + label_set + .iter() + .find_map(|k| endpoint_labels_idx.get(&(k.to_raw_id() as usize))) + .map(|label| label.to_raw_id() as i32), + ) + .collect(); let labels = label_set .iter() .map(|l| self.get_label(*l).copied()) From 7da33a2fd21bf7d2c62f7cf1688047759d66720a Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Mon, 15 Sep 2025 19:32:29 -0400 Subject: [PATCH 22/23] cleaner code for the labelset --- .../internal/profile/otel_emitter/profile.rs | 35 +++++++++++-------- datadog-trace-protobuf/src/remoteconfig.rs | 30 ++++++++++------ 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs index 9467e9fc19..1fd7370ee5 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs @@ -82,22 +82,27 @@ impl InternalProfile { } // If we have span labels, figure out the corresponding endpoint labels - // Do this into a temporary map to avoid mutating the map we're iterating over - let mut endpoint_labels = HashMap::new(); - for (idx, label) in self.labels.iter().enumerate() { - if label.get_key() == self.endpoints.local_root_span_id_label { - if let Some(endpoint_label) = self.get_endpoint_for_label(label)? { - endpoint_labels.insert(idx, endpoint_label); - } - } - } + // Split into two steps to avoid mutating the map we're iterating over + let endpoint_labels: Vec<_> = self + .labels + .iter() + .enumerate() + .filter(|(_, label)| label.get_key() == self.endpoints.local_root_span_id_label) + .filter_map(|(idx, label)| { + self.get_endpoint_for_label(label) + .ok() + .flatten() + .map(|endpoint_label| (idx, endpoint_label)) + }) + .collect(); - // Put the values from the temporary map back into the original labels map - let mut endpoint_labels_idx = HashMap::new(); - for (idx, label) in endpoint_labels { - let endpoint_idx = self.labels.dedup(label); - endpoint_labels_idx.insert(idx, endpoint_idx); - } + let endpoint_labels_idx: HashMap = endpoint_labels + .into_iter() + .map(|(idx, endpoint_label)| { + let endpoint_idx = self.labels.dedup(endpoint_label); + (idx, endpoint_idx) + }) + .collect(); for (sample, timestamp, mut values) in std::mem::take(&mut self.observations).into_iter() { let stack_index = sample.stacktrace.to_raw_id() as i32; diff --git a/datadog-trace-protobuf/src/remoteconfig.rs b/datadog-trace-protobuf/src/remoteconfig.rs index a8a4b66524..fdc89ee871 100644 --- a/datadog-trace-protobuf/src/remoteconfig.rs +++ b/datadog-trace-protobuf/src/remoteconfig.rs @@ -3,7 +3,8 @@ use serde::{Deserialize, Serialize}; // This file is @generated by prost-build. -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct File { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -11,7 +12,8 @@ pub struct File { #[serde(with = "serde_bytes")] pub raw: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Client { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, @@ -33,7 +35,8 @@ pub struct Client { #[prost(bytes = "vec", tag = "11")] pub capabilities: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientTracer { #[prost(string, tag = "1")] pub runtime_id: ::prost::alloc::string::String, @@ -52,7 +55,8 @@ pub struct ClientTracer { #[prost(string, repeated, tag = "7")] pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientAgent { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -65,7 +69,8 @@ pub struct ClientAgent { #[prost(string, repeated, tag = "5")] pub cws_workloads: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ConfigState { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -78,7 +83,8 @@ pub struct ConfigState { #[prost(string, tag = "5")] pub apply_error: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientState { #[prost(uint64, tag = "1")] pub root_version: u64, @@ -93,14 +99,16 @@ pub struct ClientState { #[prost(bytes = "vec", tag = "6")] pub backend_client_state: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TargetFileHash { #[prost(string, tag = "1")] pub algorithm: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub hash: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TargetFileMeta { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -109,14 +117,16 @@ pub struct TargetFileMeta { #[prost(message, repeated, tag = "3")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsRequest { #[prost(message, optional, tag = "1")] pub client: ::core::option::Option, #[prost(message, repeated, tag = "2")] pub cached_target_files: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsResponse { #[prost(bytes = "vec", repeated, tag = "1")] #[serde(with = "crate::serde")] From d77157cb318771cf9bf0993267531d9d10d83a5c Mon Sep 17 00:00:00 2001 From: Daniel Schwartz-Narbonne Date: Tue, 16 Sep 2025 10:16:39 -0400 Subject: [PATCH 23/23] split out tests --- .../src/internal/profile/otel_emitter/mod.rs | 5 + .../internal/profile/otel_emitter/profile.rs | 643 ------------------ .../profile/otel_emitter/profile/mod.rs | 237 +++++++ .../profile/otel_emitter/profile/tests.rs | 406 +++++++++++ datadog-trace-protobuf/src/remoteconfig.rs | 30 +- 5 files changed, 658 insertions(+), 663 deletions(-) delete mode 100644 datadog-profiling/src/internal/profile/otel_emitter/profile.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs create mode 100644 datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs diff --git a/datadog-profiling/src/internal/profile/otel_emitter/mod.rs b/datadog-profiling/src/internal/profile/otel_emitter/mod.rs index 8a91ef08a3..4929dd31eb 100644 --- a/datadog-profiling/src/internal/profile/otel_emitter/mod.rs +++ b/datadog-profiling/src/internal/profile/otel_emitter/mod.rs @@ -13,3 +13,8 @@ pub mod location; pub mod mapping; pub mod profile; pub mod stack_trace; + +#[cfg(test)] +mod tests { + include!("profile/tests.rs"); +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile.rs deleted file mode 100644 index 1fd7370ee5..0000000000 --- a/datadog-profiling/src/internal/profile/otel_emitter/profile.rs +++ /dev/null @@ -1,643 +0,0 @@ -// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -use crate::collections::identifiable::{Dedup, Id}; -use crate::internal::profile::otel_emitter::label::convert_label_to_key_value; -use crate::internal::profile::{EncodedProfile, Profile as InternalProfile}; -use crate::iter::{IntoLendingIterator, LendingIterator}; -use anyhow::{Context, Result}; -use datadog_profiling_otel::ProfilesDataExt; -use std::collections::HashMap; - -impl InternalProfile { - /// Converts the profile into OpenTelemetry format - /// - /// * `end_time` - Optional end time of the profile. Passing None will use the current time. - /// * `duration` - Optional duration of the profile. Passing None will try to calculate the - /// duration based on the end time minus the start time, but under anomalous conditions this - /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an - /// earlier time. The duration will be set to zero these cases. - pub fn convert_into_otel( - mut self, - end_time: Option, - duration: Option, - ) -> anyhow::Result { - // Calculate duration using the same logic as encode - let end = end_time.unwrap_or_else(std::time::SystemTime::now); - let start = self.start_time; - let duration_nanos = duration - .unwrap_or_else(|| { - end.duration_since(start).unwrap_or({ - // Let's not throw away the whole profile just because the clocks were wrong. - // todo: log that the clock went backward (or programmer mistake). - std::time::Duration::ZERO - }) - }) - .as_nanos() - .min(i64::MAX as u128) as i64; - - // Create individual OpenTelemetry Profiles for each ValueType - let mut profiles = Vec::with_capacity(self.sample_types.len()); - - for sample_type in self.sample_types.iter() { - // Convert the ValueType to OpenTelemetry format - let otel_sample_type = datadog_profiling_otel::ValueType { - type_strindex: sample_type.r#type.value.to_raw_id() as i32, - unit_strindex: sample_type.unit.value.to_raw_id() as i32, - aggregation_temporality: datadog_profiling_otel::AggregationTemporality::Delta - .into(), - }; - - // Create a Profile for this sample type - let profile = datadog_profiling_otel::Profile { - sample_type: Some(otel_sample_type), - sample: vec![], - time_nanos: self - .start_time - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() as i64, - duration_nanos, // Use calculated duration - period_type: self.period.as_ref().map(|(_, period_type)| { - datadog_profiling_otel::ValueType { - type_strindex: period_type.r#type.value.to_raw_id() as i32, - unit_strindex: period_type.unit.value.to_raw_id() as i32, - aggregation_temporality: - datadog_profiling_otel::AggregationTemporality::Delta.into(), - } - }), - period: self - .period - .map(|(period_value, _)| period_value) - .unwrap_or(0), - comment_strindices: vec![], // We don't have comments - profile_id: vec![], // TODO: Implement when we handle profile IDs - dropped_attributes_count: 0, // We don't drop attributes - original_payload_format: String::new(), // There is no original payload - original_payload: vec![], // There is no original payload - attribute_indices: vec![], // There are currently no attributes at this level - }; - - profiles.push(profile); - } - - // If we have span labels, figure out the corresponding endpoint labels - // Split into two steps to avoid mutating the map we're iterating over - let endpoint_labels: Vec<_> = self - .labels - .iter() - .enumerate() - .filter(|(_, label)| label.get_key() == self.endpoints.local_root_span_id_label) - .filter_map(|(idx, label)| { - self.get_endpoint_for_label(label) - .ok() - .flatten() - .map(|endpoint_label| (idx, endpoint_label)) - }) - .collect(); - - let endpoint_labels_idx: HashMap = endpoint_labels - .into_iter() - .map(|(idx, endpoint_label)| { - let endpoint_idx = self.labels.dedup(endpoint_label); - (idx, endpoint_idx) - }) - .collect(); - - for (sample, timestamp, mut values) in std::mem::take(&mut self.observations).into_iter() { - let stack_index = sample.stacktrace.to_raw_id() as i32; - let label_set = self.get_label_set(sample.labels)?; - let attribute_indicies: Vec<_> = label_set - .iter() - .map(|x| x.to_raw_id() as i32) - .chain( - label_set - .iter() - .find_map(|k| endpoint_labels_idx.get(&(k.to_raw_id() as usize))) - .map(|label| label.to_raw_id() as i32), - ) - .collect(); - let labels = label_set - .iter() - .map(|l| self.get_label(*l).copied()) - .collect::>>()?; - let link_index = 0; // TODO, handle links properly - let timestamps_unix_nano = timestamp.map_or(vec![], |ts| vec![ts.get() as u64]); - self.upscaling_rules.upscale_values(&mut values, &labels)?; - - for (idx, value) in values.iter().enumerate() { - if *value != 0 { - let otel_sample = datadog_profiling_otel::Sample { - stack_index, - attribute_indices: attribute_indicies.clone(), - link_index, - values: vec![*value], - timestamps_unix_nano: timestamps_unix_nano.clone(), - }; - profiles[idx].sample.push(otel_sample); - } - } - } - - // Convert string table using into_lending_iter - // Note: We can't use .map().collect() here because LendingIterator doesn't implement - // the standard Iterator trait. LendingIterator is designed for yielding references - // with lifetimes tied to the iterator itself, so we need to manually iterate and - // convert each string reference to an owned String. - let string_table = { - let mut strings = Vec::with_capacity(self.strings.len()); - let mut iter = self.strings.into_lending_iter(); - while let Some(s) = iter.next() { - strings.push(s.to_string()); - } - strings - }; - - // Convert labels to KeyValues for the attribute table - let mut key_to_unit_map = HashMap::new(); - let mut attribute_table = Vec::with_capacity(self.labels.len()); - - for label in self.labels.iter() { - let key_value = convert_label_to_key_value(label, &string_table, &mut key_to_unit_map) - .with_context(|| { - format!( - "Failed to convert label with key index {}", - label.get_key().to_raw_id() - ) - })?; - attribute_table.push(key_value); - } - - // Build attribute units from the key-to-unit mapping - let attribute_units = key_to_unit_map - .into_iter() - .map( - |(key_index, unit_index)| datadog_profiling_otel::AttributeUnit { - attribute_key_strindex: key_index as i32, - unit_strindex: unit_index as i32, - }, - ) - .collect(); - - // Convert the ProfilesDictionary components - let dictionary = datadog_profiling_otel::ProfilesDictionary { - mapping_table: self.mappings.into_iter().map(From::from).collect(), - location_table: self.locations.into_iter().map(From::from).collect(), - function_table: self.functions.into_iter().map(From::from).collect(), - stack_table: self.stack_traces.into_iter().map(From::from).collect(), - string_table, - attribute_table, - attribute_units, - link_table: vec![], // TODO: Implement when we handle trace links - }; - - // Create a basic ResourceProfiles structure - let resource_profiles = vec![datadog_profiling_otel::ResourceProfiles { - resource: None, // TODO: Implement when we handle resources - scope_profiles: vec![datadog_profiling_otel::ScopeProfiles { - scope: None, // It is legal to leave this unset according to the spec - profiles, // Now contains the individual profiles - schema_url: String::new(), // TODO: Implement when we handle schema URLs - default_sample_type: None, // TODO: Implement when we handle sample types - }], - schema_url: String::new(), // TODO: Implement when we handle schema URLs - }]; - - Ok(datadog_profiling_otel::ProfilesData { - resource_profiles, - dictionary: Some(dictionary), - }) - } - - /// Serializes the profile into OpenTelemetry format and compresses it using zstd. - /// - /// * `end_time` - Optional end time of the profile. Passing None will use the current time. - /// * `duration` - Optional duration of the profile. Passing None will try to calculate the - /// duration based on the end time minus the start time, but under anomalous conditions this - /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an - /// earlier time. The duration will be set to zero these cases. - pub fn serialize_into_compressed_otel( - mut self, - end_time: Option, - duration: Option, - ) -> anyhow::Result { - // Extract values before consuming self - let start = self.start_time; - let endpoints_stats = std::mem::take(&mut self.endpoints.stats); - let otel_profiles_data = self.convert_into_otel(end_time, duration)?; - let buffer = otel_profiles_data.serialize_into_compressed_proto()?; - let end = end_time.unwrap_or_else(std::time::SystemTime::now); - Ok(EncodedProfile { - start, - end, - buffer, - endpoints_stats, - }) - } -} - -#[cfg(test)] -mod tests { - use crate::internal::profile::Profile as InternalProfile; - - // Helper functions for test setup - fn create_basic_function() -> crate::api::Function<'static> { - crate::api::Function { - name: "test_function", - system_name: "test_system", - filename: "test_file.rs", - } - } - - fn create_basic_mapping() -> crate::api::Mapping<'static> { - crate::api::Mapping { - memory_start: 0x1000, - memory_limit: 0x2000, - file_offset: 0, - filename: "test_binary", - build_id: "test_build_id", - } - } - - fn setup_profile_with_function_and_location( - sample_types: &[crate::api::ValueType<'static>], - ) -> (InternalProfile, crate::api::Location<'static>) { - let mut internal_profile = InternalProfile::new(sample_types, None); - let function = create_basic_function(); - let mapping = create_basic_mapping(); - let location = crate::api::Location { - mapping, - function, - address: 0x1000, - line: 42, - }; - - let _function_id = internal_profile.try_add_function(&function); - let _mapping_id = internal_profile.try_add_mapping(&mapping); - let location_id = internal_profile.try_add_location(&location).unwrap(); - let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); - - (internal_profile, location) - } - - fn create_string_label(key: &'static str, value: &'static str) -> crate::api::Label<'static> { - crate::api::Label { - key, - str: value, - num: 0, - num_unit: "", - } - } - - fn create_numeric_label( - key: &'static str, - value: i64, - unit: &'static str, - ) -> crate::api::Label<'static> { - crate::api::Label { - key, - str: "", - num: value, - num_unit: unit, - } - } - - // Common assertion helpers - fn assert_duration_calculation(profiles: &[datadog_profiling_otel::Profile]) { - for profile in profiles { - assert!(profile.duration_nanos > 0); - } - } - - fn assert_profile_has_correct_sample( - profile: &datadog_profiling_otel::Profile, - expected_values: Vec, - expected_stack_index: i32, - expected_attribute_count: usize, - ) { - assert_eq!(profile.sample.len(), 1); - let sample = &profile.sample[0]; - assert_eq!(sample.values, expected_values); - assert_eq!(sample.stack_index, expected_stack_index); - assert_eq!(sample.attribute_indices.len(), expected_attribute_count); - } - - fn assert_sample_has_timestamp( - sample: &datadog_profiling_otel::Sample, - expected_timestamp: u64, - ) { - assert_eq!(sample.timestamps_unix_nano.len(), 1); - assert_eq!(sample.timestamps_unix_nano[0], expected_timestamp); - } - - fn assert_profiles_data_structure(otel_profiles_data: &datadog_profiling_otel::ProfilesData) { - assert!(otel_profiles_data.dictionary.is_some()); - assert_eq!(otel_profiles_data.resource_profiles.len(), 1); - assert_eq!( - otel_profiles_data.resource_profiles[0].scope_profiles.len(), - 1 - ); - } - - #[test] - fn test_convert_into_otel() { - // Test empty profile - let internal_profile = InternalProfile::new(&[], None); - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - assert_profiles_data_structure(&otel_profiles_data); - let dictionary = otel_profiles_data.dictionary.unwrap(); - assert_eq!(dictionary.mapping_table.len(), 0); - assert_eq!(dictionary.location_table.len(), 0); - assert_eq!(dictionary.function_table.len(), 0); - assert_eq!(dictionary.stack_table.len(), 0); - assert_eq!(dictionary.string_table.len(), 4); - assert_eq!(dictionary.string_table[0], ""); // Empty string - assert_eq!(dictionary.string_table[1], "local root span id"); - assert_eq!(dictionary.string_table[2], "trace endpoint"); - assert_eq!(dictionary.string_table[3], "end_timestamp_ns"); - assert_eq!(dictionary.link_table.len(), 0); - assert_eq!(dictionary.attribute_table.len(), 0); - assert_eq!(dictionary.attribute_units.len(), 0); - - // Test with functions - let mut internal_profile = InternalProfile::new(&[], None); - let function1 = crate::api::Function { - name: "test_function_1", - system_name: "test_system_1", - filename: "test_file_1.rs", - }; - let function2 = crate::api::Function { - name: "test_function_2", - system_name: "test_system_2", - filename: "test_file_2.rs", - }; - let _function1_id = internal_profile.try_add_function(&function1); - let _function2_id = internal_profile.try_add_function(&function2); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let dictionary = otel_profiles_data.dictionary.unwrap(); - assert_eq!(dictionary.function_table.len(), 2); - assert_eq!(dictionary.string_table.len(), 10); - - // Test with labels - let mut internal_profile = InternalProfile::new(&[], None); - let label1 = create_string_label("thread_id", "main"); - let label2 = create_numeric_label("memory_usage", 1024, "bytes"); - let sample = crate::api::Sample { - locations: vec![], - values: &[42], - labels: vec![label1, label2], - }; - let _ = internal_profile.try_add_sample(sample, None); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let dictionary = otel_profiles_data.dictionary.unwrap(); - assert_eq!(dictionary.attribute_table.len(), 2); - assert_eq!(dictionary.attribute_units.len(), 1); - - // Test with sample types - let sample_types = [ - crate::api::ValueType::new("cpu", "nanoseconds"), - crate::api::ValueType::new("allocations", "count"), - ]; - let internal_profile = InternalProfile::new(&sample_types, None); - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - assert_eq!(scope_profile.profiles.len(), 2); - assert_duration_calculation(&scope_profile.profiles); - } - - #[test] - fn test_sample_conversion() { - let sample_types = [ - crate::api::ValueType::new("cpu", "nanoseconds"), - crate::api::ValueType::new("memory", "bytes"), - ]; - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&sample_types); - - // Test basic sample conversion - let sample = crate::api::Sample { - locations: vec![location], - values: &[100, 2048], - labels: vec![], - }; - let _ = internal_profile.try_add_sample(sample, None); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - assert_eq!(scope_profile.profiles.len(), 2); - assert_profile_has_correct_sample(&scope_profile.profiles[0], vec![100], 0, 0); - assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![2048], 0, 0); - - // Test with labels - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&[sample_types[0]]); - let sample = crate::api::Sample { - locations: vec![location], - values: &[150], - labels: vec![ - create_string_label("thread_id", "main"), - create_numeric_label("cpu_usage", 75, "percent"), - ], - }; - let _ = internal_profile.try_add_sample(sample, None); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_profile_has_correct_sample(profile, vec![150], 0, 2); - - // Verify the sample's attribute indices point to correct attributes - let sample = &profile.sample[0]; - let dictionary = &otel_profiles_data.dictionary.as_ref().unwrap(); - - // Check that attribute indices are valid - for &attr_idx in &sample.attribute_indices { - assert!(attr_idx >= 0); - assert!(attr_idx < dictionary.attribute_table.len() as i32); - } - - // Verify the actual attribute content - let attr1 = &dictionary.attribute_table[sample.attribute_indices[0] as usize]; - let attr2 = &dictionary.attribute_table[sample.attribute_indices[1] as usize]; - - // One should be the string label, one should be the numeric label - let (string_attr, numeric_attr) = if attr1.key == "thread_id" { - (attr1, attr2) - } else { - (attr2, attr1) - }; - - // Verify string attribute - assert_eq!(string_attr.key, "thread_id"); - let s = match string_attr.value.as_ref().expect("Expected Some value") { - datadog_profiling_otel::key_value::Value::StringValue(s) => s, - _ => panic!("Expected StringValue"), - }; - assert_eq!(s, "main"); - - // Verify numeric attribute - assert_eq!(numeric_attr.key, "cpu_usage"); - let n = match numeric_attr.value.as_ref().expect("Expected Some value") { - datadog_profiling_otel::key_value::Value::IntValue(n) => n, - _ => panic!("Expected IntValue"), - }; - assert_eq!(*n, 75); - - // Verify attribute unit mapping - assert_eq!(dictionary.attribute_units.len(), 1); - let unit = &dictionary.attribute_units[0]; - assert!(unit.attribute_key_strindex > 0); - assert!(unit.unit_strindex > 0); - - // Test with timestamps - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&[sample_types[0]]); - let sample = crate::api::Sample { - locations: vec![location], - values: &[200], - labels: vec![], - }; - let timestamp = crate::internal::Timestamp::new(1234567890).unwrap(); - let _ = internal_profile.try_add_sample(sample, Some(timestamp)); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_eq!(profile.sample.len(), 1); - assert_sample_has_timestamp(&profile.sample[0], 1234567890); - - // Test zero value filtering - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&sample_types); - let sample = crate::api::Sample { - locations: vec![location], - values: &[0, 1024], // 0 nanoseconds, 1024 bytes - labels: vec![], - }; - let _ = internal_profile.try_add_sample(sample, None); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; - assert_eq!(scope_profile.profiles[0].sample.len(), 0); // Zero value filtered - assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![1024], 0, 0); - - // Test multiple samples aggregation - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&[sample_types[0]]); - let sample1 = crate::api::Sample { - locations: vec![location], - values: &[100], - labels: vec![], - }; - let sample2 = crate::api::Sample { - locations: vec![location], - values: &[200], - labels: vec![], - }; - let sample3 = crate::api::Sample { - locations: vec![location], - values: &[300], - labels: vec![], - }; - let _ = internal_profile.try_add_sample(sample1, None); - let _ = internal_profile.try_add_sample(sample2, None); - let _ = internal_profile.try_add_sample(sample3, None); - - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_eq!(profile.sample.len(), 1); - assert_eq!(profile.sample[0].values, vec![600]); // 100 + 200 + 300 - - assert_duration_calculation(&scope_profile.profiles); - } - - #[test] - fn test_duration_and_period() { - let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - - // Test duration calculation - let internal_profile = InternalProfile::new(&sample_types, None); - let explicit_duration = std::time::Duration::from_secs(5); - let otel_profiles_data = internal_profile - .convert_into_otel(None, Some(explicit_duration)) - .unwrap(); - let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_eq!(profile.duration_nanos, 5_000_000_000); - - // Test with explicit end_time - let internal_profile2 = InternalProfile::new(&sample_types, None); - let start_time = internal_profile2.start_time; - let end_time = start_time + std::time::Duration::from_secs(3); - let otel_profiles_data2 = internal_profile2 - .convert_into_otel(Some(end_time), None) - .unwrap(); - let profile2 = &otel_profiles_data2.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_eq!(profile2.duration_nanos, 3_000_000_000); - - // Test with both end_time and duration (duration should take precedence) - let internal_profile3 = InternalProfile::new(&sample_types, None); - let start_time3 = internal_profile3.start_time; - let end_time3 = start_time3 + std::time::Duration::from_secs(10); - let duration3 = std::time::Duration::from_secs(7); - let otel_profiles_data3 = internal_profile3 - .convert_into_otel(Some(end_time3), Some(duration3)) - .unwrap(); - let profile3 = &otel_profiles_data3.resource_profiles[0].scope_profiles[0].profiles[0]; - assert_eq!(profile3.duration_nanos, 7_000_000_000); - - // Test period conversion - let period = crate::api::Period { - r#type: crate::api::ValueType::new("cpu", "cycles"), - value: 1000, - }; - let internal_profile = InternalProfile::new(&sample_types, Some(period)); - let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); - let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; - assert!(profile.period_type.is_some()); - assert_eq!(profile.period, 1000); - - // Test without period - let internal_profile_no_period = InternalProfile::new(&sample_types, None); - let otel_profiles_data_no_period = internal_profile_no_period - .convert_into_otel(None, None) - .unwrap(); - let profile_no_period = - &otel_profiles_data_no_period.resource_profiles[0].scope_profiles[0].profiles[0]; - assert!(profile_no_period.period_type.is_none()); - assert_eq!(profile_no_period.period, 0); - } - - #[test] - #[cfg_attr(miri, ignore)] // Skip this test when running under Miri - fn test_serialize_into_compressed_otel() { - // Create an internal profile with sample types - let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; - let (mut internal_profile, location) = - setup_profile_with_function_and_location(&sample_types); - - // Add a sample - let sample = crate::api::Sample { - locations: vec![location], - values: &[150], - labels: vec![], - }; - let _ = internal_profile.try_add_sample(sample, None); - - // Test serialization to compressed OpenTelemetry format - let encoded_profile = internal_profile - .serialize_into_compressed_otel(None, None) - .unwrap(); - - // Verify the encoded profile structure - assert!(encoded_profile.start > std::time::UNIX_EPOCH); - assert!(encoded_profile.end > encoded_profile.start); - assert!(!encoded_profile.buffer.is_empty()); - - // Verify the buffer contains compressed data (should be smaller than uncompressed) - // The compressed buffer should be significantly smaller than a typical uncompressed profile - assert!(encoded_profile.buffer.len() < 10000); // Reasonable upper bound for this small profile - - // Verify endpoints stats are preserved - assert!(encoded_profile.endpoints_stats.is_empty()); // No endpoints added - } -} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs new file mode 100644 index 0000000000..1340546103 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile/mod.rs @@ -0,0 +1,237 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::collections::identifiable::{Dedup, Id}; +use crate::internal::profile::otel_emitter::label::convert_label_to_key_value; +use crate::internal::profile::{EncodedProfile, Profile as InternalProfile}; +use crate::iter::{IntoLendingIterator, LendingIterator}; +use anyhow::{Context, Result}; +use datadog_profiling_otel::ProfilesDataExt; +use std::collections::HashMap; + +impl InternalProfile { + /// Converts the profile into OpenTelemetry format + /// + /// * `end_time` - Optional end time of the profile. Passing None will use the current time. + /// * `duration` - Optional duration of the profile. Passing None will try to calculate the + /// duration based on the end time minus the start time, but under anomalous conditions this + /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an + /// earlier time. The duration will be set to zero these cases. + pub fn convert_into_otel( + mut self, + end_time: Option, + duration: Option, + ) -> anyhow::Result { + // Calculate duration using the same logic as encode + let end = end_time.unwrap_or_else(std::time::SystemTime::now); + let start = self.start_time; + let duration_nanos = duration + .unwrap_or_else(|| { + end.duration_since(start).unwrap_or({ + // Let's not throw away the whole profile just because the clocks were wrong. + // todo: log that the clock went backward (or programmer mistake). + std::time::Duration::ZERO + }) + }) + .as_nanos() + .min(i64::MAX as u128) as i64; + + // Create individual OpenTelemetry Profiles for each ValueType + let mut profiles = Vec::with_capacity(self.sample_types.len()); + + for sample_type in self.sample_types.iter() { + // Convert the ValueType to OpenTelemetry format + let otel_sample_type = datadog_profiling_otel::ValueType { + type_strindex: sample_type.r#type.value.to_raw_id() as i32, + unit_strindex: sample_type.unit.value.to_raw_id() as i32, + aggregation_temporality: datadog_profiling_otel::AggregationTemporality::Delta + .into(), + }; + + // Create a Profile for this sample type + let profile = datadog_profiling_otel::Profile { + sample_type: Some(otel_sample_type), + sample: vec![], + time_nanos: self + .start_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as i64, + duration_nanos, // Use calculated duration + period_type: self.period.as_ref().map(|(_, period_type)| { + datadog_profiling_otel::ValueType { + type_strindex: period_type.r#type.value.to_raw_id() as i32, + unit_strindex: period_type.unit.value.to_raw_id() as i32, + aggregation_temporality: + datadog_profiling_otel::AggregationTemporality::Delta.into(), + } + }), + period: self + .period + .map(|(period_value, _)| period_value) + .unwrap_or(0), + comment_strindices: vec![], // We don't have comments + profile_id: vec![], // TODO: Implement when we handle profile IDs + dropped_attributes_count: 0, // We don't drop attributes + original_payload_format: String::new(), // There is no original payload + original_payload: vec![], // There is no original payload + attribute_indices: vec![], // There are currently no attributes at this level + }; + + profiles.push(profile); + } + + // If we have span labels, figure out the corresponding endpoint labels + // Split into two steps to avoid mutating the map we're iterating over + let endpoint_labels: Vec<_> = self + .labels + .iter() + .enumerate() + .filter(|(_, label)| label.get_key() == self.endpoints.local_root_span_id_label) + .filter_map(|(idx, label)| { + self.get_endpoint_for_label(label) + .ok() + .flatten() + .map(|endpoint_label| (idx, endpoint_label)) + }) + .collect(); + + let endpoint_labels_idx: HashMap = endpoint_labels + .into_iter() + .map(|(idx, endpoint_label)| { + let endpoint_idx = self.labels.dedup(endpoint_label); + (idx, endpoint_idx) + }) + .collect(); + + for (sample, timestamp, mut values) in std::mem::take(&mut self.observations).into_iter() { + let stack_index = sample.stacktrace.to_raw_id() as i32; + let label_set = self.get_label_set(sample.labels)?; + let attribute_indicies: Vec<_> = label_set + .iter() + .map(|x| x.to_raw_id() as i32) + .chain( + label_set + .iter() + .find_map(|k| endpoint_labels_idx.get(&(k.to_raw_id() as usize))) + .map(|label| label.to_raw_id() as i32), + ) + .collect(); + let labels = label_set + .iter() + .map(|l| self.get_label(*l).copied()) + .collect::>>()?; + let link_index = 0; // TODO, handle links properly + let timestamps_unix_nano = timestamp.map_or(vec![], |ts| vec![ts.get() as u64]); + self.upscaling_rules.upscale_values(&mut values, &labels)?; + + for (idx, value) in values.iter().enumerate() { + if *value != 0 { + let otel_sample = datadog_profiling_otel::Sample { + stack_index, + attribute_indices: attribute_indicies.clone(), + link_index, + values: vec![*value], + timestamps_unix_nano: timestamps_unix_nano.clone(), + }; + profiles[idx].sample.push(otel_sample); + } + } + } + + // Convert string table using into_lending_iter + // Note: We can't use .map().collect() here because LendingIterator doesn't implement + // the standard Iterator trait. LendingIterator is designed for yielding references + // with lifetimes tied to the iterator itself, so we need to manually iterate and + // convert each string reference to an owned String. + let string_table = { + let mut strings = Vec::with_capacity(self.strings.len()); + let mut iter = self.strings.into_lending_iter(); + while let Some(s) = iter.next() { + strings.push(s.to_string()); + } + strings + }; + + // Convert labels to KeyValues for the attribute table + let mut key_to_unit_map = HashMap::new(); + let mut attribute_table = Vec::with_capacity(self.labels.len()); + + for label in self.labels.iter() { + let key_value = convert_label_to_key_value(label, &string_table, &mut key_to_unit_map) + .with_context(|| { + format!( + "Failed to convert label with key index {}", + label.get_key().to_raw_id() + ) + })?; + attribute_table.push(key_value); + } + + // Build attribute units from the key-to-unit mapping + let attribute_units = key_to_unit_map + .into_iter() + .map( + |(key_index, unit_index)| datadog_profiling_otel::AttributeUnit { + attribute_key_strindex: key_index as i32, + unit_strindex: unit_index as i32, + }, + ) + .collect(); + + // Convert the ProfilesDictionary components + let dictionary = datadog_profiling_otel::ProfilesDictionary { + mapping_table: self.mappings.into_iter().map(From::from).collect(), + location_table: self.locations.into_iter().map(From::from).collect(), + function_table: self.functions.into_iter().map(From::from).collect(), + stack_table: self.stack_traces.into_iter().map(From::from).collect(), + string_table, + attribute_table, + attribute_units, + link_table: vec![], // TODO: Implement when we handle trace links + }; + + // Create a basic ResourceProfiles structure + let resource_profiles = vec![datadog_profiling_otel::ResourceProfiles { + resource: None, // TODO: Implement when we handle resources + scope_profiles: vec![datadog_profiling_otel::ScopeProfiles { + scope: None, // It is legal to leave this unset according to the spec + profiles, // Now contains the individual profiles + schema_url: String::new(), // TODO: Implement when we handle schema URLs + default_sample_type: None, // TODO: Implement when we handle sample types + }], + schema_url: String::new(), // TODO: Implement when we handle schema URLs + }]; + + Ok(datadog_profiling_otel::ProfilesData { + resource_profiles, + dictionary: Some(dictionary), + }) + } + + /// Serializes the profile into OpenTelemetry format and compresses it using zstd. + /// + /// * `end_time` - Optional end time of the profile. Passing None will use the current time. + /// * `duration` - Optional duration of the profile. Passing None will try to calculate the + /// duration based on the end time minus the start time, but under anomalous conditions this + /// may fail as system clocks can be adjusted. The programmer may also accidentally pass an + /// earlier time. The duration will be set to zero these cases. + pub fn serialize_into_compressed_otel( + mut self, + end_time: Option, + duration: Option, + ) -> anyhow::Result { + // Extract values before consuming self + let start = self.start_time; + let endpoints_stats = std::mem::take(&mut self.endpoints.stats); + let otel_profiles_data = self.convert_into_otel(end_time, duration)?; + let buffer = otel_profiles_data.serialize_into_compressed_proto()?; + let end = end_time.unwrap_or_else(std::time::SystemTime::now); + Ok(EncodedProfile { + start, + end, + buffer, + endpoints_stats, + }) + } +} diff --git a/datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs b/datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs new file mode 100644 index 0000000000..1d22f6ee28 --- /dev/null +++ b/datadog-profiling/src/internal/profile/otel_emitter/profile/tests.rs @@ -0,0 +1,406 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::internal; + +// Helper functions for test setup +fn create_basic_function() -> crate::api::Function<'static> { + crate::api::Function { + name: "test_function", + system_name: "test_system", + filename: "test_file.rs", + } +} + +fn create_basic_mapping() -> crate::api::Mapping<'static> { + crate::api::Mapping { + memory_start: 0x1000, + memory_limit: 0x2000, + file_offset: 0, + filename: "test_binary", + build_id: "test_build_id", + } +} + +fn setup_profile_with_function_and_location( + sample_types: &[crate::api::ValueType<'static>], +) -> (internal::profile::Profile, crate::api::Location<'static>) { + let mut internal_profile = internal::profile::Profile::new(sample_types, None); + let function = create_basic_function(); + let mapping = create_basic_mapping(); + let location = crate::api::Location { + mapping, + function, + address: 0x1000, + line: 42, + }; + + let _function_id = internal_profile.try_add_function(&function); + let _mapping_id = internal_profile.try_add_mapping(&mapping); + let location_id = internal_profile.try_add_location(&location).unwrap(); + let _stack_trace_id = internal_profile.try_add_stacktrace(vec![location_id]); + + (internal_profile, location) +} + +fn create_string_label(key: &'static str, value: &'static str) -> crate::api::Label<'static> { + crate::api::Label { + key, + str: value, + num: 0, + num_unit: "", + } +} + +fn create_numeric_label( + key: &'static str, + value: i64, + unit: &'static str, +) -> crate::api::Label<'static> { + crate::api::Label { + key, + str: "", + num: value, + num_unit: unit, + } +} + +// Common assertion helpers +fn assert_duration_calculation(profiles: &[datadog_profiling_otel::Profile]) { + for profile in profiles { + assert!(profile.duration_nanos > 0); + } +} + +fn assert_profile_has_correct_sample( + profile: &datadog_profiling_otel::Profile, + expected_values: Vec, + expected_stack_index: i32, + expected_attribute_count: usize, +) { + assert_eq!(profile.sample.len(), 1); + let sample = &profile.sample[0]; + assert_eq!(sample.values, expected_values); + assert_eq!(sample.stack_index, expected_stack_index); + assert_eq!(sample.attribute_indices.len(), expected_attribute_count); +} + +fn assert_sample_has_timestamp( + sample: &datadog_profiling_otel::Sample, + expected_timestamp: u64, +) { + assert_eq!(sample.timestamps_unix_nano.len(), 1); + assert_eq!(sample.timestamps_unix_nano[0], expected_timestamp); +} + +fn assert_profiles_data_structure(otel_profiles_data: &datadog_profiling_otel::ProfilesData) { + assert!(otel_profiles_data.dictionary.is_some()); + assert_eq!(otel_profiles_data.resource_profiles.len(), 1); + assert_eq!( + otel_profiles_data.resource_profiles[0].scope_profiles.len(), + 1 + ); +} + +#[test] +fn test_convert_into_otel() { + // Test empty profile + let internal_profile = internal::profile::Profile::new(&[], None); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + assert_profiles_data_structure(&otel_profiles_data); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_eq!(dictionary.mapping_table.len(), 0); + assert_eq!(dictionary.location_table.len(), 0); + assert_eq!(dictionary.function_table.len(), 0); + assert_eq!(dictionary.stack_table.len(), 0); + assert_eq!(dictionary.string_table.len(), 4); + assert_eq!(dictionary.string_table[0], ""); // Empty string + assert_eq!(dictionary.string_table[1], "local root span id"); + assert_eq!(dictionary.string_table[2], "trace endpoint"); + assert_eq!(dictionary.string_table[3], "end_timestamp_ns"); + assert_eq!(dictionary.link_table.len(), 0); + assert_eq!(dictionary.attribute_table.len(), 0); + assert_eq!(dictionary.attribute_units.len(), 0); + + // Test with functions + let mut internal_profile = internal::profile::Profile::new(&[], None); + let function1 = crate::api::Function { + name: "test_function_1", + system_name: "test_system_1", + filename: "test_file_1.rs", + }; + let function2 = crate::api::Function { + name: "test_function_2", + system_name: "test_system_2", + filename: "test_file_2.rs", + }; + let _function1_id = internal_profile.try_add_function(&function1); + let _function2_id = internal_profile.try_add_function(&function2); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_eq!(dictionary.function_table.len(), 2); + assert_eq!(dictionary.string_table.len(), 10); + + // Test with labels + let mut internal_profile = internal::profile::Profile::new(&[], None); + let label1 = create_string_label("thread_id", "main"); + let label2 = create_numeric_label("memory_usage", 1024, "bytes"); + let sample = crate::api::Sample { + locations: vec![], + values: &[42], + labels: vec![label1, label2], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let dictionary = otel_profiles_data.dictionary.unwrap(); + assert_eq!(dictionary.attribute_table.len(), 2); + assert_eq!(dictionary.attribute_units.len(), 1); + + // Test with sample types + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("allocations", "count"), + ]; + let internal_profile = internal::profile::Profile::new(&sample_types, None); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + assert_eq!(scope_profile.profiles.len(), 2); + assert_duration_calculation(&scope_profile.profiles); +} + +#[test] +fn test_sample_conversion() { + let sample_types = [ + crate::api::ValueType::new("cpu", "nanoseconds"), + crate::api::ValueType::new("memory", "bytes"), + ]; + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); + + // Test basic sample conversion + let sample = crate::api::Sample { + locations: vec![location], + values: &[100, 2048], + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + assert_eq!(scope_profile.profiles.len(), 2); + assert_profile_has_correct_sample(&scope_profile.profiles[0], vec![100], 0, 0); + assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![2048], 0, 0); + + // Test with labels + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); + let sample = crate::api::Sample { + locations: vec![location], + values: &[150], + labels: vec![ + create_string_label("thread_id", "main"), + create_numeric_label("cpu_usage", 75, "percent"), + ], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_profile_has_correct_sample(profile, vec![150], 0, 2); + + // Verify the sample's attribute indices point to correct attributes + let sample = &profile.sample[0]; + let dictionary = &otel_profiles_data.dictionary.as_ref().unwrap(); + + // Check that attribute indices are valid + for &attr_idx in &sample.attribute_indices { + assert!(attr_idx >= 0); + assert!(attr_idx < dictionary.attribute_table.len() as i32); + } + + // Verify the actual attribute content + let attr1 = &dictionary.attribute_table[sample.attribute_indices[0] as usize]; + let attr2 = &dictionary.attribute_table[sample.attribute_indices[1] as usize]; + + // One should be the string label, one should be the numeric label + let (string_attr, numeric_attr) = if attr1.key == "thread_id" { + (attr1, attr2) + } else { + (attr2, attr1) + }; + + // Verify string attribute + assert_eq!(string_attr.key, "thread_id"); + let s = match string_attr.value.as_ref().expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::StringValue(s) => s, + _ => panic!("Expected StringValue"), + }; + assert_eq!(s, "main"); + + // Verify numeric attribute + assert_eq!(numeric_attr.key, "cpu_usage"); + let n = match numeric_attr.value.as_ref().expect("Expected Some value") { + datadog_profiling_otel::key_value::Value::IntValue(n) => n, + _ => panic!("Expected IntValue"), + }; + assert_eq!(*n, 75); + + // Verify attribute unit mapping + assert_eq!(dictionary.attribute_units.len(), 1); + let unit = &dictionary.attribute_units[0]; + assert!(unit.attribute_key_strindex > 0); + assert!(unit.unit_strindex > 0); + + // Test with timestamps + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); + let sample = crate::api::Sample { + locations: vec![location], + values: &[200], + labels: vec![], + }; + let timestamp = crate::internal::Timestamp::new(1234567890).unwrap(); + let _ = internal_profile.try_add_sample(sample, Some(timestamp)); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile.sample.len(), 1); + assert_sample_has_timestamp(&profile.sample[0], 1234567890); + + // Test zero value filtering + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); + let sample = crate::api::Sample { + locations: vec![location], + values: &[0, 1024], // 0 nanoseconds, 1024 bytes + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let scope_profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0]; + assert_eq!(scope_profile.profiles[0].sample.len(), 0); // Zero value filtered + assert_profile_has_correct_sample(&scope_profile.profiles[1], vec![1024], 0, 0); + + // Test multiple samples aggregation + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&[sample_types[0]]); + let sample1 = crate::api::Sample { + locations: vec![location], + values: &[100], + labels: vec![], + }; + let sample2 = crate::api::Sample { + locations: vec![location], + values: &[200], + labels: vec![], + }; + let sample3 = crate::api::Sample { + locations: vec![location], + values: &[300], + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample1, None); + let _ = internal_profile.try_add_sample(sample2, None); + let _ = internal_profile.try_add_sample(sample3, None); + + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile.sample.len(), 1); + assert_eq!(profile.sample[0].values, vec![600]); // 100 + 200 + 300 + + assert_duration_calculation(&scope_profile.profiles); +} + +#[test] +fn test_duration_and_period() { + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + + // Test duration calculation + let internal_profile = internal::profile::Profile::new(&sample_types, None); + let explicit_duration = std::time::Duration::from_secs(5); + let otel_profiles_data = internal_profile + .convert_into_otel(None, Some(explicit_duration)) + .unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile.duration_nanos, 5_000_000_000); + + // Test with explicit end_time + let internal_profile2 = internal::profile::Profile::new(&sample_types, None); + let start_time = internal_profile2.start_time; + let end_time = start_time + std::time::Duration::from_secs(3); + let otel_profiles_data2 = internal_profile2 + .convert_into_otel(Some(end_time), None) + .unwrap(); + let profile2 = &otel_profiles_data2.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile2.duration_nanos, 3_000_000_000); + + // Test with both end_time and duration (duration should take precedence) + let internal_profile3 = internal::profile::Profile::new(&sample_types, None); + let start_time3 = internal_profile3.start_time; + let end_time3 = start_time3 + std::time::Duration::from_secs(10); + let duration3 = std::time::Duration::from_secs(7); + let otel_profiles_data3 = internal_profile3 + .convert_into_otel(Some(end_time3), Some(duration3)) + .unwrap(); + let profile3 = &otel_profiles_data3.resource_profiles[0].scope_profiles[0].profiles[0]; + assert_eq!(profile3.duration_nanos, 7_000_000_000); + + // Test period conversion + let period = crate::api::Period { + r#type: crate::api::ValueType::new("cpu", "cycles"), + value: 1000, + }; + let internal_profile = internal::profile::Profile::new(&sample_types, Some(period)); + let otel_profiles_data = internal_profile.convert_into_otel(None, None).unwrap(); + let profile = &otel_profiles_data.resource_profiles[0].scope_profiles[0].profiles[0]; + assert!(profile.period_type.is_some()); + assert_eq!(profile.period, 1000); + + // Test without period + let internal_profile_no_period = internal::profile::Profile::new(&sample_types, None); + let otel_profiles_data_no_period = internal_profile_no_period + .convert_into_otel(None, None) + .unwrap(); + let profile_no_period = + &otel_profiles_data_no_period.resource_profiles[0].scope_profiles[0].profiles[0]; + assert!(profile_no_period.period_type.is_none()); + assert_eq!(profile_no_period.period, 0); +} + +#[test] +#[cfg_attr(miri, ignore)] // Skip this test when running under Miri +fn test_serialize_into_compressed_otel() { + // Create an internal profile with sample types + let sample_types = [crate::api::ValueType::new("cpu", "nanoseconds")]; + let (mut internal_profile, location) = + setup_profile_with_function_and_location(&sample_types); + + // Add a sample + let sample = crate::api::Sample { + locations: vec![location], + values: &[150], + labels: vec![], + }; + let _ = internal_profile.try_add_sample(sample, None); + + // Test serialization to compressed OpenTelemetry format + let encoded_profile = internal_profile + .serialize_into_compressed_otel(None, None) + .unwrap(); + + // Verify the encoded profile structure + assert!(encoded_profile.start > std::time::UNIX_EPOCH); + assert!(encoded_profile.end > encoded_profile.start); + assert!(!encoded_profile.buffer.is_empty()); + + // Verify the buffer contains compressed data (should be smaller than uncompressed) + // The compressed buffer should be significantly smaller than a typical uncompressed profile + assert!(encoded_profile.buffer.len() < 10000); // Reasonable upper bound for this small profile + + // Verify endpoints stats are preserved + assert!(encoded_profile.endpoints_stats.is_empty()); // No endpoints added +} + diff --git a/datadog-trace-protobuf/src/remoteconfig.rs b/datadog-trace-protobuf/src/remoteconfig.rs index fdc89ee871..a8a4b66524 100644 --- a/datadog-trace-protobuf/src/remoteconfig.rs +++ b/datadog-trace-protobuf/src/remoteconfig.rs @@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize}; // This file is @generated by prost-build. -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct File { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -12,8 +11,7 @@ pub struct File { #[serde(with = "serde_bytes")] pub raw: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct Client { #[prost(message, optional, tag = "1")] pub state: ::core::option::Option, @@ -35,8 +33,7 @@ pub struct Client { #[prost(bytes = "vec", tag = "11")] pub capabilities: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientTracer { #[prost(string, tag = "1")] pub runtime_id: ::prost::alloc::string::String, @@ -55,8 +52,7 @@ pub struct ClientTracer { #[prost(string, repeated, tag = "7")] pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientAgent { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, @@ -69,8 +65,7 @@ pub struct ClientAgent { #[prost(string, repeated, tag = "5")] pub cws_workloads: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ConfigState { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, @@ -83,8 +78,7 @@ pub struct ConfigState { #[prost(string, tag = "5")] pub apply_error: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientState { #[prost(uint64, tag = "1")] pub root_version: u64, @@ -99,16 +93,14 @@ pub struct ClientState { #[prost(bytes = "vec", tag = "6")] pub backend_client_state: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct TargetFileHash { #[prost(string, tag = "1")] pub algorithm: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub hash: ::prost::alloc::string::String, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct TargetFileMeta { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, @@ -117,16 +109,14 @@ pub struct TargetFileMeta { #[prost(message, repeated, tag = "3")] pub hashes: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsRequest { #[prost(message, optional, tag = "1")] pub client: ::core::option::Option, #[prost(message, repeated, tag = "2")] pub cached_target_files: ::prost::alloc::vec::Vec, } -#[derive(Deserialize, Serialize)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Deserialize, Serialize, Clone, PartialEq, ::prost::Message)] pub struct ClientGetConfigsResponse { #[prost(bytes = "vec", repeated, tag = "1")] #[serde(with = "crate::serde")]