Rename Field to Record, document more things, etc

Author: morrisonlevi

datadog-profiling-protobuf/src/function.rs

@@ -1,27 +1,27 @@
 // Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
-use super::{Field, StringOffset, Value, WireType, NO_OPT_ZERO, OPT_ZERO};
+use super::{Record, StringOffset, Value, WireType, NO_OPT_ZERO, OPT_ZERO};
 use std::io::{self, Write};
 
 /// Represents a function in a profile. Omits the start line because it's not
-/// useful to libdatadog right now, so we save the bytes/ops.
+/// useful to Datadog right now, so we save the bytes/ops.
 #[repr(C)]
 #[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
 #[cfg_attr(test, derive(bolero::generator::TypeGenerator))]
 pub struct Function {
     /// Unique nonzero id for the function.
-    pub id: Field<u64, 1, NO_OPT_ZERO>,
+    pub id: Record<u64, 1, NO_OPT_ZERO>,
     /// Name of the function, in human-readable form if available.
-    pub name: Field<StringOffset, 2, OPT_ZERO>,
+    pub name: Record<StringOffset, 2, OPT_ZERO>,
     /// Name of the function, as identified by the system.
     /// For instance, it can be a C++ mangled name.
-    pub system_name: Field<StringOffset, 3, OPT_ZERO>,
+    pub system_name: Record<StringOffset, 3, OPT_ZERO>,
     /// Source file containing the function.
-    pub filename: Field<StringOffset, 4, OPT_ZERO>,
+    pub filename: Record<StringOffset, 4, OPT_ZERO>,
 }
 
-impl Value for Function {
+unsafe impl Value for Function {
     const WIRE_TYPE: WireType = WireType::LengthDelimited;
 
     fn proto_len(&self) -> u64 {

datadog-profiling-protobuf/src/label.rs

@@ -1,7 +1,7 @@
 // Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
-use super::{Field, StringOffset, Value, WireType, OPT_ZERO};
+use super::{Record, StringOffset, Value, WireType, OPT_ZERO};
 use std::io::{self, Write};
 
 /// Label includes additional context for this sample. It can include things
@@ -11,22 +11,22 @@ use std::io::{self, Write};
 #[cfg_attr(test, derive(bolero::generator::TypeGenerator))]
 pub struct Label {
     /// An annotation for a sample, e.g. "allocation_size".
-    pub key: Field<StringOffset, 1, OPT_ZERO>,
+    pub key: Record<StringOffset, 1, OPT_ZERO>,
     /// At most, one of the str and num should be used.
-    pub str: Field<StringOffset, 2, OPT_ZERO>,
+    pub str: Record<StringOffset, 2, OPT_ZERO>,
     /// At most, one of the str and num should be used.
-    pub num: Field<i64, 3, OPT_ZERO>,
+    pub num: Record<i64, 3, OPT_ZERO>,
 
     // todo: if we don't use num_unit, then we can save 8 bytes--4 from
     //       num_unit plus 4 from padding.
     /// Should only be present when num is present.
     /// Specifies the units of num.
     /// Use arbitrary string (for example, "requests") as a custom count unit.
     /// If no unit is specified, consumer may apply heuristic to deduce it.
-    pub num_unit: Field<StringOffset, 4, OPT_ZERO>,
+    pub num_unit: Record<StringOffset, 4, OPT_ZERO>,
 }
 
-impl Value for Label {
+unsafe impl Value for Label {
     const WIRE_TYPE: WireType = WireType::LengthDelimited;
 
     fn proto_len(&self) -> u64 {

datadog-profiling-protobuf/src/lib.rs

@@ -6,8 +6,8 @@
 #![cfg_attr(not(test), deny(clippy::expect_used))]
 #![cfg_attr(not(test), deny(clippy::unimplemented))]
 
-//! This crate implements Protobuf serializers for [`profiles`], including
-//! serializers for:
+//! This crate implements Protobuf encoders for [`profiles`] which write to a
+//! [`Write`]. It has encoders for:
 //!
 //! - [Function]
 //! - [Label]
@@ -16,19 +16,42 @@
 //! - [Sample]
 //! - [ValueType]
 //!
-//! Serialization often happens one byte at a time, so a buffered writer
-//! should probably be used.
+//! There is no encoder for a Profile message. It would require borrowing a
+//! lot of data, which becomes unwieldy. It also isn't very compatible with
+//! writing a streaming serializer to lower peak memory usage.
 //!
-//! There is no serializer for Profile. It would require borrowing a lot of
-//! data, which becomes unwieldy. It also isn't very compatible with writing
-//! a streaming serializer to lower peak memory usage.
+//! Encoding often happens one byte at a time, so a buffered writer should
+//! probably be used.
 //!
 //! Indices into the string table are represented by [StringOffset], which uses
 //! a 32-bit number. ID fields are still 64-bit, so the user can control their
 //! values, potentially using a 64-bit address for its value.
 //!
 //! The types are generally `#[repr(C)]` so they can be used in FFI one day.
 //!
+//! Here is a condensed reference for the parts of protobuf used by profiles:
+//!
+//! ```reference
+//! message    := (tag value)*
+//! tag        := (field << 3) bit-or wire_type;
+//!                 encoded as uint32 varint
+//! value      := varint      for wire_type == VARINT,
+//!               len-prefix  for wire_type == LEN,
+//! varint     := int64 | uint64
+//! len-prefix := size (message | string | packed);
+//!                 size encoded as int32 varint
+//! string     := valid UTF-8 string;
+//!                 max 2GB of bytes
+//! packed     := varint*
+//!                 consecutive values of the type specified in `.proto`
+//! ```
+//!
+//! A [`Record`] represents a [`Tag`] and [`Value`] pair, where the
+//! [`WireType`] comes from [`Value::WIRE_TYPE`].
+//!
+//! Protos must be smaller than 2 GiB when encoded. Many proto implementations
+//! will refuse to encode or decode messages that exceed this limit.
+//!
 //! [`profiles`]: https://github.com/google/pprof/blob/main/proto/profile.proto
 
 mod function;
@@ -48,61 +71,78 @@ pub use label::*;
 pub use location::*;
 pub use mapping::*;
 pub use sample::*;
-use std::fmt::{Debug, Formatter};
 pub use string::*;
 pub use value_type::*;
 
+use std::fmt::{Debug, Formatter};
 use std::io::{self, Write};
 
-/// Create a field of a given type, field number, and whether to perform the
-/// zero-size optimization or not.
+/// A record is responsible for encoding the field number, wire type and
+/// payload. The wire type tells the parser how big the payload after it is.
+/// For more details, refer to the [Condensed Reference Card].
+///
+/// The `P` is the payload, the `F` is the field number, and `O` is whether to
+/// apply the zero-sized optimization or not. Most of the time, it shouldn't
+/// matter if the optimization is applied. However, if something is part of
+/// a repeated field, then applying the optimization would change the number
+/// of elements in the array.
+///
+/// [Condensed Reference Card]: https://protobuf.dev/programming-guides/encoding/#cheat-sheet
 #[derive(Copy, Clone, Default, Eq, PartialEq)]
 #[repr(transparent)]
 #[cfg_attr(test, derive(bolero::generator::TypeGenerator))]
-pub struct Field<T: Value, const N: u32, const O: bool> {
-    pub value: T,
+pub struct Record<P: Value, const F: u32, const O: bool> {
+    pub value: P,
 }
 
 /// Represents the wire type for the in-wire protobuf encoding. There are more
-/// types than are represented here; these are just the supported ones.
+/// types than are represented here; these are just the ones used in profiles.
+/// See [Message Structure] for more documentation.
+///
+/// [Message Structure]: https://protobuf.dev/programming-guides/encoding/#structure
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 #[repr(u8)]
 pub enum WireType {
     Varint = 0,
     LengthDelimited = 2,
 }
 
-/// A value is stored differently depending on the wire_type.
-pub trait Value: Default + Eq {
+/// A value (or payload) is stored differently depending on the wire_type. In
+/// profiles, there two types of payloads: varints and len-prefixed types.
+///
+/// # Safety
+///
+/// The [`Default`] implementation _must_ provide all zero values.
+pub unsafe trait Value: Default + Eq {
     /// The wire type this value uses.
     const WIRE_TYPE: WireType;
 
-    /// The number of bytes it takes to encode this value.
+    /// The number of bytes it takes to encode this value. Do not include the
+    /// number of bytes it takes to encode the length-prefix as a varint. For
+    /// example, using this snippet of the reference:
+    ///
+    /// ```reference
+    /// len-prefix := size (message | string | packed);
+    ///                size encoded as int32 varint
+    /// ```
+    ///
+    /// Calculate the number of bytes for `(message |  string | packed)` only.
+    ///
+    /// For a varint, returns between 1 and 10 bytes for the number of bytes
+    /// used to encode the varint.
+    ///
+    /// Returns u64 rather than u31 to avoid a lot of overflow checking.
     fn proto_len(&self) -> u64;
 
-    /// Encode the value to the in-wire protobuf format.
+    /// Encode the value to the in-wire protobuf format. For length-delimited
+    /// types, do not include the length-prefix; see [`Value::proto_len`] for
+    /// more details.
     ///
-    /// Serialization often happens one byte at a time, so a buffered writer
-    /// should probably be used.
+    /// Encoding often happens one byte at a time, so a buffered writer should
+    /// probably be used.
     fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()>;
 }
 
-/// You can use varint to store any of the listed data types:
-/// int32 | int64 | uint32 | uint64 | bool | enum | sint32 | sint64
-///
-/// # Safety
-///
-/// The [`Value::WIRE_TYPE`] must be [`WireType::Varint`]!
-pub unsafe trait Varint: Value + Sized {}
-
-/// You can use LengthDelimited to store any of the listed data types:
-/// string, bytes, embedded messages, packed repeated fields
-///
-/// # Safety
-///
-/// The [`Value::WIRE_TYPE`] must be [`WireType::LengthDelimited`]!
-pub unsafe trait LengthDelimited: Value + Sized {}
-
 /// Intended to be provided to a Field to mean that it _should_ optimize for a
 /// value of zero. See also [`NO_OPT_ZERO`].
 pub const OPT_ZERO: bool = true;
@@ -112,44 +152,46 @@ pub const OPT_ZERO: bool = true;
 /// Mapping.id.
 pub const NO_OPT_ZERO: bool = false;
 
-impl<T: Value, const N: u32, const O: bool> From<T> for Field<T, N, O> {
-    fn from(value: T) -> Self {
-        Field { value }
+impl<P: Value, const F: u32, const O: bool> From<P> for Record<P, F, O> {
+    fn from(value: P) -> Self {
+        Record { value }
     }
 }
 
-impl<T: Value, const N: u32, const O: bool> Field<T, N, O> {
-    pub fn proto_len(&self) -> u64 {
-        if O && self.value == T::default() {
+unsafe impl<P: Value, const F: u32, const O: bool> Value for Record<P, F, O> {
+    const WIRE_TYPE: WireType = P::WIRE_TYPE;
+
+    fn proto_len(&self) -> u64 {
+        if O && self.value == P::default() {
             return 0;
         }
         let proto_len = self.value.proto_len();
-        let len = if T::WIRE_TYPE == WireType::LengthDelimited {
+        let len = if P::WIRE_TYPE == WireType::LengthDelimited {
             proto_len.proto_len()
         } else {
             0
         };
-        let tag = Tag::new(N, T::WIRE_TYPE).proto_len();
+        let tag = Tag::new(F, P::WIRE_TYPE).proto_len();
         tag + len + proto_len
     }
 
-    pub fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()> {
-        if O && self.value == T::default() {
+    fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()> {
+        if O && self.value == P::default() {
             return Ok(());
         }
-        Tag::new(N, T::WIRE_TYPE).encode(writer)?;
-        if T::WIRE_TYPE == WireType::LengthDelimited {
-            self.value.proto_len().encode(writer)?;
+        Tag::new(F, P::WIRE_TYPE).encode(writer)?;
+        if P::WIRE_TYPE == WireType::LengthDelimited {
+            varint::encode(self.value.proto_len(), writer)?;
         }
         self.value.encode(writer)
     }
 }
 
-impl<T: Debug + Value, const N: u32, const O: bool> Debug for Field<T, N, O> {
+impl<P: Debug + Value, const F: u32, const O: bool> Debug for Record<P, F, O> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Field")
             .field("value", &self.value)
-            .field("number", &N)
+            .field("number", &F)
             .field("optimize_for_zero", &O)
             .finish()
     }
@@ -176,11 +218,26 @@ impl Tag {
 
     #[inline]
     pub fn proto_len(self) -> u64 {
-        (self.0 as u64).proto_len()
+        varint::proto_len(self.0 as u64)
     }
 
     #[inline]
     pub fn encode<W: Write>(self, writer: &mut W) -> io::Result<()> {
-        (self.0 as u64).encode(writer)
+        varint::encode(self.0 as u64, writer)
+    }
+}
+
+unsafe impl<T: Value> Value for &'_ [T] {
+    const WIRE_TYPE: WireType = WireType::LengthDelimited;
+
+    fn proto_len(&self) -> u64 {
+        self.iter().map(Value::proto_len).sum()
+    }
+
+    fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()> {
+        for value in self.iter() {
+            value.encode(writer)?;
+        }
+        Ok(())
     }
 }