docs: create examples and document types

Author: morrisonlevi

datadog-profiling-protobuf/src/function.rs

@@ -4,33 +4,47 @@
 use super::{StringOffset, Value, Varint, WireType};
 use std::io::{self, Write};
 
+/// Represents a function in a profile. Omits the start line because it's not
+/// useful to libdatadog right now, so we save the bytes/ops.
 #[repr(C)]
 #[derive(Copy, Clone, Debug)]
 #[cfg_attr(test, derive(bolero::generator::TypeGenerator))]
 pub struct Function {
-    pub id: u64,                   // 1
-    pub name: StringOffset,        // 2
+    /// Unique nonzero id for the function.
+    pub id: u64, // 1
+    /// Name of the function, in human-readable form if available.
+    pub name: StringOffset, // 2
+    /// Name of the function, as identified by the system.
+    /// For instance, it can be a C++ mangled name.
     pub system_name: StringOffset, // 3
-    pub filename: StringOffset,    // 4
+    /// Source file containing the function.
+    pub filename: StringOffset, // 4
 }
 
 impl Value for Function {
     const WIRE_TYPE: WireType = WireType::LengthDelimited;
 
     fn proto_len(&self) -> u64 {
         Varint(self.id).field(1).proto_len()
-            + Varint::from(self.name).field(2).proto_len_small()
-            + Varint::from(self.system_name).field(3).proto_len_small()
-            + Varint::from(self.filename).field(4).proto_len_small()
+            + Varint::from(self.name).field(2).zero_opt().proto_len()
+            + Varint::from(self.system_name)
+                .field(3)
+                .zero_opt()
+                .proto_len()
+            + Varint::from(self.filename).field(4).zero_opt().proto_len()
     }
 
     fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         Varint(self.id).field(1).encode(writer)?;
-        Varint::from(self.name).field(2).encode_small(writer)?;
+        Varint::from(self.name).field(2).zero_opt().encode(writer)?;
         Varint::from(self.system_name)
             .field(3)
-            .encode_small(writer)?;
-        Varint::from(self.filename).field(4).encode_small(writer)
+            .zero_opt()
+            .encode(writer)?;
+        Varint::from(self.filename)
+            .field(4)
+            .zero_opt()
+            .encode(writer)
     }
 }
 

datadog-profiling-protobuf/src/label.rs

@@ -4,15 +4,25 @@
 use super::{StringOffset, Value, Varint, WireType};
 use std::io::{self, Write};
 
-// todo: if we don't use num_unit, then we can save 8 bytes--4 from num_unit
-//       plus 4 from padding.
+/// Label includes additional context for this sample. It can include things
+/// like a thread id, allocation size, etc.
 #[repr(C)]
 #[derive(Copy, Clone, Debug)]
 #[cfg_attr(test, derive(bolero::generator::TypeGenerator))]
 pub struct Label {
-    pub key: StringOffset,      // 1
-    pub str: StringOffset,      // 2
-    pub num: i64,               // 3
+    /// An annotation for a sample, e.g. "allocation_size".
+    pub key: StringOffset, // 1
+    /// At most, one of the str and num should be used.
+    pub str: StringOffset, // 2
+    /// At most, one of the str and num should be used.
+    pub num: i64, // 3
+
+    // todo: if we don't use num_unit, then we can save 8 bytes--4 from
+    //       num_unit plus 4 from padding.
+    /// Should only be present when num is present.
+    /// Specifies the units of num.
+    /// Use arbitrary string (for example, "requests") as a custom count unit.
+    /// If no unit is specified, consumer may apply heuristic to deduce it.
     pub num_unit: StringOffset, // 4
 }
 
@@ -21,16 +31,19 @@ impl Value for Label {
 
     fn proto_len(&self) -> u64 {
         Varint::from(self.key).field(1).proto_len()
-            + Varint::from(self.str).field(2).proto_len_small()
-            + Varint::from(self.num).field(3).proto_len_small()
-            + Varint::from(self.num_unit).field(4).proto_len_small()
+            + Varint::from(self.str).field(2).zero_opt().proto_len()
+            + Varint::from(self.num).field(3).zero_opt().proto_len()
+            + Varint::from(self.num_unit).field(4).zero_opt().proto_len()
     }
 
     fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()> {
         Varint::from(self.key).field(1).encode(writer)?;
-        Varint::from(self.str).field(2).encode_small(writer)?;
-        Varint::from(self.num).field(3).encode_small(writer)?;
-        Varint::from(self.num_unit).field(4).encode_small(writer)
+        Varint::from(self.str).field(2).zero_opt().encode(writer)?;
+        Varint::from(self.num).field(3).zero_opt().encode(writer)?;
+        Varint::from(self.num_unit)
+            .field(4)
+            .zero_opt()
+            .encode(writer)
     }
 }
 

datadog-profiling-protobuf/src/lib.rs

@@ -6,6 +6,27 @@
 #![cfg_attr(not(test), deny(clippy::expect_used))]
 #![cfg_attr(not(test), deny(clippy::unimplemented))]
 
+//! This crate implements Protobuf serializers for [`profiles`], including:
+//!
+//! - [Function]
+//! - [Label]
+//! - [Location] and [Line]
+//! - [Mapping]
+//! - [Sample]
+//! - [ValueType]
+//!
+//! There is no serializer for Profile. It would require borrowing a lot of
+//! data, which becomes unwieldy. It also isn't very compatible with writing
+//! a streaming serializer to lower peak memory usage.
+//!
+//! Indices into the string table are represented by [StringOffset], which uses
+//! a 32-bit number. ID fields are still 64-bit, so the user can control their
+//! values, potentially using a 64-bit address for its value.
+//!
+//! The types are generally `#[repr(C)]` so they can be used in FFI one day.
+//!
+//! [`profiles`]: https://github.com/google/pprof/blob/main/proto/profile.proto
+
 mod function;
 mod label;
 mod location;
@@ -29,19 +50,27 @@ pub use varint::*;
 
 use std::io::{self, Write};
 
-/// A tag is a combination of a wire_type, stored in the least significant
-/// three bits, and the field number that is defined in the .proto file.
-#[derive(Copy, Clone)]
-pub struct Tag(u32);
+/// Represents the wire type for the in-wire protobuf encoding. There are more
+/// types than are represented here; these are just the supported ones.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[repr(u8)]
+pub enum WireType {
+    Varint = 0,
+    LengthDelimited = 2,
+}
 
 /// A value is stored differently depending on the wire_type.
 pub trait Value {
     const WIRE_TYPE: WireType;
 
+    /// The number of bytes it takes to encode this value.
     fn proto_len(&self) -> u64;
 
+    /// Encode the value to the in-wire protobuf format.
     fn encode<W: Write>(&self, writer: &mut W) -> io::Result<()>;
 
+    /// Create a Pair with the given field. The wire type will be added
+    /// implicitly, and will be this type's [Self::WIRE_TYPE].
     #[inline]
     fn field(self, field: u32) -> Pair<Self>
     where
@@ -61,7 +90,7 @@ pub struct Pair<V: Value> {
 }
 
 impl<V: Value> Pair<V> {
-    /// Calculate the size of pair, without using the zero-size optimization.
+    /// Calculate the size of pair, without the zero-size optimization.
     pub fn proto_len(&self) -> u64 {
         let tag = Tag::new(self.field, V::WIRE_TYPE).proto_len();
         let value = self.value.proto_len();
@@ -73,17 +102,31 @@ impl<V: Value> Pair<V> {
         tag + len_prefix + value
     }
 
-    /// Calculate the size of pair, using the zero-size optimization.
-    #[inline]
-    pub fn proto_len_small(&self) -> u64 {
-        if self.value.proto_len() != 0 {
-            self.proto_len()
-        } else {
-            0
-        }
-    }
-
-    /// Encodes into protobuf, without using the zero-size optimization.
+    /// Encodes the pair into protobuf, without the zero-size optimization.
+    ///
+    /// # Examples
+    ///
+    /// Given a message like:
+    ///
+    /// ```protobuf
+    /// message ValueType {
+    ///   int64 type = 1;
+    ///   int64 unit = 2;
+    /// }
+    /// ```
+    ///
+    /// You can encode it like this:
+    ///
+    /// ```
+    /// # use datadog_profiling_protobuf::{Value, Varint};
+    /// # struct ValueType { r#type: i64, unit: i64 }
+    /// # fn main() -> std::io::Result<()> {
+    /// let mut w = Vec::new();
+    /// let value_type = ValueType { r#type: 4, unit: 5 };
+    /// Varint::from(value_type.r#type).field(1).encode(&mut w)?;
+    /// Varint::from(value_type.unit).field(2).encode(&mut w)?;
+    /// # Ok(()) }
+    /// ```
     pub fn encode(&self, writer: &mut impl Write) -> io::Result<()> {
         Tag::new(self.field, V::WIRE_TYPE).encode(writer)?;
         if V::WIRE_TYPE == WireType::LengthDelimited {
@@ -93,19 +136,90 @@ impl<V: Value> Pair<V> {
         self.value.encode(writer)
     }
 
-    /// Encodes into protobuf, using the zero-size optimization.
+    /// Convert the pair into one that will apply the zero-size optimization.
+    ///
+    /// Note that the zero-size optimization should be applied to the field
+    /// consistently in its [Value::proto_len] and [Value::encode] methods.
+    /// If it's done to one, it should be done to the other.
+    #[inline]
+    pub fn zero_opt(self) -> WithZeroOptimization<V> {
+        WithZeroOptimization { pair: self }
+    }
+}
+
+pub struct WithZeroOptimization<V: Value> {
+    pair: Pair<V>,
+}
+
+impl<V: Value> WithZeroOptimization<V> {
+    /// Calculate the size of pair, using the zero-size optimization.
+    #[inline]
+    pub fn proto_len(&self) -> u64 {
+        if self.pair.value.proto_len() != 0 {
+            self.pair.proto_len()
+        } else {
+            0
+        }
+    }
+
+    /// Encodes into protobuf, using the zero-size optimization. Protobuf
+    /// doesn't require fields with values of zero to be present, so to save
+    /// space, they can be omitted them altogether.
+    ///
+    /// # Examples
+    ///
+    /// Label is a great message to demonstrate how the optimization is useful
+    /// because it has multiple optional values:
+    ///
+    /// ```protobuf
+    /// message Label {
+    ///   int64 key = 1;
+    ///
+    ///   // At most one of the following must be present
+    ///   int64 str = 2;
+    ///   int64 num = 3;
+    ///
+    ///   // Should only be present when num is present.
+    ///   int64 num_unit = 4;
+    /// }
+    /// ```
+    ///
+    /// This can be taken advantage of by using `zero_opt`:
+    ///
+    /// ```
+    /// # use datadog_profiling_protobuf::{Value, Varint};
+    /// # struct Label { key: i64, str: i64, num: i64, num_unit: i64 }
+    /// # fn main() -> std::io::Result<()> {
+    /// let mut w = Vec::new();
+    ///
+    /// let label = Label {
+    ///     key: 1,
+    ///     str: 0,
+    ///     num: 4194303,
+    ///     num_unit: 0,
+    /// };
+    ///
+    /// Varint::from(label.key).field(1).zero_opt().encode(&mut w)?;
+    /// Varint::from(label.str).field(2).zero_opt().encode(&mut w)?;
+    /// Varint::from(label.num).field(3).zero_opt().encode(&mut w)?;
+    /// Varint::from(label.num_unit)
+    ///     .field(4)
+    ///     .zero_opt()
+    ///     .encode(&mut w)?;
+    /// # Ok(()) }
+    /// ```
     #[inline]
-    pub fn encode_small(&self, writer: &mut impl Write) -> io::Result<()> {
-        let len = self.value.proto_len();
+    pub fn encode(&self, writer: &mut impl Write) -> io::Result<()> {
+        let len = self.pair.value.proto_len();
         if len == 0 {
             return Ok(());
         }
 
-        Tag::new(self.field, V::WIRE_TYPE).encode(writer)?;
+        Tag::new(self.pair.field, V::WIRE_TYPE).encode(writer)?;
         if V::WIRE_TYPE == WireType::LengthDelimited {
             Varint(len).encode(writer)?;
         }
-        self.value.encode(writer)
+        self.pair.value.encode(writer)
     }
 }
 
@@ -115,14 +229,10 @@ const MIN_FIELD: u32 = 1;
 /// The largest possible protobuf field number.
 const MAX_FIELD: u32 = (1 << 29) - 1;
 
-/// Represents the wire type for in-wire protobuf. There are more types than
-/// are represented here; these are just the supported ones.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-#[repr(u8)]
-pub enum WireType {
-    Varint = 0,
-    LengthDelimited = 2,
-}
+/// A tag is a combination of a wire_type, stored in the least significant
+/// three bits, and the field number that is defined in the .proto file.
+#[derive(Copy, Clone)]
+pub struct Tag(u32);
 
 impl Tag {
     #[cfg_attr(debug_assertions, track_caller)]
@@ -143,6 +253,21 @@ impl Tag {
     }
 }
 
+/// Represents a packed varint. There are other kinds of things which can be
+/// packed in protobuf, but profiles don't currently use them.
+///
+/// # Examples
+///
+/// Packed is generic over `Into<Varint>`, so packed values of i64 and u64 can
+/// both be used.
+///
+/// ```
+/// # use datadog_profiling_protobuf::Packed;
+/// // u64
+/// _ = Packed::new(&[42u64, 67u64]);
+/// // i64
+/// _ = Packed::new(&[42i64, 67i64]);
+/// ```
 pub struct Packed<'a, T: Into<Varint>> {
     values: &'a [T],
 }