feat: Add new histogram metric type (#386)

Author: yinggeh

include/triton/core/tritonserver.h

@@ -64,6 +64,7 @@ struct TRITONSERVER_Server;
 struct TRITONSERVER_ServerOptions;
 struct TRITONSERVER_Metric;
 struct TRITONSERVER_MetricFamily;
+struct TRITONSERVER_MetricArgs;
 
 ///
 /// TRITONSERVER API Version
@@ -91,7 +92,7 @@ struct TRITONSERVER_MetricFamily;
 ///   }
 ///
 #define TRITONSERVER_API_VERSION_MAJOR 1
-#define TRITONSERVER_API_VERSION_MINOR 33
+#define TRITONSERVER_API_VERSION_MINOR 34
 
 /// Get the TRITONBACKEND API version supported by the Triton shared
 /// library. This value can be compared against the
@@ -2615,7 +2616,8 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_ServerInferAsync(
 ///
 typedef enum TRITONSERVER_metrickind_enum {
   TRITONSERVER_METRIC_KIND_COUNTER,
-  TRITONSERVER_METRIC_KIND_GAUGE
+  TRITONSERVER_METRIC_KIND_GAUGE,
+  TRITONSERVER_METRIC_KIND_HISTOGRAM
 } TRITONSERVER_MetricKind;
 
 /// Create a new metric family object. The caller takes ownership of the
@@ -2644,6 +2646,44 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricFamilyNew(
 TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
 TRITONSERVER_MetricFamilyDelete(struct TRITONSERVER_MetricFamily* family);
 
+/// Get the TRITONSERVER_MetricKind of the metric family.
+///
+/// \param family The metric family object to query.
+/// \param kind Returns the TRITONSERVER_MetricKind of metric.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
+TRITONSERVER_GetMetricFamilyKind(
+    struct TRITONSERVER_MetricFamily* family, TRITONSERVER_MetricKind* kind);
+
+/// Create a new metric args object. The caller takes ownership of the
+/// TRITONSERVER_MetricArgs object and must call TRITONSERVER_MetricArgsDelete
+/// to release the object.
+///
+/// \param args Returns the new metric args object.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsNew(
+    struct TRITONSERVER_MetricArgs** args);
+
+/// Set metric args with histogram metric parameter.
+///
+/// \param args The metric args object to set.
+/// \param buckets The array of bucket boundaries for the expected range of
+/// observed values.
+///
+/// \param buckets_count The number of bucket boundaries.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error*
+TRITONSERVER_MetricArgsSetHistogram(
+    struct TRITONSERVER_MetricArgs* args, const double* buckets,
+    const uint64_t buckets_count);
+
+/// Delete a metric args object.
+///
+/// \param args The metric args object.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricArgsDelete(
+    struct TRITONSERVER_MetricArgs* args);
+
 /// Create a new metric object. The caller takes ownership of the
 /// TRITONSERVER_Metric object and must call
 /// TRITONSERVER_MetricDelete to release the object. The caller is also
@@ -2661,6 +2701,28 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricNew(
     struct TRITONSERVER_MetricFamily* family,
     const struct TRITONSERVER_Parameter** labels, const uint64_t label_count);
 
+/// Create a new metric object. The caller takes ownership of the
+/// TRITONSERVER_Metric object and must call
+/// TRITONSERVER_MetricDelete to release the object. The caller is also
+/// responsible for ownership of the labels passed in.
+/// Each label can be deleted immediately after creating the metric with
+/// TRITONSERVER_ParameterDelete if not re-using the labels.
+/// Metric args can be deleted immediately after creating the metric with
+/// TRITONSERVER_MetricArgsDelete if not re-using the metric args.
+///
+/// \param metric Returns the new metric object.
+/// \param family The metric family to add this new metric to.
+/// \param labels The array of labels to associate with this new metric.
+/// \param label_count The number of labels.
+/// \param args Metric args that store additional arguments to construct
+/// particular metric types, e.g. histogram.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricNewWithArgs(
+    struct TRITONSERVER_Metric** metric,
+    struct TRITONSERVER_MetricFamily* family,
+    const struct TRITONSERVER_Parameter** labels, const uint64_t label_count,
+    const struct TRITONSERVER_MetricArgs* args);
+
 /// Delete a metric object.
 /// All TRITONSERVER_Metric* objects should be deleted BEFORE their
 /// corresponding TRITONSERVER_MetricFamily* objects have been deleted.
@@ -2705,7 +2767,17 @@ TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricIncrement(
 TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricSet(
     struct TRITONSERVER_Metric* metric, double value);
 
-/// Get the TRITONSERVER_MetricKind of metric and its corresponding family.
+/// Sample an observation and count it to the appropriate bucket of a metric.
+/// Supports metrics of kind TRITONSERVER_METRIC_KIND_HISTOGRAM and returns
+/// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind.
+///
+/// \param metric The metric object to update.
+/// \param value The amount for metric to sample observation.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC struct TRITONSERVER_Error* TRITONSERVER_MetricObserve(
+    struct TRITONSERVER_Metric* metric, double value);
+
+/// Get the TRITONSERVER_MetricKind of metric of its corresponding family.
 ///
 /// \param metric The metric object to query.
 /// \param kind Returns the TRITONSERVER_MetricKind of metric.

src/metric_family.cc

@@ -1,4 +1,5 @@
-// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights
+// reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -54,6 +55,12 @@ MetricFamily::MetricFamily(
                                              .Help(description)
                                              .Register(*registry));
       break;
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM:
+      family_ = reinterpret_cast<void*>(&prometheus::BuildHistogram()
+                                             .Name(name)
+                                             .Help(description)
+                                             .Register(*registry));
+      break;
     default:
       throw std::invalid_argument(
           "Unsupported kind passed to MetricFamily constructor.");
@@ -63,24 +70,49 @@ MetricFamily::MetricFamily(
 }
 
 void*
-MetricFamily::Add(std::map<std::string, std::string> label_map, Metric* metric)
+MetricFamily::Add(
+    std::map<std::string, std::string> label_map, Metric* metric,
+    const TritonServerMetricArgs* args)
 {
   void* prom_metric = nullptr;
   switch (kind_) {
     case TRITONSERVER_METRIC_KIND_COUNTER: {
+      if (args != nullptr) {
+        throw std::invalid_argument(
+            "Unexpected args found in counter Metric constructor.");
+      }
       auto counter_family_ptr =
           reinterpret_cast<prometheus::Family<prometheus::Counter>*>(family_);
       auto counter_ptr = &counter_family_ptr->Add(label_map);
       prom_metric = reinterpret_cast<void*>(counter_ptr);
       break;
     }
     case TRITONSERVER_METRIC_KIND_GAUGE: {
+      if (args != nullptr) {
+        throw std::invalid_argument(
+            "Unexpected args found in gauge Metric constructor.");
+      }
       auto gauge_family_ptr =
           reinterpret_cast<prometheus::Family<prometheus::Gauge>*>(family_);
       auto gauge_ptr = &gauge_family_ptr->Add(label_map);
       prom_metric = reinterpret_cast<void*>(gauge_ptr);
       break;
     }
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
+      if (args == nullptr) {
+        throw std::invalid_argument(
+            "Bucket boundaries not found in Metric args.");
+      }
+      if (args->kind() != TRITONSERVER_METRIC_KIND_HISTOGRAM) {
+        throw std::invalid_argument("Metric args not set to histogram kind.");
+      }
+      auto histogram_family_ptr =
+          reinterpret_cast<prometheus::Family<prometheus::Histogram>*>(family_);
+      auto histogram_ptr =
+          &histogram_family_ptr->Add(label_map, args->buckets());
+      prom_metric = reinterpret_cast<void*>(histogram_ptr);
+      break;
+    }
     default:
       throw std::invalid_argument(
           "Unsupported family kind passed to Metric constructor.");
@@ -134,6 +166,14 @@ MetricFamily::Remove(void* prom_metric, Metric* metric)
       gauge_family_ptr->Remove(gauge_ptr);
       break;
     }
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
+      auto histogram_family_ptr =
+          reinterpret_cast<prometheus::Family<prometheus::Histogram>*>(family_);
+      auto histogram_ptr =
+          reinterpret_cast<prometheus::Histogram*>(prom_metric);
+      histogram_family_ptr->Remove(histogram_ptr);
+      break;
+    }
     default:
       // Invalid kind should be caught in constructor
       LOG_ERROR << "Unsupported kind in Metric destructor.";
@@ -169,7 +209,8 @@ MetricFamily::~MetricFamily()
 //
 Metric::Metric(
     TRITONSERVER_MetricFamily* family,
-    std::vector<const InferenceParameter*> labels)
+    std::vector<const InferenceParameter*> labels,
+    const TritonServerMetricArgs* args)
 {
   family_ = reinterpret_cast<MetricFamily*>(family);
   kind_ = family_->Kind();
@@ -188,7 +229,7 @@ Metric::Metric(
         std::string(reinterpret_cast<const char*>(param->ValuePointer()));
   }
 
-  metric_ = family_->Add(label_map, this);
+  metric_ = family_->Add(label_map, this, args);
 }
 
 Metric::~Metric()
@@ -235,6 +276,11 @@ Metric::Value(double* value)
       *value = gauge_ptr->Value();
       break;
     }
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
+      return TRITONSERVER_ErrorNew(
+          TRITONSERVER_ERROR_UNSUPPORTED,
+          "TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Value");
+    }
     default:
       return TRITONSERVER_ErrorNew(
           TRITONSERVER_ERROR_UNSUPPORTED,
@@ -279,6 +325,11 @@ Metric::Increment(double value)
       }
       break;
     }
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
+      return TRITONSERVER_ErrorNew(
+          TRITONSERVER_ERROR_UNSUPPORTED,
+          "TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Increment");
+    }
     default:
       return TRITONSERVER_ErrorNew(
           TRITONSERVER_ERROR_UNSUPPORTED,
@@ -308,6 +359,45 @@ Metric::Set(double value)
       gauge_ptr->Set(value);
       break;
     }
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
+      return TRITONSERVER_ErrorNew(
+          TRITONSERVER_ERROR_UNSUPPORTED,
+          "TRITONSERVER_METRIC_KIND_HISTOGRAM does not support Set");
+    }
+    default:
+      return TRITONSERVER_ErrorNew(
+          TRITONSERVER_ERROR_UNSUPPORTED,
+          "Unsupported TRITONSERVER_MetricKind");
+  }
+
+  return nullptr;  // Success
+}
+
+TRITONSERVER_Error*
+Metric::Observe(double value)
+{
+  if (metric_ == nullptr) {
+    return TRITONSERVER_ErrorNew(
+        TRITONSERVER_ERROR_INTERNAL,
+        "Could not set metric value. Metric has been invalidated.");
+  }
+
+  switch (kind_) {
+    case TRITONSERVER_METRIC_KIND_COUNTER: {
+      return TRITONSERVER_ErrorNew(
+          TRITONSERVER_ERROR_UNSUPPORTED,
+          "TRITONSERVER_METRIC_KIND_COUNTER does not support Observe");
+    }
+    case TRITONSERVER_METRIC_KIND_GAUGE: {
+      return TRITONSERVER_ErrorNew(
+          TRITONSERVER_ERROR_UNSUPPORTED,
+          "TRITONSERVER_METRIC_KIND_GAUGE does not support Observe");
+    }
+    case TRITONSERVER_METRIC_KIND_HISTOGRAM: {
+      auto histogram_ptr = reinterpret_cast<prometheus::Histogram*>(metric_);
+      histogram_ptr->Observe(value);
+      break;
+    }
     default:
       return TRITONSERVER_ErrorNew(
           TRITONSERVER_ERROR_UNSUPPORTED,

src/metric_family.h

@@ -1,4 +1,5 @@
-// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights
+// reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -27,6 +28,7 @@
 
 #ifdef TRITON_ENABLE_METRICS
 
+#include <cstring>
 #include <mutex>
 #include <set>
 #include <unordered_map>
@@ -37,6 +39,29 @@
 
 namespace triton { namespace core {
 
+//
+// TritonServerMetricArgs
+//
+// Implementation for TRITONSERVER_MetricArgs.
+//
+class TritonServerMetricArgs {
+ public:
+  TritonServerMetricArgs() = default;
+
+  void* SetHistogramArgs(const double* buckets, uint64_t bucket_count)
+  {
+    kind_ = TRITONSERVER_METRIC_KIND_HISTOGRAM;
+    buckets_ = std::vector<double>(buckets, buckets + bucket_count);
+    return nullptr;
+  }
+  TRITONSERVER_MetricKind kind() const { return kind_; }
+  const std::vector<double>& buckets() const { return buckets_; }
+
+ private:
+  TRITONSERVER_MetricKind kind_;
+  std::vector<double> buckets_;
+};
+
 //
 // Implementation for TRITONSERVER_MetricFamily.
 //
@@ -50,7 +75,9 @@ class MetricFamily {
   void* Family() const { return family_; }
   TRITONSERVER_MetricKind Kind() const { return kind_; }
 
-  void* Add(std::map<std::string, std::string> label_map, Metric* metric);
+  void* Add(
+      std::map<std::string, std::string> label_map, Metric* metric,
+      const TritonServerMetricArgs* args);
   void Remove(void* prom_metric, Metric* metric);
 
   int NumMetrics()
@@ -86,7 +113,8 @@ class Metric {
  public:
   Metric(
       TRITONSERVER_MetricFamily* family,
-      std::vector<const InferenceParameter*> labels);
+      std::vector<const InferenceParameter*> labels,
+      const TritonServerMetricArgs* args);
   ~Metric();
 
   MetricFamily* Family() const { return family_; }
@@ -95,6 +123,7 @@ class Metric {
   TRITONSERVER_Error* Value(double* value);
   TRITONSERVER_Error* Increment(double value);
   TRITONSERVER_Error* Set(double value);
+  TRITONSERVER_Error* Observe(double value);
 
   // If a MetricFamily is deleted before its dependent Metric, we want to
   // invalidate the references so we don't access invalid memory.

src/metrics.h

@@ -1,4 +1,4 @@
-// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -35,6 +35,7 @@
 
 #include "prometheus/counter.h"
 #include "prometheus/gauge.h"
+#include "prometheus/histogram.h"
 #include "prometheus/registry.h"
 #include "prometheus/serializer.h"
 #include "prometheus/summary.h"