Feat/telemetry grouped counter (#7)

* feat: add grouped counter to telemetry

* chore: bump to 0.3.0

* feat: replce string with constants

* feat: manage no registered metrics

* chore: update packages

* feat: rename test

* chore: bump to 0.3.0-beta-1

* feat: replce string with constants

Author: codeflyer

README.md

@@ -0,0 +1,101 @@
+# e-ipfs-core-lib
+
+A library that collect utils and shared code for e-ipfs ecosystem.
+
+## Quickstart
+
+```
+npm i e-ipfs-core-lib
+```
+
+## Api
+
+### Telemetry
+#### Telemetry
+
+Create a telemetry manager that provide a [`prometheus`](https://github.com/prometheus) valid file [format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md) 
+
+Create metrics yaml configuration file:
+
+```yaml
+---
+component: bitswap-peer
+metrics:
+  bitswap-total-connections: BitSwap Total Connections
+  s3-request: AWS S3 requests
+  dynamo-request: AWS DynamoDB requests
+version: 0.1.0
+buildDate: "20220307.1423"
+```
+
+
+```javascript
+import path from 'path'
+import { Telemetry, dirname } from 'e-ipfs-core-lib'
+
+const configFile = path.join(dirname(import.meta.url), '../metrics.yml')
+const telemetry = new Telemetry({ configFile })
+telemetry.increaseCount('bitswap-total-connections', 2)
+
+const result = telemetry.export()
+console.log(result)
+```
+
+#### Telemetry instance methods
+* clear: Clear the metrics
+* createMetric(category, description, metric, type): Create a new metric
+  * category: String - The given name of the category
+  * description: String - The category description
+  * metric: METRIC_COUNT | METRIC_DURATIONS | METRIC_GROUPED_COUNT - The metric defined
+  * type: null | TYPE_GAUGE - The type of the metric. If not passed the value is defined based on the `metric` attribute. 
+* export: Export the metrics in `prometheus` format
+  ```
+    # HELP counter_grouped_count_total COUNTER (grouped-count)
+    # TYPE counter_grouped_count_total counter
+    counter_grouped_count_total{id="123"} 1 now
+    counter_grouped_count_total{id="456"} 2 now
+    ```
+* increaseCount(category, amount = 1): Increase the count for a category
+  * category: String - The given name of the category
+  * amount: Number (Default 1) - The amount to add to the metric
+* decreaseCount(category, amount = 1): Decrease the count for a category
+  * category: String - The given name of the category
+  * amount: Number (Default 1) - The amount to remove from the metric
+* increaseCountWithKey(category, key, amount = 1): Increase the count for a key in a category
+  * category: String - The given name of the category
+  * key: String - The key of the metric
+  * amount: Number (Default 1) - The amount to add to the metric
+* decreaseCountWithKey(category, key, amount = 1): Decrease the count for a key in a category
+  * category: String - The given name of the category
+  * key: String - The key of the metric
+  * amount: Number (Default 1) - The amount to remove from the metric
+* trackDuration(category, promise): Track the duration of an async call
+  * category: String - The given name of the category
+  * promise: Promise - The function to be tracked
+
+#### Metrics and types constants
+
+The constants below are exported
+
+* METRIC_COUNT
+* METRIC_DURATIONS
+* METRIC_GROUPED_COUNT
+* TYPE_COUNTER
+* TYPE_HISTOGRAM
+* TYPE_GAUGE
+
+### Utils
+#### dirname
+#### version
+#### cidToKey
+
+### AWS-Client
+#### createAwsClient,
+#### awsClientOptions,
+#### Client
+
+### Logger
+#### createLogger
+
+### Protocol
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "e-ipfs-core-lib",
-  "version": "0.2.2",
+  "version": "0.3.0-beta-1",
   "description": "E-IPFS core library",
   "license": "(Apache-2.0 AND MIT)",
   "homepage": "https://github.com/elastic-ipfs/core-lib",
@@ -14,7 +14,7 @@
     "node": "18.12"
   },
   "dependencies": {
-    "@aws-sdk/util-dynamodb": "^3.216.0",
+    "@aws-sdk/util-dynamodb": "^3.218.0",
     "bl": "^6.0.0",
     "hdr-histogram-js": "^3.0.0",
     "js-yaml": "^4.1.0",
@@ -24,7 +24,7 @@
     "piscina": "^3.2.0",
     "protobufjs": "^7.1.2",
     "sodium-native": "^3.4.1",
-    "undici": "^5.12.0",
+    "undici": "^5.13.0",
     "xml-js": "^1.6.11"
   },
   "devDependencies": {

src/telemetry.js

@@ -1,10 +1,17 @@
-
 import { readFileSync } from 'fs'
 import { load as ymlLoad } from 'js-yaml'
 import * as hdr from 'hdr-histogram-js'
 
 const PERCENTILES = [0.001, 0.01, 0.1, 1, 2.5, 10, 25, 50, 75, 90, 97.5, 99, 99.9, 99.99, 99.999]
 
+const METRIC_GROUPED_COUNT = 'grouped-count'
+const METRIC_DURATIONS = 'durations'
+const METRIC_COUNT = 'count'
+
+const TYPE_COUNTER = 'counter'
+const TYPE_HISTOGRAM = 'histogram'
+const TYPE_GAUGE = 'gauge'
+
 class Aggregator {
   constructor (category, description, metric, type) {
     this.tag = `${category}-${metric}`
@@ -14,16 +21,18 @@ class Aggregator {
     // type is optional
     if (!type) {
       // set the type by the metric
-      if (metric === 'durations') {
-        this.type = 'histogram'
+      if (metric === METRIC_DURATIONS) {
+        this.type = TYPE_HISTOGRAM
       } else {
-        this.type = 'counter'
+        this.type = TYPE_COUNTER
         this.exportName += '_total'
+        this.isGrouped = metric === METRIC_GROUPED_COUNT
       }
     } else {
       this.type = type
     }
 
+    this.groupedSum = {}
     this.sum = 0
     hdr.initWebAssemblySync()
     this.histogram = hdr.build({
@@ -37,22 +46,35 @@ class Aggregator {
   record (value) {
     this.sum += value
 
-    if (this.type === 'histogram') {
+    if (this.type === TYPE_HISTOGRAM) {
       this.histogram.recordValue(value)
     }
   }
 
+  recordWithKey (key, value) {
+    if (!this.groupedSum[key]) {
+      this.groupedSum[key] = value
+    } else {
+      this.groupedSum[key] += value
+    }
+  }
+
   reset () {
     this.sum = 0
+    this.groupedSum = {}
     this.histogram.reset()
   }
 
   current () {
     const { minNonZeroValue: min, maxValue: max, mean, stdDeviation: stdDev, totalCount: count } = this.histogram
 
     const value = {
-      empty: (this.type === 'histogram' && count === 0) || (this.type === 'counter' && this.sum === 0),
+      empty: (this.type === TYPE_HISTOGRAM && count === 0) ||
+        ((this.type === TYPE_COUNTER && !this.isGrouped) && this.sum === 0) ||
+        ((this.type === TYPE_COUNTER && this.isGrouped) && Object.keys(this.groupedSum).length === 0),
       sum: this.sum,
+      isGrouped: this.isGrouped,
+      groupedSum: this.groupedSum,
       histogram:
         count > 0
           ? {
@@ -96,8 +118,9 @@ class Telemetry {
       // Create metrics
       this.metrics = new Map()
       for (const [category, description] of Object.entries(metrics)) {
-        this.createMetric(category, description, 'count')
-        this.createMetric(category, description, 'durations')
+        this.createMetric(category, description, METRIC_COUNT)
+        this.createMetric(category, description, METRIC_GROUPED_COUNT)
+        this.createMetric(category, description, METRIC_DURATIONS)
       }
     } catch (err) {
       logger.error({ err }, 'error in telemetry constructor')
@@ -138,14 +161,18 @@ class Telemetry {
       output += `# HELP ${metric.exportName} ${metric.description}\n`
       output += `# TYPE ${metric.exportName} ${metric.type}\n`
 
-      if (metric.type === 'histogram') {
+      if (metric.type === TYPE_HISTOGRAM) {
         output += `${metric.exportName}_count ${current.histogram.count} ${current.timestamp}\n`
         output += `${metric.exportName}_sum ${current.sum} ${current.timestamp}\n`
 
         const percentilesValues = current.histogram.percentiles
         for (const percentile of PERCENTILES) {
           output += `${metric.exportName}_bucket{le="${percentile}"} ${percentilesValues[percentile]} ${current.timestamp}\n`
         }
+      } else if (metric.type === TYPE_COUNTER && metric.isGrouped) {
+        for (const [key, value] of Object.entries(current.groupedSum)) {
+          output += `${metric.exportName}${key} ${value} ${current.timestamp}\n`
+        }
       } else {
         output += `${metric.exportName} ${current.sum} ${current.timestamp}\n`
       }
@@ -159,17 +186,27 @@ class Telemetry {
   }
 
   increaseCount (category, amount = 1) {
-    const metric = this.ensureMetric(category, 'count')
+    const metric = this.ensureMetric(category, METRIC_COUNT)
     metric.record(amount)
   }
 
   decreaseCount (category, amount = 1) {
-    const metric = this.ensureMetric(category, 'count')
+    const metric = this.ensureMetric(category, METRIC_COUNT)
     metric.record(-1 * amount)
   }
 
+  increaseCountWithKey (category, key, amount = 1) {
+    const metric = this.ensureMetric(category, METRIC_GROUPED_COUNT)
+    metric.recordWithKey(key, amount)
+  }
+
+  decreaseCountWithKey (category, key, amount = 1) {
+    const metric = this.ensureMetric(category, METRIC_GROUPED_COUNT)
+    metric.recordWithKey(key, -1 * amount)
+  }
+
   async trackDuration (category, promise) {
-    const metric = this.ensureMetric(category, 'durations')
+    const metric = this.ensureMetric(category, METRIC_DURATIONS)
     const startTime = process.hrtime.bigint()
 
     try {
@@ -180,4 +217,4 @@ class Telemetry {
   }
 }
 
-export { Telemetry }
+export { Telemetry, METRIC_COUNT, METRIC_DURATIONS, METRIC_GROUPED_COUNT, TYPE_COUNTER, TYPE_HISTOGRAM, TYPE_GAUGE }