Add support of fluentd metrics export (#10)

Author: Watson1978

Gemfile

@@ -5,11 +5,13 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in fluent-plugin-opentelemetry.gemspec
 gemspec
 
-gem "grpc", "~> 1.73"
+gem "grpc"
 gem "grpc-tools"
 gem "irb"
-gem "rake", "~> 13.0"
-gem "test-unit", "~> 3.0"
+gem "rake"
+gem "rr"
+gem "test-unit"
+gem "test-unit-rr"
 gem "timecop"
 
 gem "rubocop", "~> 1.75"

README.md

@@ -24,7 +24,7 @@ $ bundle
 
 ## Configuration
 
-### Input plugin
+### Input `opentelemetry` plugin
 
 To receive data, this plugin requires `<http>` or `<grpc>` section, or both.
 
@@ -106,7 +106,40 @@ Refer [Config: Transport Section](https://docs.fluentd.org/configuration/transpo
 </source>
 ```
 
-### Output plugin
+### Input `opentelemetry_metrics` plugin
+
+This plugin emits Fluentd's metric data that conforms to the OpenTelemetry Protocol.
+To output the data, it requires to use output `opentelemetry` plugin.
+
+#### Root section
+
+| parameter          | type   | description                                           | default     |
+|--------------------|--------|-------------------------------------------------------|-------------|
+| tag                | string | The tag of the event                                  | required    |
+| emit_interval      | time   | Determine the rate to emit internal metrics as events | `60`        |
+| metric_name_prefix | string | The prefix of metric name                             | `fluentd_`  |
+
+#### Example
+
+```
+# Emit Fluentd metrics
+<source>
+  @type opentelemetry_metrics
+  tag opentelemetry.fluentd.metrics
+  emit_interval 300s
+</source>
+
+# Send Fluentd metrics to OpenTelemetry Collector
+<match opentelemetry.fluentd.metrics>
+  @type opentelemetry
+
+  <http>
+    endpoint "https://127.0.0.1:4318"
+  </http>
+</match>
+```
+
+### Output `opentelemetry` plugin
 
 To send data, this plugin requires `<http>` or `<grpc>` section.
 

example/README.md

@@ -12,12 +12,12 @@ The telemetry data have been provided by js application in `app` directory.
 
 ```mermaid
 flowchart LR
-    A[app/metrics.js] -->|metric data / otlp| C[Fluentd]
-    B[app/tracing.js] -->|trace data / otlp| C[Fluentd]
-    C[Fluentd] -->|metric data / otlp| D[Otel Collector]
-    C[Fluentd] -->|trace data / otlp| F[Jaeger]
+    A[app/metrics.js] -->|app metric data / otlp| C[Fluentd]
+    B[app/tracing.js] -->|app trace data / otlp| C[Fluentd]
+    C[Fluentd] -->|app metric data / otlp <br> Fluentd metric data / otlp| D[Otel Collector]
+    C[Fluentd] -->|app trace data / otlp| F[Jaeger]
     C[Fluentd] -->|sample log / Fluentd Forward Protocol| D[Otel Collector]
-    D[Otel Collector] -->|metric data| E[Prometheus / Grafana]
+    D[Otel Collector] -->|app metric data <br> Fluentd metric data | E[Prometheus / Grafana]
     D[Otel Collector] -->|sample log| G[Elasticsearch / Kibana]
 ```
 
@@ -31,7 +31,7 @@ This demo uses Prometheus / Grafana to visualize the metrics data.
 
 1. Go to [`localhost:3000`](http://localhost:3000) and login to Grafana using credentials admin, admin
 
-2. Then, show `demo > Basic Service Trace` in [dashboard](http://localhost:3000/dashboards).
+2. Then, show `demo > OpenTelemetry` in [dashboard](http://localhost:3000/dashboards).
 
 ![dashboard](./assets/dashboard.png)
 
@@ -54,7 +54,7 @@ This demo uses Elasticsearch / Kibana to visualize the trace data.
 
 ![discover](./assets/discover.png)
 
-2. Specify `logs-*` to Index pattern and click Save data view to Kibana. 
+2. Specify `logs-*` to Index pattern and click Save data view to Kibana.
 
 ![create data view](./assets/create_data_view.png)
 

example/assets/dashboard.png

@@ -14,6 +14,12 @@
   </http>
 </source>
 
+<source>
+  @type opentelemetry_metrics
+  tag opentelemetry.fluentd.metrics
+  emit_interval 15s
+</source>
+
 # Send traces data to jaeger directly using OpenTelemetry Protocol
 <match opentelemetry.traces>
   @type opentelemetry
@@ -71,4 +77,4 @@
     port 24224
     weight 60
   </server>
-</match>
+</match>

example/fluentd/fluent.conf

@@ -24,7 +24,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 2,
+  "id": 3,
   "links": [],
   "panels": [
     {
@@ -61,6 +61,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -76,7 +77,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -107,7 +108,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "11.5.2",
+      "pluginVersion": "12.2.0",
       "targets": [
         {
           "datasource": {
@@ -126,13 +127,107 @@
           "useBackend": false
         }
       ],
-      "title": "Basic Service",
+      "title": "Application metrics",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.2.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "{__name__=~\"^fluentd_.*\"}",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fluentd metrics",
       "type": "timeseries"
     }
   ],
   "preload": false,
   "refresh": "5s",
-  "schemaVersion": 40,
+  "schemaVersion": 42,
   "tags": [],
   "templating": {
     "list": []
@@ -143,8 +238,7 @@
   },
   "timepicker": {},
   "timezone": "",
-  "title": "Basic Service Traces",
+  "title": "OpenTelemetry",
   "uid": "y36s8FHVz",
-  "version": 1,
-  "weekStart": ""
-}
+  "version": 2
+}

example/grafana/dashboard-files/Basic_Services_Traces.json renamed to example/grafana/dashboard-files/OpenTelemetry.json

@@ -1,5 +1,5 @@
 global:
-  scrape_interval: 15s 
+  scrape_interval: 15s
   evaluation_interval: 15s
 
 scrape_configs:

example/prometheus/prometheus.yml

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require_relative "lib/fluent/plugin/opentelemetry/version"
+
 Gem::Specification.new do |spec|
   spec.name = "fluent-plugin-opentelemetry"
-  spec.version = "0.3.0"
+  spec.version = Fluent::Plugin::Opentelemetry::VERSION
   spec.authors = ["Shizuo Fujita"]
   spec.email = ["[email protected]"]
 

fluent-plugin-opentelemetry.gemspec

@@ -4,7 +4,9 @@ source "https://rubygems.org"
 
 gemspec path: "../"
 
-gem "grpc", "~> 1.73"
-gem "rake", "~> 13.0"
-gem "test-unit", "~> 3.0"
+gem "grpc"
+gem "rake"
+gem "rr"
+gem "test-unit"
+gem "test-unit-rr"
 gem "timecop"

gemfiles/grpc.gemfile

@@ -4,7 +4,9 @@ source "https://rubygems.org"
 
 gemspec path: "../"
 
-# gem "grpc", "~> 1.73"
-gem "rake", "~> 13.0"
-gem "test-unit", "~> 3.0"
+# gem "grpc"
+gem "rake"
+gem "rr"
+gem "test-unit"
+gem "test-unit-rr"
 gem "timecop"