Kafka dashboard fix fields (elastic#4530)

lalit-satapathy · web-flow · commit 347df35fd6ca · 2022-12-02T10:21:05.000+05:30
* Kafka dashboard fix fields.
* Remove deprecated fields from integration also, as done per beats#27938
diff --git a/packages/kafka/changelog.yml b/packages/kafka/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.1"
+  changes:
+    - description: Update the datashoboard fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4530
 - version: "1.3.0"
   changes:
     - description: Added infrastructure category.
diff --git a/packages/kafka/data_stream/consumergroup/fields/fields.yml b/packages/kafka/data_stream/consumergroup/fields/fields.yml
@@ -1,26 +1,9 @@
 - name: kafka.consumergroup
   type: group
   fields:
-    - name: broker
-      type: group
-      fields:
-        - name: id
-          type: long
-          description: |
-            Broker id
-        - name: address
-          type: keyword
-          description: |
-            Broker address
     - name: id
       type: keyword
       description: Consumer Group ID
-    - name: topic
-      type: keyword
-      description: Topic name
-    - name: partition
-      type: long
-      description: Partition ID
     - name: offset
       type: long
       description: consumer offset into partition being read
diff --git a/packages/kafka/data_stream/consumergroup/sample_event.json b/packages/kafka/data_stream/consumergroup/sample_event.json
@@ -12,14 +12,9 @@
     },
     "kafka": {
         "consumergroup": {
-            "topic": "messages",
             "error": {
                 "code": 0
             },
-            "broker": {
-                "id": 0,
-                "address": "kafka-01:9092"
-            },
             "id": "console-consumer-99447",
             "offset": -1,
             "consumer_lag": 112,
@@ -28,7 +23,6 @@
                 "id": "consumer-console-consumer-99447-1",
                 "host": "127.0.0.1"
             },
-            "partition": 0,
             "meta": ""
         },
         "broker": {
diff --git a/packages/kafka/data_stream/partition/fields/fields.yml b/packages/kafka/data_stream/partition/fields/fields.yml
@@ -15,18 +15,10 @@
     - name: partition
       type: group
       fields:
-        - name: id
-          type: long
-          description: |
-            Partition id.
         - name: leader
           type: long
           description: |
             Leader id (broker).
-        - name: isr
-          type: keyword
-          description: |
-            List of isr ids.
         - name: replica
           type: long
           description: |
@@ -42,20 +34,4 @@
         - name: error.code
           type: long
           description: |
-            Error code from fetching partition.
-    - name: topic.error.code
-      type: long
-      description: |
-        topic error code.
-    - name: topic.name
-      type: keyword
-      description: |
-        Topic name
-    - name: broker.id
-      type: long
-      description: |
-        Broker id
-    - name: broker.address
-      type: keyword
-      description: |
-        Broker address
+            Error code from fetching partition.
diff --git a/packages/kafka/data_stream/partition/sample_event.json b/packages/kafka/data_stream/partition/sample_event.json
@@ -17,17 +17,9 @@
             "id": 0,
             "topic_id": "0-messages",
             "topic_broker_id": "0-messages-0",
-            "topic": {
-                "name": "messages"
-            },
-            "broker": {
-                "id": 0,
-                "address": "kafka-01:9092"
-            },
             "partition": {
                 "is_leader": true,
                 "insync_replica": true,
-                "id": 0,
                 "leader": 0,
                 "replica": 0
             }
diff --git a/packages/kafka/docs/README.md b/packages/kafka/docs/README.md
@@ -216,14 +216,9 @@ An example event for `consumergroup` looks as following:
     },
     "kafka": {
         "consumergroup": {
-            "topic": "messages",
             "error": {
                 "code": 0
             },
-            "broker": {
-                "id": 0,
-                "address": "kafka-01:9092"
-            },
             "id": "console-consumer-99447",
             "offset": -1,
             "consumer_lag": 112,
@@ -232,7 +227,6 @@ An example event for `consumergroup` looks as following:
                 "id": "consumer-console-consumer-99447-1",
                 "host": "127.0.0.1"
             },
-            "partition": 0,
             "meta": ""
         },
         "broker": {
@@ -306,8 +300,6 @@ An example event for `consumergroup` looks as following:
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
 | kafka.broker.address | Broker advertised address | keyword |
 | kafka.broker.id | Broker id | long |
-| kafka.consumergroup.broker.address | Broker address | keyword |
-| kafka.consumergroup.broker.id | Broker id | long |
 | kafka.consumergroup.client.host | Client host | keyword |
 | kafka.consumergroup.client.id | Client ID (kafka setting client.id) | keyword |
 | kafka.consumergroup.client.member_id | internal consumer group member ID | keyword |
@@ -316,8 +308,6 @@ An example event for `consumergroup` looks as following:
 | kafka.consumergroup.id | Consumer Group ID | keyword |
 | kafka.consumergroup.meta | custom consumer meta data string | keyword |
 | kafka.consumergroup.offset | consumer offset into partition being read | long |
-| kafka.consumergroup.partition | Partition ID | long |
-| kafka.consumergroup.topic | Topic name | keyword |
 | kafka.partition.id | Partition id. | long |
 | kafka.partition.topic_broker_id | Unique id of the partition in the topic and the broker. | keyword |
 | kafka.partition.topic_id | Unique id of the partition in the topic. | keyword |
@@ -351,17 +341,9 @@ An example event for `partition` looks as following:
             "id": 0,
             "topic_id": "0-messages",
             "topic_broker_id": "0-messages-0",
-            "topic": {
-                "name": "messages"
-            },
-            "broker": {
-                "id": 0,
-                "address": "kafka-01:9092"
-            },
             "partition": {
                 "is_leader": true,
                 "insync_replica": true,
-                "id": 0,
                 "leader": 0,
                 "replica": 0
             }
@@ -436,20 +418,14 @@ An example event for `partition` looks as following:
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
 | kafka.broker.address | Broker advertised address | keyword |
 | kafka.broker.id | Broker id | long |
-| kafka.partition.broker.address | Broker address | keyword |
-| kafka.partition.broker.id | Broker id | long |
 | kafka.partition.id | Partition id. | long |
 | kafka.partition.offset.newest | Newest offset of the partition. | long |
 | kafka.partition.offset.oldest | Oldest offset of the partition. | long |
 | kafka.partition.partition.error.code | Error code from fetching partition. | long |
-| kafka.partition.partition.id | Partition id. | long |
 | kafka.partition.partition.insync_replica | Indicates if replica is included in the in-sync replicate set (ISR). | boolean |
 | kafka.partition.partition.is_leader | Indicates if replica is the leader | boolean |
-| kafka.partition.partition.isr | List of isr ids. | keyword |
 | kafka.partition.partition.leader | Leader id (broker). | long |
 | kafka.partition.partition.replica | Replica id (broker). | long |
-| kafka.partition.topic.error.code | topic error code. | long |
-| kafka.partition.topic.name | Topic name | keyword |
 | kafka.partition.topic_broker_id | Unique id of the partition in the topic and the broker. | keyword |
 | kafka.partition.topic_id | Unique id of the partition in the topic. | keyword |
 | kafka.topic.error.code | Topic error code. | long |
diff --git a/packages/kafka/kibana/visualization/kafka-587f2360-8f21-11e8-8fa2-3d5f811fbd0f.json b/packages/kafka/kibana/visualization/kafka-587f2360-8f21-11e8-8fa2-3d5f811fbd0f.json
@@ -34,7 +34,7 @@
                         "line_width": "1",
                         "metrics": [
                             {
-                                "field": "kafka.consumergroup.partition",
+                                "field": "kafka.partition.id",
                                 "id": "61ca57f2-469d-11e7-af02-69e470af7417",
                                 "type": "sum"
                             },
diff --git a/packages/kafka/kibana/visualization/kafka-944188f0-8e79-11e8-8fa2-3d5f811fbd0f.json b/packages/kafka/kibana/visualization/kafka-944188f0-8e79-11e8-8fa2-3d5f811fbd0f.json
@@ -33,7 +33,7 @@
                 "default_timefield": "@timestamp",
                 "filter": {
                     "language": "lucene",
-                    "query": "(metricset.name: consumergroup) AND NOT kafka.consumergroup.topic.name:__consumer_offsets"
+                    "query": "(metricset.name: consumergroup) AND NOT kafka.topic.name:__consumer_offsets"
                 },
                 "gauge_color_rules": [
                     {
@@ -69,7 +69,7 @@
                         "split_color_mode": "rainbow",
                         "split_mode": "terms",
                         "stacked": "none",
-                        "terms_field": "kafka.consumergroup.topic",
+                        "terms_field": "kafka.topic.name",
                         "type": "timeseries"
                     }
                 ],
diff --git a/packages/kafka/kibana/visualization/kafka-dc89f8d0-8e8e-11e8-8fa2-3d5f811fbd0f.json b/packages/kafka/kibana/visualization/kafka-dc89f8d0-8e8e-11e8-8fa2-3d5f811fbd0f.json
@@ -84,7 +84,7 @@
                     "id": "3",
                     "params": {
                         "customLabel": "Brokers",
-                        "field": "kafka.partition.broker.id"
+                        "field": "kafka.broker.id"
                     },
                     "schema": "metric",
                     "type": "cardinality"
diff --git a/packages/kafka/manifest.yml b/packages/kafka/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: kafka
 title: Kafka
-version: 1.3.0
+version: 1.3.1
 license: basic
 description: Collect logs and metrics from Kafka servers with Elastic Agent.
 type: integration

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@`
`34`	`34`	`"line_width": "1",`
`35`	`35`	`"metrics": [`
`36`	`36`	`{`
`37`		`- "field": "kafka.consumergroup.partition",`
	`37`	`+ "field": "kafka.partition.id",`
`38`	`38`	`"id": "61ca57f2-469d-11e7-af02-69e470af7417",`
`39`	`39`	`"type": "sum"`
`40`	`40`	`},`