Fixed ElapsedTime value. Added elasticsearch index template. Added Kibana visualization.

Author: rahulsinghai

docs/Elasticsearch_jmeter_metrics_template.md

@@ -0,0 +1,183 @@
+# Elasticsearch index template
+
+Below are instructions to create an Elasticsearch template for the index that will be used to store JMeter metrics:
+
+```text
+HEAD _template/jmeter_metrics_template
+GET /_template/jmeter_metrics_template
+GET _template/*jmeter_metrics*
+DELETE /_template/jmeter_metrics_template
+
+PUT _template/jmeter_metrics_template
+{
+  "order": 1,
+  "template": "jmeter_metrics-*",
+  "settings": {
+    "index": {
+      "codec": "best_compression",
+      "mapping": {
+        "total_fields": {
+          "limit": "256"
+        }
+      },
+      "refresh_interval": "1s",
+      "number_of_replicas": "2",
+      "number_of_shards": "1"
+    }
+  },
+  "mappings": {
+    "logs": {
+      "dynamic_templates": [
+        {
+          "strings_as_keywords": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "keyword"
+            }
+          }
+        }
+      ],
+      "_all": {
+        "enabled": false
+      },
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword",
+          "ignore_above": 256
+        },
+        "AllThreads": {
+          "type": "integer"
+        },
+        "AssertionResults": {
+          "type": "text"
+        },
+        "BodySize": {
+          "type": "long"
+        },
+        "BuildNumber": {
+          "type": "integer"
+        },
+        "Bytes": {
+          "type": "long"
+        },
+        "ConnectTime": {
+          "type": "long"
+        },
+        "ContentType": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "DataType": {
+          "type": "keyword",
+          "ignore_above": 256
+        },
+        "ElapsedTime": {
+          "type": "date"
+        },
+        "ElapsedTimeComparison": {
+          "type": "date"
+        },
+        "ErrorCount": {
+          "type": "integer"
+        },
+        "FailureMessage": {
+          "type": "text"
+        },
+        "GrpThreads": {
+          "type": "integer"
+        },
+        "IdleTime": {
+          "type": "long"
+        },
+        "InjectorHostname": {
+          "type": "keyword",
+          "ignore_above": 256
+        },
+        "Latency": {
+          "type": "long"
+        },
+        "RequestBody": {
+          "type": "text"
+        },
+        "RequestHeaders": {
+          "type": "text"
+        },
+        "ResponseBody": {
+          "type": "text"
+        },
+        "ResponseCode": {
+          "type": "keyword",
+          "ignore_above": 256
+        },
+        "ResponseHeaders": {
+          "type": "text"
+        },
+        "ResponseMessage": {
+          "type": "text"
+        },
+        "ResponseTime": {
+          "type": "long"
+        },
+        "SampleCount": {
+          "type": "integer"
+        },
+        "SampleEndTime": {
+          "type": "date"
+        },
+        "SampleLabel": {
+          "type": "keyword",
+          "ignore_above": 256
+        },
+        "SampleStartTime": {
+          "type": "date"
+        },
+        "SentBytes": {
+          "type": "long"
+        },
+        "Success": {
+          "type": "boolean"
+        },
+        "TestElement": {
+          "properties": {
+            "name": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "TestStartTime": {
+          "type": "long"
+        },
+        "ThreadName": {
+          "type": "keyword",
+          "ignore_above": 256
+        },
+        "Timestamp": {
+          "type": "date"
+        },
+        "URL": {
+          "type": "keyword",
+          "ignore_above": 256
+        }
+      }
+    }
+  },
+  "aliases": {
+    "jmeter_metrics": {}
+  }
+}
+
+DELETE jmeter_metrics-2019-06-28
+PUT jmeter_metrics-2019-06-28
+GET jmeter_metrics-2019-06-28/_mapping
+GET jmeter_metrics-2019-06-28/_aliases
+GET jmeter_metrics-2019-06-28/_settings
+```

docs/Listener visualization.json

@@ -0,0 +1,89 @@
+[
+  {
+    "_id": "JMeter_statistic_dashboard",
+    "_type": "dashboard",
+    "_source": {
+      "title": "JMeter statistic",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"panelIndex\":\"5\",\"gridData\":{\"x\":0,\"y\":4,\"w\":12,\"h\":4,\"i\":\"5\"},\"embeddableConfig\":{},\"id\":\"Aggregate_report_visualization\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"panelIndex\":\"6\",\"gridData\":{\"x\":4,\"y\":1,\"w\":8,\"h\":3,\"i\":\"6\"},\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"id\":\"TPS_visualization\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":0,\"y\":1,\"w\":4,\"h\":3,\"i\":\"7\"},\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 500\":\"rgb(165,0,38)\",\"1000 - 5000\":\"rgb(0,104,55)\",\"500 - 1000\":\"rgb(255,255,190)\"},\"legendOpen\":false}},\"id\":\"Intensity_visualization\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":0,\"y\":0,\"w\":12,\"h\":1,\"i\":\"8\"},\"version\":\"6.2.4\",\"type\":\"visualization\",\"id\":\"Controls_visualization\"}]",
+      "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":true,\"useMargins\":false}",
+      "version": 1,
+      "timeRestore": false,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 1
+    }
+  },
+  {
+    "_id": "Intensity_visualization",
+    "_type": "visualization",
+    "_source": {
+      "title": "Intensity",
+      "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"1-bucket\",\"params\":{\"customInterval\":\"30s\",\"extended_bounds\":{},\"field\":\"Timestamp\",\"interval\":\"m\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"TPS\",\"customMetric\":{\"enabled\":true,\"id\":\"1-metric\",\"params\":{\"field\":\"SampleCount\",\"json\":\"{ \\\"script\\\" : \\\"_value/60\\\" }\"},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"sum\"}},\"schema\":\"metric\",\"type\":\"avg_bucket\"}],\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":500},{\"from\":500,\"to\":1000},{\"from\":1000,\"to\":5000}],\"extendRange\":true,\"gaugeColorMode\":\"Labels\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Arc\",\"invertColors\":true,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true},\"style\":{\"bgColor\":false,\"bgFill\":\"#eee\",\"bgMask\":false,\"bgWidth\":0.9,\"fontSize\":60,\"labelColor\":true,\"mask\":false,\"maskBars\":50,\"subText\":\"\",\"width\":0.9},\"type\":\"meter\",\"verticalSplit\":false},\"isDisplayWarning\":false,\"type\":\"gauge\"},\"title\":\"Intensity\",\"type\":\"gauge\"}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 500\":\"rgb(165,0,38)\",\"500 - 1000\":\"rgb(255,255,190)\",\"1000 - 5000\":\"rgb(0,104,55)\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"jmeter_metrics\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 1
+    }
+  },
+  {
+    "_id": "TPS_visualization",
+    "_type": "visualization",
+    "_source": {
+      "title": "TPS",
+      "visState": "{\"title\":\"TPS\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"RightAxis-1\",\"type\":\"value\",\"position\":\"right\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"silhouette\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"TPS\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"TPS\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"SampleCount\",\"json\":\"{ \\\"script\\\" : \\\"_value/60\\\" }\",\"customLabel\":\"TPS\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"Timestamp\",\"interval\":\"m\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\".\"}}]}",
+      "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"jmeter_metrics\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 1
+    }
+  },
+  {
+    "_id": "Controls_visualization",
+    "_type": "visualization",
+    "_source": {
+      "title": "Controls",
+      "visState": "{\"title\":\"Controls\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"fieldName\":\"es-testName.keyword\",\"id\":\"1542568300307\",\"indexPattern\":\"jmeter_metrics\",\"label\":\"Profile\",\"options\":{\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"type\":\"list\"},{\"id\":\"1542568419863\",\"indexPattern\":\"jmeter_metrics\",\"fieldName\":\"es-runid.keyword\",\"label\":\"RunId\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"size\":5,\"order\":\"desc\"}}],\"updateFiltersOnChange\":true,\"useTimeFilter\":true},\"aggs\":[]}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 1
+    }
+  },
+  {
+    "_id": "Aggregate_report_visualization",
+    "_type": "visualization",
+    "_source": {
+      "title": "Aggregate report",
+      "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Total\"},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"4\",\"params\":{\"customLabel\":\"Records\",\"field\":\"SampleCount\"},\"schema\":\"metric\",\"type\":\"sum\"},{\"enabled\":true,\"id\":\"6\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"6-bucket\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"Timestamp\",\"interval\":\"m\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"per sec.\",\"customMetric\":{\"enabled\":true,\"id\":\"6-metric\",\"params\":{\"field\":\"SampleCount\",\"json\":\"{ \\\"script\\\" : \\\"_value/60\\\" }\"},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"sum\"}},\"schema\":\"metric\",\"type\":\"avg_bucket\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"AVG resp.\",\"field\":\"ResponseTime\",\"json\":\"{ \\\"script\\\" : \\\"_value/1000\\\" }\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"7\",\"params\":{\"customLabel\":\".\",\"field\":\"ResponseTime\",\"json\":\"{ \\\"script\\\" : \\\"_value/1000\\\" }\",\"percents\":[97]},\"schema\":\"metric\",\"type\":\"percentiles\"},{\"enabled\":true,\"id\":\"5\",\"params\":{\"customLabel\":\"Errors\",\"field\":\"ErrorCount\"},\"schema\":\"metric\",\"type\":\"sum\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Transaction\",\"field\":\"SampleLabel.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":5},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":null,\"showMeticsAtAllLevels\":false,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"count\"},\"title\":\"Aggregate report\",\"type\":\"table\"}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"jmeter_metrics\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
+      }
+    },
+    "_meta": {
+      "savedObjectVersion": 1
+    }
+  }
+]

docs/logstash.conf

@@ -1,9 +1,9 @@
 input {
   kafka {
     bootstrap_servers => "localhost:9092"
-    topics => [ "test" ]
+    topics => [ "JMETER_METRICS" ]
     codec => "json"
-    group_id => "GRP_TEST_LOGSTASH_P_1"
+    group_id => "GRP_JMETER_METRICS_LOGSTASH_01"
     auto_offset_reset => "earliest"
   }
 }
@@ -12,15 +12,52 @@ filter {
   json {
     source => "message"
   }
+
+  # Logstash has strange implementation for json filter above.
+  # If it sees a field named @timestamp, then it expects it to be a seconds based value (UNIX rather than UNIX_MS) and tries to parse on its own to reuse as event's @timestamp.
+  # In our case, @timestamp is milli-seconds based long, hence it fails to parse and causes a warning with _timestampparsefailure tag.
+
+  if "_timestampparsefailure" in [tags] {
+    date {
+      match => [ "_@timestamp", "UNIX_MS" ]
+      remove_tag => "_timestampparsefailure"
+      remove_field => "_@timestamp"
+    }
+  }
+
+  date{
+    match => [ "ElapsedTime", "yyyy-MM-dd HH:mm:ss" ]
+    target => ElapsedTime
+  }
+
+  date{
+    match => [ "Timestamp", "ISO8601" ]
+    target => Timestamp
+  }
+
+  date{
+    match => [ "SampleStartTime", "ISO8601" ]
+    target => SampleStartTime
+  }
+
+  date{
+    match => [ "SampleEndTime", "ISO8601" ]
+    target => SampleEndTime
+  }
+
+  ruby {
+    code => "if (event.get('tags') != nil) && event.get('tags').length == 0 then event.remove('tags') end"
+  }
 }
 
 output {
   #stdout { codec => rubydebug }
 
   elasticsearch {
-    index => "jmeter_metrics-%{+xxxx-ww}"  # Weekly index
+    index => "jmeter_metrics-%{+yyyy-MM-dd}"
     hosts => ["localhost:9201"]
     user => "elastic"
     password => "changeme"
+    document_type => "logs"
   }
 }

src/main/java/io/github/rahulsinghai/jmeter/backendlistener/kafka/JSONMetric.java

@@ -288,19 +288,19 @@ public Date getElapsedTime(boolean forBuildComparison) {
     if (forBuildComparison) {
       sElapsed =
           String.format(
-              "2017-01-01 %02d:%02d:%02d",
+              "2019-07-01 %02d:%02d:%02d",
               cal.get(Calendar.HOUR_OF_DAY), cal.get(Calendar.MINUTE), cal.get(Calendar.SECOND));
     } else {
       sElapsed =
           String.format(
               "%s %02d:%02d:%02d",
-              DateTimeFormatter.ofPattern("yyyy-mm-dd").format(LocalDateTime.now()),
+              DateTimeFormatter.ofPattern("yyyy-MM-dd").format(LocalDateTime.now()),
               cal.get(Calendar.HOUR_OF_DAY),
               cal.get(Calendar.MINUTE),
               cal.get(Calendar.SECOND));
     }
 
-    SimpleDateFormat formatter = new SimpleDateFormat("yyyy-mm-dd HH:mm:ss");
+    SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     try {
       return formatter.parse(sElapsed);
     } catch (ParseException e) {