Skip to content

Commit d8aa3d6

Browse files
dianyang12138ZQKC
dianyang12138
authored and
ZQKC
committed
fix:修复es模版错误
1 parent 13d8fd5 commit d8aa3d6

File tree

5 files changed

+312
-103
lines changed

5 files changed

+312
-103
lines changed

‎bin/init_es_template.sh‎

Lines changed: 90 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -439,7 +439,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
439439
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{
440440
"order" : 10,
441441
"index_patterns" : [
442-
"ks_kafka_partition_metric*"
442+
"ks_kafka_replication_metric*"
443443
],
444444
"settings" : {
445445
"index" : {
@@ -500,30 +500,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
500500
}
501501
},
502502
"aliases" : { }
503-
}[root@10-255-0-23 template]# cat ks_kafka_replication_metric
504-
PUT _template/ks_kafka_replication_metric
505-
{
506-
"order" : 10,
507-
"index_patterns" : [
508-
"ks_kafka_replication_metric*"
509-
],
510-
"settings" : {
511-
"index" : {
512-
"number_of_shards" : "10"
513-
}
514-
},
515-
"mappings" : {
516-
"properties" : {
517-
"timestamp" : {
518-
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
519-
"index" : true,
520-
"type" : "date",
521-
"doc_values" : true
522-
}
523-
}
524-
},
525-
"aliases" : { }
526-
}'
503+
}'
527504

528505
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{
529506
"order" : 10,
@@ -640,7 +617,92 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
640617
}
641618
},
642619
"aliases" : { }
643-
}'
620+
}'
621+
622+
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_zookeeper_metric -d '{
623+
"order" : 10,
624+
"index_patterns" : [
625+
"ks_kafka_zookeeper_metric*"
626+
],
627+
"settings" : {
628+
"index" : {
629+
"number_of_shards" : "10"
630+
}
631+
},
632+
"mappings" : {
633+
"properties" : {
634+
"routingValue" : {
635+
"type" : "text",
636+
"fields" : {
637+
"keyword" : {
638+
"ignore_above" : 256,
639+
"type" : "keyword"
640+
}
641+
}
642+
},
643+
"clusterPhyId" : {
644+
"type" : "long"
645+
},
646+
"metrics" : {
647+
"properties" : {
648+
"AvgRequestLatency" : {
649+
"type" : "double"
650+
},
651+
"MinRequestLatency" : {
652+
"type" : "double"
653+
},
654+
"MaxRequestLatency" : {
655+
"type" : "double"
656+
},
657+
"OutstandingRequests" : {
658+
"type" : "double"
659+
},
660+
"NodeCount" : {
661+
"type" : "double"
662+
},
663+
"WatchCount" : {
664+
"type" : "double"
665+
},
666+
"NumAliveConnections" : {
667+
"type" : "double"
668+
},
669+
"PacketsReceived" : {
670+
"type" : "double"
671+
},
672+
"PacketsSent" : {
673+
"type" : "double"
674+
},
675+
"EphemeralsCount" : {
676+
"type" : "double"
677+
},
678+
"ApproximateDataSize" : {
679+
"type" : "double"
680+
},
681+
"OpenFileDescriptorCount" : {
682+
"type" : "double"
683+
},
684+
"MaxFileDescriptorCount" : {
685+
"type" : "double"
686+
}
687+
}
688+
},
689+
"key" : {
690+
"type" : "text",
691+
"fields" : {
692+
"keyword" : {
693+
"ignore_above" : 256,
694+
"type" : "keyword"
695+
}
696+
}
697+
},
698+
"timestamp" : {
699+
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
700+
"type" : "date"
701+
}
702+
}
703+
},
704+
"aliases" : { }
705+
}'
644706

645707
for i in {0..6};
646708
do
@@ -650,6 +712,7 @@ do
650712
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \
651713
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \
652714
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \
715+
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_zookeeper_metric${logdate} && \
653716
curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \
654717
exit 2
655-
done
718+
done

‎km-common/src/main/java/com/xiaojukeji/know/streaming/km/common/constant/ESIndexConstant.java‎

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -558,7 +558,7 @@ public class ESIndexConstant {
558558
public final static String REPLICATION_TEMPLATE = "{\n" +
559559
" \"order\" : 10,\n" +
560560
" \"index_patterns\" : [\n" +
561-
" \"ks_kafka_partition_metric*\"\n" +
561+
" \"ks_kafka_replication_metric*\"\n" +
562562
" ],\n" +
563563
" \"settings\" : {\n" +
564564
" \"index\" : {\n" +
@@ -619,29 +619,6 @@ public class ESIndexConstant {
619619
" }\n" +
620620
" },\n" +
621621
" \"aliases\" : { }\n" +
622-
" }[root@10-255-0-23 template]# cat ks_kafka_replication_metric\n" +
623-
"PUT _template/ks_kafka_replication_metric\n" +
624-
"{\n" +
625-
" \"order\" : 10,\n" +
626-
" \"index_patterns\" : [\n" +
627-
" \"ks_kafka_replication_metric*\"\n" +
628-
" ],\n" +
629-
" \"settings\" : {\n" +
630-
" \"index\" : {\n" +
631-
" \"number_of_shards\" : \"10\"\n" +
632-
" }\n" +
633-
" },\n" +
634-
" \"mappings\" : {\n" +
635-
" \"properties\" : {\n" +
636-
" \"timestamp\" : {\n" +
637-
" \"format\" : \"yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis\",\n" +
638-
" \"index\" : true,\n" +
639-
" \"type\" : \"date\",\n" +
640-
" \"doc_values\" : true\n" +
641-
" }\n" +
642-
" }\n" +
643-
" },\n" +
644-
" \"aliases\" : { }\n" +
645622
" }";
646623

647624
public final static String ZOOKEEPER_INDEX = "ks_kafka_zookeeper_metric";

‎km-dist/docker/manager/es_template_create.sh‎

Lines changed: 87 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
443443
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_replication_metric -d '{
444444
"order" : 10,
445445
"index_patterns" : [
446-
"ks_kafka_partition_metric*"
446+
"ks_kafka_replication_metric*"
447447
],
448448
"settings" : {
449449
"index" : {
@@ -504,29 +504,6 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
504504
}
505505
},
506506
"aliases" : { }
507-
}[root@10-255-0-23 template]# cat ks_kafka_replication_metric
508-
PUT _template/ks_kafka_replication_metric
509-
{
510-
"order" : 10,
511-
"index_patterns" : [
512-
"ks_kafka_replication_metric*"
513-
],
514-
"settings" : {
515-
"index" : {
516-
"number_of_shards" : "10"
517-
}
518-
},
519-
"mappings" : {
520-
"properties" : {
521-
"timestamp" : {
522-
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
523-
"index" : true,
524-
"type" : "date",
525-
"doc_values" : true
526-
}
527-
}
528-
},
529-
"aliases" : { }
530507
}'
531508

532509
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_topic_metric -d '{
@@ -646,6 +623,91 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
646623
"aliases" : { }
647624
}'
648625

626+
curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_zookeeper_metric -d '{
627+
"order" : 10,
628+
"index_patterns" : [
629+
"ks_kafka_zookeeper_metric*"
630+
],
631+
"settings" : {
632+
"index" : {
633+
"number_of_shards" : "10"
634+
}
635+
},
636+
"mappings" : {
637+
"properties" : {
638+
"routingValue" : {
639+
"type" : "text",
640+
"fields" : {
641+
"keyword" : {
642+
"ignore_above" : 256,
643+
"type" : "keyword"
644+
}
645+
}
646+
},
647+
"clusterPhyId" : {
648+
"type" : "long"
649+
},
650+
"metrics" : {
651+
"properties" : {
652+
"AvgRequestLatency" : {
653+
"type" : "double"
654+
},
655+
"MinRequestLatency" : {
656+
"type" : "double"
657+
},
658+
"MaxRequestLatency" : {
659+
"type" : "double"
660+
},
661+
"OutstandingRequests" : {
662+
"type" : "double"
663+
},
664+
"NodeCount" : {
665+
"type" : "double"
666+
},
667+
"WatchCount" : {
668+
"type" : "double"
669+
},
670+
"NumAliveConnections" : {
671+
"type" : "double"
672+
},
673+
"PacketsReceived" : {
674+
"type" : "double"
675+
},
676+
"PacketsSent" : {
677+
"type" : "double"
678+
},
679+
"EphemeralsCount" : {
680+
"type" : "double"
681+
},
682+
"ApproximateDataSize" : {
683+
"type" : "double"
684+
},
685+
"OpenFileDescriptorCount" : {
686+
"type" : "double"
687+
},
688+
"MaxFileDescriptorCount" : {
689+
"type" : "double"
690+
}
691+
}
692+
},
693+
"key" : {
694+
"type" : "text",
695+
"fields" : {
696+
"keyword" : {
697+
"ignore_above" : 256,
698+
"type" : "keyword"
699+
}
700+
}
701+
},
702+
"timestamp" : {
703+
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
704+
"type" : "date"
705+
}
706+
}
707+
},
708+
"aliases" : { }
709+
}'
710+
649711
for i in {0..6};
650712
do
651713
logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
@@ -654,6 +716,7 @@ do
654716
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_group_metric${logdate} && \
655717
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_partition_metric${logdate} && \
656718
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_replication_metric${logdate} && \
719+
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_zookeeper_metric${logdate} && \
657720
curl -s -o /dev/null -X PUT http://${SERVER_ES_ADDRESS}/ks_kafka_topic_metric${logdate} || \
658721
exit 2
659722
done

‎km-dist/init/template/ks_kafka_replication_metric‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,49 @@ PUT _template/ks_kafka_replication_metric
1111
},
1212
"mappings" : {
1313
"properties" : {
14+
"brokerId" : {
15+
"type" : "long"
16+
},
17+
"partitionId" : {
18+
"type" : "long"
19+
},
20+
"routingValue" : {
21+
"type" : "text",
22+
"fields" : {
23+
"keyword" : {
24+
"ignore_above" : 256,
25+
"type" : "keyword"
26+
}
27+
}
28+
},
29+
"clusterPhyId" : {
30+
"type" : "long"
31+
},
32+
"topic" : {
33+
"type" : "keyword"
34+
},
35+
"metrics" : {
36+
"properties" : {
37+
"LogStartOffset" : {
38+
"type" : "float"
39+
},
40+
"Messages" : {
41+
"type" : "float"
42+
},
43+
"LogEndOffset" : {
44+
"type" : "float"
45+
}
46+
}
47+
},
48+
"key" : {
49+
"type" : "text",
50+
"fields" : {
51+
"keyword" : {
52+
"ignore_above" : 256,
53+
"type" : "keyword"
54+
}
55+
}
56+
},
1457
"timestamp" : {
1558
"format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
1659
"index" : true,
@@ -20,4 +63,4 @@ PUT _template/ks_kafka_replication_metric
2063
}
2164
},
2265
"aliases" : { }
23-
}
66+
}

0 commit comments

Comments
 (0)