Skip to content

Commit 3842a9a

Browse files
altenhofaltenhof
committed
Extended beats scripts to write "better" ES templates.
We now define explicit mappings for all "knoen" fields and set "ignore_malformed" to true to avoid field mapping issues with bogus logs.
1 parent 3d7f4e4 commit 3842a9a

File tree

6 files changed

+76
-5
lines changed

6 files changed

+76
-5
lines changed

‎cf-java-logging-support-core/beats/app-logs/etc/app-logs.template.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,15 @@
1818
}
1919
],
2020
"properties": {
21+
"@message": {
22+
"index": "analyzed",
23+
"type": "string"
24+
},
25+
"@timestamp": {
26+
"doc_values": true,
27+
"ignore_malformed": true,
28+
"type": "date"
29+
},
2130
"categories": {
2231
"doc_values": true,
2332
"index": "not_analyzed",
@@ -103,17 +112,24 @@
103112
"index": "not_analyzed",
104113
"type": "string"
105114
},
115+
"timestamp": {
116+
"doc_values": true,
117+
"ignore_malformed": true,
118+
"type": "long"
119+
},
106120
"type": {
107121
"doc_values": true,
108122
"index": "not_analyzed",
109123
"type": "string"
110124
},
111125
"written_at": {
112126
"doc_values": true,
127+
"ignore_malformed": true,
113128
"type": "date"
114129
},
115130
"written_ts": {
116131
"doc_values": true,
132+
"ignore_malformed": true,
117133
"type": "long"
118134
}
119135
}

‎cf-java-logging-support-core/beats/request-metrics/etc/request-metrics.template.json

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,15 @@
1818
}
1919
],
2020
"properties": {
21+
"@message": {
22+
"index": "analyzed",
23+
"type": "string"
24+
},
25+
"@timestamp": {
26+
"doc_values": true,
27+
"ignore_malformed": true,
28+
"type": "date"
29+
},
2130
"component_id": {
2231
"doc_values": true,
2332
"index": "not_analyzed",
@@ -115,14 +124,17 @@
115124
},
116125
"request_received_at": {
117126
"doc_values": true,
127+
"ignore_malformed": true,
118128
"type": "date"
119129
},
120130
"request_sent_at": {
121131
"doc_values": true,
132+
"ignore_malformed": true,
122133
"type": "date"
123134
},
124135
"request_size_b": {
125136
"doc_values": true,
137+
"ignore_malformed": true,
126138
"type": "long"
127139
},
128140
"response_content_type": {
@@ -132,22 +144,27 @@
132144
},
133145
"response_received_at": {
134146
"doc_values": true,
147+
"ignore_malformed": true,
135148
"type": "date"
136149
},
137150
"response_sent_at": {
138151
"doc_values": true,
152+
"ignore_malformed": true,
139153
"type": "date"
140154
},
141155
"response_size_b": {
142156
"doc_values": true,
157+
"ignore_malformed": true,
143158
"type": "long"
144159
},
145160
"response_status": {
146161
"doc_values": true,
162+
"ignore_malformed": true,
147163
"type": "integer"
148164
},
149165
"response_time_ms": {
150166
"doc_values": true,
167+
"ignore_malformed": true,
151168
"type": "float"
152169
},
153170
"space_id": {
@@ -160,17 +177,24 @@
160177
"index": "not_analyzed",
161178
"type": "string"
162179
},
180+
"timestamp": {
181+
"doc_values": true,
182+
"ignore_malformed": true,
183+
"type": "long"
184+
},
163185
"type": {
164186
"doc_values": true,
165187
"index": "not_analyzed",
166188
"type": "string"
167189
},
168190
"written_at": {
169191
"doc_values": true,
192+
"ignore_malformed": true,
170193
"type": "date"
171194
},
172195
"written_ts": {
173196
"doc_values": true,
197+
"ignore_malformed": true,
174198
"type": "long"
175199
},
176200
"x_forwarded_for": {

‎cf-java-logging-support-core/beats/scripts/gen_java_fields.rb

100644100755
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env python
2+
13
require 'yaml'
24
require 'set'
35
require 'pp'

‎cf-java-logging-support-core/beats/scripts/generate_fields_docs.py

100644100755
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env python
2+
13
import yaml
24
import sys
35

‎cf-java-logging-support-core/beats/scripts/generate_template.py

100644100755
Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,33 @@ def fields_to_es_template(input, output, index):
6767
}
6868
}
6969

70+
# make sure we do have a few "default" fields
7071
properties = {}
72+
7173
for doc, section in docs.items():
7274
if doc not in ["version", "defaults", "summary"]:
7375
prop = fill_section_properties(section, defaults)
7476
properties.update(prop)
7577

7678
template["mappings"]["_default_"]["properties"] = properties
79+
#
80+
# Add these two "defaults"
81+
# THIS IS SUPER IMPORTANT FOR US, OTHERWISE ES MAY CHOKE
82+
#
83+
template["mappings"]["_default_"]["properties"]["@message"] = {
84+
"type": "string",
85+
"index": "analyzed"
86+
}
87+
template["mappings"]["_default_"]["properties"]["timestamp"] = {
88+
"type": "long",
89+
"doc_values": True,
90+
"ignore_malformed": True
91+
}
92+
template["mappings"]["_default_"]["properties"]["@timestamp"] = {
93+
"type": "date",
94+
"doc_values": True,
95+
"ignore_malformed": True
96+
}
7797

7898
json.dump(template, output,
7999
indent=2, separators=(',', ': '),
@@ -124,27 +144,32 @@ def fill_field_properties(field, defaults):
124144
elif field.get("type") == "date":
125145
properties[field["name"]] = {
126146
"type": "date",
127-
"doc_values": True
147+
"doc_values": True,
148+
"ignore_malformed": True
128149
}
129150
elif field.get("type") == "long":
130151
properties[field["name"]] = {
131152
"type": "long",
132-
"doc_values": True
153+
"doc_values": True,
154+
"ignore_malformed": True
133155
}
134156
elif field.get("type") == "integer":
135157
properties[field["name"]] = {
136158
"type": "integer",
137-
"doc_values": True
159+
"doc_values": True,
160+
"ignore_malformed": True
138161
}
139162
elif field.get("type") == "double":
140163
properties[field["name"]] = {
141164
"type": "double",
142-
"doc_values": True
165+
"doc_values": True,
166+
"ignore_malformed": True
143167
}
144168
elif field.get("type") == "float":
145169
properties[field["name"]] = {
146170
"type": "float",
147-
"doc_values": True
171+
"doc_values": True,
172+
"ignore_malformed": True
148173
}
149174
elif field.get("type") == "string":
150175
properties[field["name"]] = {

‎cf-java-logging-support-core/beats/scripts/merge_templates.rb

100644100755
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env ruby
2+
13
require 'json'
24

35
if ARGV.size < 3

0 commit comments

Comments
 (0)