Skip to content

Commit 9c532f4

Browse files
mnasiadkayoctozepto
mnasiadka
authored and
yoctozepto
committed
Refactor fluentd syslog logging
Co-Authored-By: Mark Goddard <[email protected]> Change-Id: I75ca59d981bcd2dd51faa296ab0b4223a891f5cb (cherry picked from commit b97832d)
1 parent f582c52 commit 9c532f4

File tree

4 files changed

+40
-250
lines changed

4 files changed

+40
-250
lines changed

‎ansible/roles/common/defaults/main.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,30 @@ syslog_haproxy_facility: "local1"
7070
syslog_glance_tls_proxy_facility: "local2"
7171
syslog_neutron_tls_proxy_facility: "local4"
7272

73+
syslog_facilities:
74+
- name: "swift"
75+
enabled: "{{ enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) }}"
76+
facility: "{{ syslog_swift_facility }}"
77+
logdir: "swift"
78+
logfile: "swift_latest"
79+
output_tag: true
80+
output_time: true
81+
- name: "haproxy"
82+
enabled: "{{ enable_haproxy | bool and inventory_hostname in groups['haproxy'] }}"
83+
facility: "{{ syslog_haproxy_facility }}"
84+
logdir: "haproxy"
85+
logfile: "haproxy_latest"
86+
- name: "glance_tls_proxy"
87+
enabled: "{{ glance_enable_tls_backend | bool and inventory_hostname in groups['glance-api'] }}"
88+
facility: "{{ syslog_glance_tls_proxy_facility }}"
89+
logdir: "glance-tls-proxy"
90+
logfile: "glance-tls-proxy"
91+
- name: "neutron_tls_proxy"
92+
enabled: "{{ neutron_enable_tls_backend | bool and inventory_hostname in groups['neutron-server'] }}"
93+
facility: "{{ syslog_neutron_tls_proxy_facility }}"
94+
logdir: "neutron-tls-proxy"
95+
logfile: "neutron-tls-proxy"
96+
7397
kolla_toolbox_default_volumes:
7498
- "{{ node_config_directory }}/kolla-toolbox/:{{ container_config_directory }}/:ro"
7599
- "/etc/localtime:/etc/localtime:ro"

‎ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2‎

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,33 +22,14 @@
2222
</record>
2323
</filter>
2424

25-
<filter syslog.local0.**>
25+
{% for item in syslog_facilities | selectattr('enabled') %}
26+
<filter syslog.{{ item.facility }}.**>
2627
@type record_transformer
2728
<record>
28-
programname swift
29-
</record>
30-
</filter>
31-
32-
<filter syslog.local1.**>
33-
@type record_transformer
34-
<record>
35-
programname haproxy
36-
</record>
37-
</filter>
38-
39-
<filter syslog.{{ syslog_glance_tls_proxy_facility }}.**>
40-
@type record_transformer
41-
<record>
42-
programname glance-tls-proxy
43-
</record>
44-
</filter>
45-
46-
<filter syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
47-
@type record_transformer
48-
<record>
49-
programname neutron-tls-proxy
29+
programname {{ item.logdir }}
5030
</record>
5131
</filter>
32+
{% endfor %}
5233

5334
# Rename internal Fluent message field to match other logs. This removes
5435
# all other fields by default, including the original message field. This is

‎ansible/roles/common/templates/conf/output/00-local.conf.j2‎

Lines changed: 9 additions & 203 deletions
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,17 @@
1-
{% if enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) %}
2-
<match syslog.{{ syslog_swift_facility }}.**>
1+
{% for item in syslog_facilities | selectattr('enabled') %}
2+
<match syslog.{{ item.facility }}.**>
33
@type copy
44
<store>
55
@type file
6-
path /var/log/kolla/swift/swift_latest
6+
path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}
77
append true
88
# Disable timestamp in filename for logs
99
<buffer []>
10-
path /var/log/kolla/swift/swift_latest.*.buffer
10+
path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}.*.buffer
1111
</buffer>
12-
</store>
13-
{% if log_direct_to_elasticsearch %}
14-
<store>
15-
@type elasticsearch
16-
host {{ elasticsearch_address }}
17-
port {{ elasticsearch_port }}
18-
scheme {{ fluentd_elasticsearch_scheme }}
19-
{% if fluentd_elasticsearch_path != '' %}
20-
path {{ fluentd_elasticsearch_path }}
21-
{% endif %}
22-
{% if fluentd_elasticsearch_scheme == 'https' %}
23-
ssl_version {{ fluentd_elasticsearch_ssl_version }}
24-
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
25-
{% if fluentd_elasticsearch_cacert | length > 0 %}
26-
ca_file {{ fluentd_elasticsearch_cacert }}
27-
{% endif %}
28-
{% endif %}
29-
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
30-
user {{ fluentd_elasticsearch_user }}
31-
password {{ fluentd_elasticsearch_password }}
32-
{% endif %}
33-
logstash_format true
34-
logstash_prefix {{ kibana_log_prefix }}
35-
flush_interval 15s
36-
reconnect_on_error true
37-
buffer_type file
38-
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
39-
suppress_type_name true
40-
</store>
41-
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
42-
<store>
43-
@type monasca
44-
keystone_url {{ keystone_internal_url }}
45-
monasca_api {{ monasca_log_api_internal_endpoint }}
46-
monasca_api_version v2.0
47-
username {{ monasca_agent_user }}
48-
password {{ monasca_agent_password }}
49-
domain_id default
50-
project_name {{ monasca_control_plane_project }}
51-
message_field_name Payload
52-
buffer_type file
53-
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_swift_facility }}.*
54-
max_retry_wait 1800s
55-
disable_retry_limit true
56-
<buffer>
57-
chunk_limit_size 8m
58-
</buffer>
59-
</store>
60-
{% endif %}
61-
</match>
62-
{% endif %}
63-
64-
{% if enable_haproxy | bool and inventory_hostname in groups['haproxy'] %}
65-
<match syslog.{{ syslog_haproxy_facility }}.**>
66-
@type copy
67-
<store>
68-
@type file
69-
path /var/log/kolla/haproxy/haproxy_latest
70-
append true
71-
# Disable timestamp in filename for logs
72-
<buffer []>
73-
path /var/log/kolla/haproxy/haproxy_latest.*.buffer
74-
</buffer>
75-
# Don't prepend syslog tag or timestamp to log output
7612
<format>
77-
output_tag false
78-
output_time false
13+
output_tag {{ item.output_tag | default(false) | lower }}
14+
output_time {{ item.output_time | default(false) | lower }}
7915
</format>
8016
</store>
8117
{% if log_direct_to_elasticsearch %}
@@ -103,7 +39,7 @@
10339
flush_interval 15s
10440
reconnect_on_error true
10541
buffer_type file
106-
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
42+
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.*
10743
suppress_type_name true
10844
</store>
10945
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
@@ -118,7 +54,7 @@
11854
project_name {{ monasca_control_plane_project }}
11955
message_field_name Payload
12056
buffer_type file
121-
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_haproxy_facility }}.*
57+
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ item.facility }}.*
12258
max_retry_wait 1800s
12359
disable_retry_limit true
12460
<buffer>
@@ -127,134 +63,4 @@
12763
</store>
12864
{% endif %}
12965
</match>
130-
{% endif %}
131-
132-
{% if glance_enable_tls_backend | bool %}
133-
<match syslog.{{ syslog_glance_tls_proxy_facility }}.**>
134-
@type copy
135-
<store>
136-
@type file
137-
path /var/log/kolla/glance-tls-proxy/glance-tls-proxy
138-
append true
139-
# Disable timestamp in filename for logs
140-
<buffer []>
141-
path /var/log/kolla/glance-tls-proxy/glance-tls-proxy.*.buffer
142-
</buffer>
143-
# Don't prepend syslog tag or timestamp to log output
144-
<format>
145-
output_tag false
146-
output_time false
147-
</format>
148-
</store>
149-
{% if log_direct_to_elasticsearch %}
150-
<store>
151-
@type elasticsearch
152-
host {{ elasticsearch_address }}
153-
port {{ elasticsearch_port }}
154-
scheme {{ fluentd_elasticsearch_scheme }}
155-
{% if fluentd_elasticsearch_path != '' %}
156-
path {{ fluentd_elasticsearch_path }}
157-
{% endif %}
158-
{% if fluentd_elasticsearch_scheme == 'https' %}
159-
ssl_version {{ fluentd_elasticsearch_ssl_version }}
160-
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
161-
{% endif %}
162-
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
163-
user {{ fluentd_elasticsearch_user }}
164-
password {{ fluentd_elasticsearch_password }}
165-
{% endif %}
166-
logstash_format true
167-
logstash_prefix {{ kibana_log_prefix }}
168-
flush_interval 15s
169-
reconnect_on_error true
170-
buffer_type file
171-
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.*
172-
suppress_type_name true
173-
</store>
174-
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
175-
<store>
176-
@type monasca
177-
keystone_url {{ keystone_internal_url }}
178-
monasca_api {{ monasca_log_api_internal_endpoint }}
179-
monasca_api_version v2.0
180-
username {{ monasca_agent_user }}
181-
password {{ monasca_agent_password }}
182-
domain_id default
183-
project_name {{ monasca_control_plane_project }}
184-
message_field_name Payload
185-
buffer_type file
186-
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_glance_tls_proxy_facility }}.*
187-
max_retry_wait 1800s
188-
disable_retry_limit true
189-
<buffer>
190-
chunk_limit_size 8m
191-
</buffer>
192-
</store>
193-
{% endif %}
194-
</match>
195-
{% endif %}
196-
197-
{% if neutron_enable_tls_backend | bool %}
198-
<match syslog.{{ syslog_neutron_tls_proxy_facility }}.**>
199-
@type copy
200-
<store>
201-
@type file
202-
path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy
203-
append true
204-
# Disable timestamp in filename for logs
205-
<buffer []>
206-
path /var/log/kolla/neutron-tls-proxy/neutron-tls-proxy.*.buffer
207-
</buffer>
208-
# Don't prepend syslog tag or timestamp to log output
209-
<format>
210-
output_tag false
211-
output_time false
212-
</format>
213-
</store>
214-
{% if log_direct_to_elasticsearch %}
215-
<store>
216-
@type elasticsearch
217-
host {{ elasticsearch_address }}
218-
port {{ elasticsearch_port }}
219-
scheme {{ fluentd_elasticsearch_scheme }}
220-
{% if fluentd_elasticsearch_path != '' %}
221-
path {{ fluentd_elasticsearch_path }}
222-
{% endif %}
223-
{% if fluentd_elasticsearch_scheme == 'https' %}
224-
ssl_version {{ fluentd_elasticsearch_ssl_version }}
225-
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
226-
{% endif %}
227-
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
228-
user {{ fluentd_elasticsearch_user }}
229-
password {{ fluentd_elasticsearch_password }}
230-
{% endif %}
231-
logstash_format true
232-
logstash_prefix {{ kibana_log_prefix }}
233-
flush_interval 15s
234-
reconnect_on_error true
235-
buffer_type file
236-
buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
237-
suppress_type_name true
238-
</store>
239-
{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
240-
<store>
241-
@type monasca
242-
keystone_url {{ keystone_internal_url }}
243-
monasca_api {{ monasca_log_api_internal_endpoint }}
244-
monasca_api_version v2.0
245-
username {{ monasca_agent_user }}
246-
password {{ monasca_agent_password }}
247-
domain_id default
248-
project_name {{ monasca_control_plane_project }}
249-
message_field_name Payload
250-
buffer_type file
251-
buffer_path /var/lib/fluentd/data/monasca.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
252-
max_retry_wait 1800s
253-
disable_retry_limit true
254-
<buffer>
255-
chunk_limit_size 8m
256-
</buffer>
257-
</store>
258-
{% endif %}
259-
</match>
260-
{% endif %}
66+
{% endfor %}

‎ansible/roles/common/templates/fluentd.json.j2‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -29,34 +29,13 @@
2929
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
3030
"recurse": true
3131
},
32-
{% if enable_haproxy | bool and inventory_hostname in groups['haproxy'] %}
32+
{% for facility in syslog_facilities | selectattr('enabled') %}
3333
{
34-
"path": "/var/log/kolla/haproxy",
34+
"path": "/var/log/kolla/{{ facility.logdir }}",
3535
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
3636
"recurse": true
3737
},
38-
{% endif %}
39-
{% if glance_enable_tls_backend | bool %}
40-
{
41-
"path": "/var/log/kolla/glance-tls-proxy",
42-
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
43-
"recurse": true
44-
},
45-
{% endif %}
46-
{% if neutron_enable_tls_backend | bool %}
47-
{
48-
"path": "/var/log/kolla/neutron-tls-proxy",
49-
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
50-
"recurse": true
51-
},
52-
{% endif %}
53-
{% if enable_swift | bool and (inventory_hostname in groups['swift-proxy-server'] or inventory_hostname in groups['swift-account-server'] or inventory_hostname in groups['swift-container-server'] or inventory_hostname in groups['swift-object-server']) %}
54-
{
55-
"path": "/var/log/kolla/swift",
56-
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
57-
"recurse": true
58-
},
59-
{% endif %}
38+
{% endfor %}
6039
{
6140
"path": "/var/lib/fluentd/data",
6241
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",

0 commit comments

Comments
 (0)