Skip to content

Commit 754282b

Browse files
markgoddardmarkgoddard
authored
Merge pull request #1140 from stackhpc/fluentd-buffer
Fix fluentd getting 413 errors from OpenSearch
2 parents 4c7b7c9 + bbe240a commit 754282b

File tree

2 files changed

+136
-0
lines changed

2 files changed

+136
-0
lines changed

‎etc/kayobe/kolla/config/fluentd/output/00-local.conf‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{% raw %}
2+
{% for item in syslog_facilities | selectattr('enabled') %}
3+
<match syslog.{{ item.facility }}.**>
4+
@type copy
5+
<store>
6+
@type file
7+
path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}
8+
append true
9+
# Disable timestamp in filename for logs
10+
<buffer []>
11+
path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}.*.buffer
12+
</buffer>
13+
<format>
14+
output_tag {{ item.output_tag | default(false) | lower }}
15+
output_time {{ item.output_time | default(false) | lower }}
16+
</format>
17+
</store>
18+
{% if log_direct_to_elasticsearch %}
19+
<store>
20+
@type elasticsearch
21+
host {{ elasticsearch_address }}
22+
port {{ elasticsearch_port | default('9200') }}
23+
scheme {{ fluentd_elasticsearch_scheme }}
24+
{% if fluentd_elasticsearch_path != '' %}
25+
path {{ fluentd_elasticsearch_path }}
26+
{% endif %}
27+
{% if fluentd_elasticsearch_scheme == 'https' %}
28+
ssl_version {{ fluentd_elasticsearch_ssl_version }}
29+
ssl_verify {{ fluentd_elasticsearch_ssl_verify }}
30+
{% if fluentd_elasticsearch_cacert | length > 0 %}
31+
ca_file {{ fluentd_elasticsearch_cacert }}
32+
{% endif %}
33+
{% endif %}
34+
{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%}
35+
user {{ fluentd_elasticsearch_user }}
36+
password {{ fluentd_elasticsearch_password }}
37+
{% endif %}
38+
logstash_format true
39+
logstash_prefix {{ opensearch_log_index_prefix }}
40+
reconnect_on_error true
41+
request_timeout {{ fluentd_elasticsearch_request_timeout }}
42+
suppress_type_name true
43+
<buffer>
44+
@type file
45+
path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.*
46+
flush_interval 15s
47+
</buffer>
48+
</store>
49+
{% elif log_direct_to_opensearch %}
50+
<store>
51+
@type opensearch
52+
host {{ opensearch_address }}
53+
port {{ opensearch_port }}
54+
scheme {{ fluentd_opensearch_scheme }}
55+
{% if fluentd_opensearch_path != '' %}
56+
path {{ fluentd_opensearch_path }}
57+
{% endif %}
58+
{% if fluentd_opensearch_scheme == 'https' %}
59+
ssl_version {{ fluentd_opensearch_ssl_version }}
60+
ssl_verify {{ fluentd_opensearch_ssl_verify }}
61+
{% if fluentd_opensearch_cacert | length > 0 %}
62+
ca_file {{ fluentd_opensearch_cacert }}
63+
{% endif %}
64+
{% endif %}
65+
{% if fluentd_opensearch_user != '' and fluentd_opensearch_password != ''%}
66+
user {{ fluentd_opensearch_user }}
67+
password {{ fluentd_opensearch_password }}
68+
{% endif %}
69+
logstash_format true
70+
logstash_prefix {{ opensearch_log_index_prefix }}
71+
reconnect_on_error true
72+
request_timeout {{ fluentd_opensearch_request_timeout }}
73+
suppress_type_name true
74+
bulk_message_request_threshold 20M
75+
<buffer>
76+
@type file
77+
path /var/lib/fluentd/data/opensearch.buffer/{{ item.facility }}.*
78+
flush_interval 15s
79+
chunk_limit_size 8M
80+
</buffer>
81+
</store>
82+
{% endif %}
83+
</match>
84+
{% endfor %}
85+
{% endraw %}

‎etc/kayobe/kolla/config/fluentd/output/03-opensearch.conf‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
{% raw %}
2+
{% if enable_caso | bool and inventory_hostname in groups['caso'] %}
3+
<match apel.events>
4+
@type copy
5+
<store>
6+
@type opensearch
7+
host { opensearch_address }}
8+
port {{ opensearch_port }}
9+
logstash_format true
10+
logstash_prefix apel
11+
flush_interval 15s
12+
</store>
13+
</match>
14+
{% endif %}
15+
16+
<match **>
17+
@type copy
18+
<store>
19+
@type opensearch
20+
host {{ opensearch_address }}
21+
port {{ opensearch_port }}
22+
scheme {{ fluentd_opensearch_scheme }}
23+
{% if fluentd_opensearch_path != '' %}
24+
path {{ fluentd_opensearch_path }}
25+
{% endif %}
26+
{% if fluentd_opensearch_scheme == 'https' %}
27+
ssl_version {{ fluentd_opensearch_ssl_version }}
28+
ssl_verify {{ fluentd_opensearch_ssl_verify }}
29+
{% if fluentd_opensearch_cacert | length > 0 %}
30+
ca_file {{ fluentd_opensearch_cacert }}
31+
{% endif %}
32+
{% endif %}
33+
{% if fluentd_opensearch_user != '' and fluentd_opensearch_password != ''%}
34+
user {{ fluentd_opensearch_user }}
35+
password {{ fluentd_opensearch_password }}
36+
{% endif %}
37+
logstash_format true
38+
logstash_prefix {{ opensearch_log_index_prefix }}
39+
reconnect_on_error true
40+
request_timeout {{ fluentd_opensearch_request_timeout }}
41+
suppress_type_name true
42+
bulk_message_request_threshold 20M
43+
<buffer>
44+
@type file
45+
path /var/lib/fluentd/data/opensearch.buffer/openstack.*
46+
flush_interval 15s
47+
chunk_limit_size 8M
48+
</buffer>
49+
</store>
50+
</match>
51+
{% endraw %}

0 commit comments

Comments
 (0)