diff --git a/CHANGELOG.md b/CHANGELOG.md index d14e0d4af..9ba4dd6ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,11 @@ Role:acme_sh ### Breaking Changes +Role:graylog_server +* Removed creation a default index set and default input. Take a look at the README for the new intended method. +* Renamed `graylog_server__system_default_index_set` to `graylog_server__system_index_sets__*_var` & `graylog_server__system_inputs` to `graylog_server__system_inputs__*_var` +* Add state management for `graylog_server__system_index_sets__host_var` & `graylog_server__system_inputs__host_var` + Role:mariadb_server * Removed support for EOL version 10.5 * Removed `mariadb_server__cnf_expire_logs_days__group_var` / `mariadb_server__cnf_expire_logs_days__host_var`, use `mariadb_server__cnf_binlog_expire_logs_seconds__group_var` / `mariadb_server__cnf_binlog_expire_logs_seconds__host_var` instead diff --git a/roles/graylog_server/README.md b/roles/graylog_server/README.md index 0f6aa29e5..53fe74ae1 100644 --- a/roles/graylog_server/README.md +++ b/roles/graylog_server/README.md @@ -32,10 +32,11 @@ If you use the ["Setup Graylog Server" Playbook](https://github.com/Linuxfabrik/ | Tag | What it does | Reload / Restart | | --- | ------------ | ---------------- | -| `graylog_server` | Installs and configures Graylog Server | Restarts graylog-server.service | -| `graylog_server:configure` | Deploys the config files, manages the CA keystore, creates the system inputs and a default index set | Restarts graylog-server.service | -| `graylog_server:configure_defaults` | Only executed on demand. Configure Graylog Indices, Index Sets and Inputs. | - | -| `graylog_server:state` | Manages the state of the Graylog Server service | - | +| `graylog_server` | Installs and configures Graylog Server. | Restarts graylog-server.service | +| `graylog_server:configure` | Deploys the config files and manages the CA keystore. | Restarts graylog-server.service | +| `graylog_server:configure_system_inputs` | Only executed on demand. Configure Graylog Inputs. | - | +| `graylog_server:configure_system_index_sets` | Only executed on demand. Configure Graylog Index Sets. | - | +| `graylog_server:state` | Manages the state of the Graylog Server service. | - | ## Mandatory Role Variables @@ -84,40 +85,46 @@ graylog_server__timezone: 'Europe/Zurich' ## Configure Graylog Indices, Index Sets and Inputs -Use the tag `graylog_server:configure_defaults` to configure Graylog indices, index sets and inputs. +Use the tags `graylog_server:configure_system_inputs` or `graylog_server:configure_system_index_sets` to configure Graylog index sets and inputs. | Variable | Description | Default Value | | -------- | ----------- | ------------- | -| `graylog_server__system_default_index_set` | Creates a default index set. Subkeys: | One index per day; 365 indices max | -| `graylog_server__system_inputs` | Creates system inputs. Subkeys: | Gelf (12201/TCP), Gelf (12201/UDP), Syslog (1514/UDP) | +| `graylog_server__system_index_sets__host_var` / `graylog_server__system_index_sets__group_var` | Creates additional index sets. Subkeys: | `unset` | +| `graylog_server__system_inputs__host_var` / `graylog_server__system_inputs__group_var` | Creates system inputs. Subkeys: | `unset` | Example: ```yaml # optional -graylog_server__system_default_index_set: - can_be_default: true - creation_date: '{{ ansible_date_time.iso8601 }}' - description: 'One index per day; 365 indices max' - field_type_refresh_interval: 5000 - index_analyzer: 'standard' - index_optimization_disabled: false - index_optimization_max_num_segments: 1 - index_prefix: 'lfops-default' - replicas: 0 - retention_strategy: - max_number_of_indices: 365 - type: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig' - retention_strategy_class: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy' - rotation_strategy: - rotation_period: 'P1D' - rotate_empty_index_set: false - type: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig' - rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy' - shards: 4 - title: 'Linuxfabrik Index Set (managed by Ansible - do not edit)' - writable: true -graylog_server__system_inputs: - - configuration: +graylog_server__system_index_sets__host_var: + - index_prefix: 'linuxfabrik' # <-- Must be unique. Must not start with the same phrase, e.g. linuxfabrik and linuxfabrik02 would conflict with each other + can_be_default: true + data_tiering: + index_lifetime_min: 'P30D' + index_lifetime_max: 'P40D' + type: 'hot_only' + description: 'Another Index Set' + field_type_refresh_interval: 5000 + index_analyzer: 'standard' + index_optimization_disabled: false + index_optimization_max_num_segments: 1 + replicas: 0 + retention_strategy: + max_number_of_indices: 20 + type: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig' + retention_strategy_class: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy' + rotation_strategy: + index_lifetime_min: 'P7D' + index_lifetime_max: 'P10D' + type: 'org.graylog2.indexer.rotation.strategies.TimeBasedSizeOptimizingStrategyConfig' + rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.TimeBasedSizeOptimizingStrategy' + shards: 1 + title: 'Linuxfabrik Index Set (managed by Ansible - do not edit)' + use_legacy_rotation: false + writable: true + state: 'present' +graylog_server__system_inputs__host_var: + - title: 'Beats (5044/TCP - managed by Ansible - do not edit)' # <-- Must be unique + configuration: bind_address: '0.0.0.0' number_worker_threads: 4 override_source: '' @@ -131,9 +138,10 @@ graylog_server__system_inputs: tls_key_file: '' tls_key_password: '' global: true - title: 'Beats (5044/TCP - managed by Ansible - do not edit)' type: 'org.graylog.plugins.beats.Beats2Input' - - configuration: + state: 'present' + - title: 'Gelf (12201/TCP - managed by Ansible - do not edit)' # <-- Must be unique + configuration: bind_address: '0.0.0.0' decompress_size_limit: 8388608 max_message_size: 2097152 @@ -150,9 +158,10 @@ graylog_server__system_inputs: tls_key_password: '' use_null_delimiter: true global: true - title: 'Gelf (12201/TCP - managed by Ansible - do not edit)' type: 'org.graylog2.inputs.gelf.tcp.GELFTCPInput' - - configuration: + state: 'present' + - title: 'Gelf (12201/UDP - managed by Ansible - do not edit)' # <-- Must be unique + configuration: bind_address: '0.0.0.0' decompress_size_limit: 8388608 number_worker_threads: 4 @@ -160,9 +169,10 @@ graylog_server__system_inputs: port: 12201 recv_buffer_size: 1048576 global: true - title: 'Gelf (12201/UDP - managed by Ansible - do not edit)' type: 'org.graylog2.inputs.gelf.udp.GELFUDPInput' - - configuration: + state: 'present' + - title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' # <-- Must be unique + configuration: allow_override_date: true bind_address: '0.0.0.0' decompress_size_limit: 8388608 @@ -174,8 +184,8 @@ graylog_server__system_inputs: recv_buffer_size: 1048576 store_full_message: false global: true - title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' type: 'org.graylog2.inputs.syslog.udp.SyslogUDPInput' + state: 'present' ``` diff --git a/roles/graylog_server/defaults/main.yml b/roles/graylog_server/defaults/main.yml index c7efc75d7..b287b1a12 100644 --- a/roles/graylog_server/defaults/main.yml +++ b/roles/graylog_server/defaults/main.yml @@ -8,89 +8,29 @@ graylog_server__service_enabled: true graylog_server__stale_leader_timeout_ms: 2000 graylog_server__timezone: 'Europe/Zurich' -graylog_server__system_default_index_set: - can_be_default: true - creation_date: '{{ ansible_date_time.iso8601 }}' - description: 'One index per day; 365 indices max' - field_type_refresh_interval: 5000 - index_analyzer: 'standard' - index_optimization_disabled: false - index_optimization_max_num_segments: 1 - index_prefix: 'lfops-default' - replicas: 0 - retention_strategy: - max_number_of_indices: 365 - type: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig' - retention_strategy_class: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy' - rotation_strategy: - rotation_period: 'P1D' - rotate_empty_index_set: false - type: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig' - rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy' - shards: 4 - title: 'Linuxfabrik Index Set (managed by Ansible - do not edit)' - writable: true +graylog_server__system_index_sets__dependent_var: [] +graylog_server__system_index_sets__group_var: [] +graylog_server__system_index_sets__host_var: [] +graylog_server__system_index_sets__role_var: [] +graylog_server__system_index_sets__combined_var: '{{ ( + graylog_server__system_index_sets__role_var + + graylog_server__system_index_sets__dependent_var + + graylog_server__system_index_sets__group_var + + graylog_server__system_index_sets__host_var + ) | linuxfabrik.lfops.combine_lod(unique_key="index_prefix") + }}' -graylog_server__system_inputs: - - configuration: - bind_address: '0.0.0.0' - number_worker_threads: 4 - override_source: '' - port: 5044 - recv_buffer_size: 1048576 - tcp_keepalive: false - tls_cert_file: '' - tls_client_auth: 'disabled' - tls_client_auth_cert_file: '' - tls_enable: false - tls_key_file: '' - tls_key_password: '' - global: true - title: 'Beats (5044/TCP - managed by Ansible - do not edit)' - type: 'org.graylog.plugins.beats.Beats2Input' - - configuration: - bind_address: '0.0.0.0' - decompress_size_limit: 8388608 - max_message_size: 2097152 - number_worker_threads: 4 - override_source: '' - port: 12201 - recv_buffer_size: 1048576 - tcp_keepalive: false - tls_cert_file: '' - tls_client_auth: 'disabled' - tls_client_auth_cert_file: '' - tls_enable: false - tls_key_file: '' - tls_key_password: '' - use_null_delimiter: true - global: true - title: 'Gelf (12201/TCP - managed by Ansible - do not edit)' - type: 'org.graylog2.inputs.gelf.tcp.GELFTCPInput' - - configuration: - bind_address: '0.0.0.0' - decompress_size_limit: 8388608 - number_worker_threads: 4 - override_source: '' - port: 12201 - recv_buffer_size: 1048576 - global: true - title: 'Gelf (12201/UDP - managed by Ansible - do not edit)' - type: 'org.graylog2.inputs.gelf.udp.GELFUDPInput' - - configuration: - allow_override_date: true - bind_address: '0.0.0.0' - decompress_size_limit: 8388608 - expand_structured_data: false - force_rdns: false - number_worker_threads: 4 - override_source: '' - port: 1514 - recv_buffer_size: 1048576 - store_full_message: false - global: true - title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' - type: 'org.graylog2.inputs.syslog.udp.SyslogUDPInput' +graylog_server__system_inputs__dependent_var: [] +graylog_server__system_inputs__group_var: [] +graylog_server__system_inputs__host_var: [] +graylog_server__system_inputs__role_var: [] +graylog_server__system_inputs__combined_var: '{{ ( + graylog_server__system_inputs__role_var + + graylog_server__system_inputs__dependent_var + + graylog_server__system_inputs__group_var + + graylog_server__system_inputs__host_var + ) | linuxfabrik.lfops.combine_lod(unique_key=["title", "type"]) + }}' # --------------------------------------------------- @@ -99,7 +39,7 @@ graylog_server__kernel_settings__sysctl__dependent_var: # `WARN [UdpTransport] receiveBufferSize (SO_RCVBUF) for input ... should be >= 1048576 but is 425984` # this warning is related to how the operating system is configured (udp receive buffer size) - name: 'net.core.rmem_max' - value: 1048576 # default: 212992 + value: 1048576 # default: 212992 graylog_server__kernel_settings__transparent_hugepages__dependent_var: 'never' graylog_server__selinux__booleans__dependent_var: diff --git a/roles/graylog_server/tasks/main.yml b/roles/graylog_server/tasks/main.yml index e61528cc9..9e64edff6 100644 --- a/roles/graylog_server/tasks/main.yml +++ b/roles/graylog_server/tasks/main.yml @@ -99,44 +99,83 @@ password: '{{ graylog_server__root_user["password"] }}' method: 'GET' force_basic_auth: true - # status_code: 200 - register: 'graylog_server__input_result' + register: 'graylog_server__inputs_result' changed_when: false # no actual config change check_mode: false # run task even if `--check` is specified - name: 'Graylog inputs:' ansible.builtin.debug: - var: 'graylog_server__input_result.json.inputs' + var: 'graylog_server__inputs_result.json.inputs' - - name: 'Terminate input on this node' + - name: 'Update existing inputs' ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs/{{ item }}' + url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs/{{ graylog_server__inputs_result.json.inputs | selectattr("title", "equalto", item["title"]) | map(attribute="id") | first }}' user: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' - method: 'DELETE' + method: 'PUT' + body: '{{ item | dict2items | rejectattr("key", "equalto", "state") | items2dict | to_json }}' force_basic_auth: true - status_code: 204 + status_code: 201 + body_format: 'json' headers: + Accept: 'application/json' X-Requested-By: 'cli' - loop: '{{ graylog_server__input_result | community.general.json_query("json.inputs[*].id") }}' + changed_when: true + loop: '{{ graylog_server__system_inputs__combined_var }}' + loop_control: + label: '{{ item["title"] }}, state={{ item["state"] | d("present") }}' when: - - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate inputs - - 'graylog_server__input_result.json.inputs is defined and graylog_server__input_result.json.inputs | length' + - "graylog_server__inputs_result | community.general.json_query(\"json.inputs[?title==`\" ~ item['title'] ~ \"`].id\") | length != 0" + - 'item["state"] | d("present") != "absent"' - - name: 'Launch input on this node' + - name: 'Create inputs' ansible.builtin.uri: url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs' user: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' method: 'POST' - body: '{{ item | to_json }}' + body: '{{ item | dict2items | rejectattr("key", "equalto", "state") | items2dict | to_json }}' force_basic_auth: true status_code: 201 body_format: 'json' headers: Accept: 'application/json' X-Requested-By: 'cli' - loop: '{{ graylog_server__system_inputs }}' + changed_when: true + loop: '{{ graylog_server__system_inputs__combined_var }}' + loop_control: + label: '{{ item["title"] }}, state={{ item["state"] | d("present") }}' + when: + - "graylog_server__inputs_result | community.general.json_query(\"json.inputs[?title==`\" ~ item['title'] ~ \"`].id\") | length == 0" + - 'item["state"] | d("present") != "absent"' + + - name: 'Delete inputs' + ansible.builtin.uri: + url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs/{{ graylog_server__inputs_result.json.inputs | selectattr("title", "equalto", item["title"]) | map(attribute="id") | first }}' + user: '{{ graylog_server__root_user["username"] }}' + password: '{{ graylog_server__root_user["password"] }}' + method: 'DELETE' + force_basic_auth: true + status_code: 204 + headers: + Accept: 'application/json' + X-Requested-By: 'cli' + changed_when: true + loop: '{{ graylog_server__system_inputs__combined_var }}' + loop_control: + label: '{{ item["title"] }}, state={{ item["state"] | d("present") }}' + when: + - "graylog_server__inputs_result | community.general.json_query(\"json.inputs[?title==`\" ~ item['title'] ~ \"`].id\") | length != 0" + - 'item["state"] | d("present") != "present"' + + when: + - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate inputs + tags: + - 'never' + - 'graylog_server:configure_system_inputs' + + +- block: - name: 'Get a list of all index sets' ansible.builtin.uri: @@ -145,52 +184,75 @@ password: '{{ graylog_server__root_user["password"] }}' method: 'GET' force_basic_auth: true - # status_code: 200 - register: 'graylog_server__get_index_sets_result' + register: 'graylog_server__index_sets_result' changed_when: false # no actual config change check_mode: false # run task even if `--check` is specified - - name: 'Create index set' + - name: 'Graylog indices:' + ansible.builtin.debug: + var: 'graylog_server__index_sets_result.json.index_sets' + + - name: 'Update existing index sets' ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets' + url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets/{{ graylog_server__index_sets_result.json.index_sets | selectattr("index_prefix", "equalto", item["index_prefix"]) | map(attribute="id") | first }}' user: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' - method: 'POST' - body: '{{ graylog_server__system_default_index_set | to_json }}' + method: 'PUT' + body: '{{ item | dict2items | rejectattr("key", "equalto", "state") | items2dict | to_json }}' force_basic_auth: true - # status_code: 200 body_format: 'json' headers: Accept: 'application/json' X-Requested-By: 'cli' - when: "graylog_server__get_index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ graylog_server__system_default_index_set['index_prefix'] ~ \"`].id\") | length == 0" + changed_when: true + loop: '{{ graylog_server__system_index_sets__combined_var }}' + loop_control: + label: '{{ item["index_prefix"] }}, state={{ item["state"] | d("present") }}' + when: + - "graylog_server__index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ item['index_prefix'] ~ \"`].id\") | length != 0" + - 'item["state"] | d("present") != "absent"' - - name: 'Get a list of all index sets' + - name: 'Create index sets' ansible.builtin.uri: url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets' user: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' - method: 'GET' + method: 'POST' + body: '{{ item | dict2items | rejectattr("key", "equalto", "state") | items2dict | to_json }}' force_basic_auth: true - # status_code: 200 - register: 'graylog_server__get_index_sets_result' - changed_when: false # no actual config change - check_mode: false # run task even if `--check` is specified + body_format: 'json' + headers: + Accept: 'application/json' + X-Requested-By: 'cli' + changed_when: true + loop: '{{ graylog_server__system_index_sets__combined_var }}' + loop_control: + label: '{{ item["index_prefix"] }}, state={{ item["state"] | d("present") }}' + when: + - "graylog_server__index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ item['index_prefix'] ~ \"`].id\") | length == 0" + - 'item["state"] | d("present") != "absent"' - - name: 'Set default index set' + - name: 'Delete index sets' ansible.builtin.uri: - url: "http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets/{{ graylog_server__get_index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ graylog_server__system_default_index_set['index_prefix'] ~ \"`].id\") | regex_replace(\"\\[\\'|\\'\\]\", \"\") }}/default" + url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets/{{ graylog_server__index_sets_result.json.index_sets | selectattr("index_prefix", "equalto", item["index_prefix"]) | map(attribute="id") | first }}' user: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' - method: 'PUT' + method: 'DELETE' force_basic_auth: true - # status_code: 200 + status_code: 204 headers: + Accept: 'application/json' X-Requested-By: 'cli' - when: "graylog_server__get_index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ graylog_server__system_default_index_set['index_prefix'] ~ \"`].id\") | length > 0" + changed_when: true + loop: '{{ graylog_server__system_index_sets__combined_var }}' + loop_control: + label: '{{ item["index_prefix"] }}, state={{ item["state"] | d("present") }}' + when: + - "graylog_server__index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ item['index_prefix'] ~ \"`].id\") | length != 0" + - 'item["state"] | d("present") != "present"' when: - - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate inputs + - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate index sets tags: - 'never' - - 'graylog_server:configure_defaults' + - 'graylog_server:configure_system_index_sets'