Merge pull request #140 from stackhpc/feature/monitoring_sp2_victoria

Feature/monitoring sp2 victoria

Author: dougszumski

ansible/group_vars/all.yml

@@ -684,7 +684,7 @@ enable_searchlight: "no"
 enable_senlin: "no"
 enable_skydive: "no"
 enable_solum: "no"
-enable_storm: "{{ enable_monasca | bool }}"
+enable_storm: "{{ enable_monasca | bool and monasca_enable_alerting_pipeline | bool }}"
 enable_swift: "no"
 enable_swift_s3api: "no"
 enable_swift_recon: "no"
@@ -718,6 +718,13 @@ enable_destroy_images: "no"
 ####################
 # Monasca options
 ####################
+monasca_enable_alerting_pipeline: True
+
+# Send logs from the control plane to the Monasca API. Monasca will then persist
+# them in Elasticsearch. If this is disabled, control plane logs will be sent
+# directly to Elasticsearch.
+monasca_ingest_control_plane_logs: True
+
 monasca_api_admin_base_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn  | put_address_in_context('url') }}:{{ monasca_api_port }}"
 monasca_api_internal_base_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn  | put_address_in_context('url') }}:{{ monasca_api_port }}"
 monasca_api_public_base_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn  | put_address_in_context('url') }}:{{ monasca_api_port }}"
@@ -1119,6 +1126,11 @@ infuxdb_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn | p
 #################
 kafka_datadir_volume: "kafka"
 
+# The number of brokers in a Kafka cluster. This is used for automatically
+# setting quantities such as topic replicas and it is not recommended to
+# change it unless you know what you are doing.
+kafka_broker_count: "{{ groups['kafka'] | length }}"
+
 #########################
 # Internal Image options
 #########################

ansible/monasca_cleanup.yml

@@ -0,0 +1,22 @@
+---
+- name: Cleanup unused Monasca services
+  hosts:
+    - monasca-api
+    - monasca-grafana
+    - monasca-log-transformer
+    - monasca-log-persister
+    - monasca-log-metrics
+    - monasca-thresh
+    - monasca-notification
+    - monasca-persister
+  roles:
+    - { role: monasca,
+        tags: monasca }
+
+- name: Cleanup unused Storm services
+  hosts:
+    - storm-worker
+    - storm-nimbus
+  roles:
+    - { role: storm,
+        tags: storm }

ansible/roles/common/tasks/config.yml

@@ -106,7 +106,7 @@
     log_direct_to_elasticsearch: >-
       {{ ( enable_elasticsearch | bool or
            ( elasticsearch_address != kolla_internal_vip_address )) and
-         not enable_monasca | bool }}
+           ( not enable_monasca | bool or not monasca_ingest_control_plane_logs | bool ) }}
     fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
     # Inputs
     fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
@@ -143,7 +143,7 @@
       - name: "conf/output/01-es.conf.j2"
         enabled: "{{ log_direct_to_elasticsearch }}"
       - name: "conf/output/02-monasca.conf.j2"
-        enabled: "{{ enable_monasca | bool }}"
+        enabled: "{{ enable_monasca | bool and monasca_ingest_control_plane_logs | bool }}"
     customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}"
   template:
     src: "td-agent.conf.j2"

ansible/roles/common/templates/conf/filter/00-record_transformer.conf.j2

@@ -71,7 +71,7 @@
     </record>
 </filter>
 
-{% if enable_monasca | bool %}
+{% if enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
 # Kolla configures Fluentd to extract timestamps from OpenStack service
 # logs, however these timestamps are not saved in the event and are not
 # forwarded to Monasca. Here we save the timestamp which has been

ansible/roles/common/templates/conf/output/00-local.conf.j2

@@ -36,7 +36,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_swift_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}
@@ -98,7 +98,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_haproxy_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}
@@ -158,7 +158,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_glance_tls_proxy_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}
@@ -219,7 +219,7 @@
        buffer_type file
        buffer_path /var/lib/fluentd/data/elasticsearch.buffer/{{ syslog_neutron_tls_proxy_facility }}.*
   </store>
-{% elif enable_monasca | bool %}
+{% elif enable_monasca | bool and monasca_ingest_control_plane_logs | bool %}
   <store>
      @type monasca
      keystone_url {{ keystone_internal_url }}

ansible/roles/elasticsearch/defaults/main.yml

@@ -59,7 +59,7 @@ elasticsearch_curator_dry_run: false
 
 # Index prefix pattern. Any indices matching this regex will
 # be managed by Curator.
-elasticsearch_curator_index_pattern: "^{{ 'monasca' if enable_monasca|bool else kibana_log_prefix }}-.*"
+elasticsearch_curator_index_pattern: "^{{ '(monasca|' + kibana_log_prefix + ')' if enable_monasca|bool else kibana_log_prefix }}-.*"
 
 # Duration after which an index is staged for deletion. This is
 # implemented by closing the index. Whilst in this state the index

ansible/roles/kafka/defaults/main.yml

@@ -20,7 +20,6 @@ kafka_cluster_name: "kolla_kafka"
 kafka_log_dir: "/var/log/kolla/kafka"
 kafka_heap_opts: "-Xmx1G -Xms1G"
 kafka_zookeeper: "{% for host in groups['zookeeper'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
-kafka_broker_count: "{{ groups['kafka'] | length }}"
 
 ####################
 # Docker

ansible/roles/monasca/defaults/main.yml

@@ -20,10 +20,12 @@ monasca_services:
         mode: "http"
         external: true
         port: "{{ monasca_api_port }}"
+  # NOTE(dszumski): We can remove log_transformer and all other references after the
+  # Wallaby release
   monasca-log-transformer:
     container_name: monasca_log_transformer
     group: monasca-log-transformer
-    enabled: true
+    enabled: false
     image: "{{ monasca_logstash_image_full }}"
     volumes: "{{ monasca_log_transformer_default_volumes + monasca_log_transformer_extra_volumes }}"
     dimensions: "{{ monasca_log_transformer_dimensions }}"
@@ -37,21 +39,21 @@ monasca_services:
   monasca-log-metrics:
     container_name: monasca_log_metrics
     group: monasca-log-metrics
-    enabled: true
+    enabled: "{{ monasca_enable_log_metrics_service | bool }}"
     image: "{{ monasca_logstash_image_full }}"
     volumes: "{{ monasca_log_metrics_default_volumes + monasca_log_metrics_extra_volumes }}"
     dimensions: "{{ monasca_log_metrics_dimensions }}"
   monasca-thresh:
     container_name: monasca_thresh
     group: monasca-thresh
-    enabled: true
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
     image: "{{ monasca_thresh_image_full }}"
     volumes: "{{ monasca_thresh_default_volumes + monasca_thresh_extra_volumes }}"
     dimensions: "{{ monasca_thresh_dimensions }}"
   monasca-notification:
     container_name: monasca_notification
     group: monasca-notification
-    enabled: true
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
     image: "{{ monasca_notification_image_full }}"
     volumes: "{{ monasca_notification_default_volumes + monasca_notification_extra_volumes }}"
     dimensions: "{{ monasca_notification_dimensions }}"
@@ -123,6 +125,11 @@ monasca_influxdb_retention_policy:
 ####################
 # Monasca
 ####################
+
+# NOTE(dszumski): This can be removed in the Xena cycle when the
+# log metrics service is removed
+monasca_enable_log_metrics_service: False
+
 monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ 'api' | kolla_address(host) | put_address_in_context('memcache') }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
@@ -131,7 +138,22 @@ monasca_storm_nimbus_servers: "{% for host in groups['storm-nimbus'] %}'{{ 'api'
 # NOTE(dszumski): Only one NTP server is currently supported by the Monasca Agent plugin
 monasca_ntp_server: "{{ external_ntp_servers | first }}"
 
-# Kafka topics used by Monasca services
+# The default number of Kafka topic partitions. This effectively limits
+# the maximum number of workers per topic, counted over all nodes in the
+# Monasca deployment. For example, if you have a 3 node Monasca
+# deployment, you will by default have 3 instances of Monasca Persister,
+# with each instance having 2 workers by default for the metrics topic.
+# In this case, each worker on the metrics topic will be assigned 5
+# partitions of the metrics topic. If you increase the worker or instance
+# count, you may need to increase the partition count to ensure that all
+# workers can get a share of the work.
+monasca_default_topic_partitions: 30
+
+# The default number of topic replicas. Generally you should not change
+# this.
+monasca_default_topic_replication_factor: "{{ kafka_broker_count if kafka_broker_count|int < 3 else 3 }}"
+
+# Kafka topic names used by Monasca services
 monasca_metrics_topic: "metrics"
 monasca_raw_logs_topic: "logs"
 monasca_transformed_logs_topic: "transformed-logs"
@@ -141,6 +163,47 @@ monasca_alarm_notifications_topic: "alarm-notifications"
 monasca_alarm_notifications_retry_topic: "retry-notifications"
 monasca_periodic_notifications_topic: "60-seconds-notifications"
 
+# Kafka topic configuration. Most users will not need to modify these
+# settings, however for deployments where resources are tightly
+# constrained, or very large deployments where there are many parallel
+# workers, it is worth considering changing them. Note that if you do
+# change these settings, then you will need to manually remove each
+# topic from the Kafka deployment for the change to take effect when
+# the Monasca service is reconfigured.
+monasca_all_topics:
+  - name: "{{ monasca_metrics_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: True
+  - name: "{{ monasca_raw_logs_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: True
+  - name: "{{ monasca_transformed_logs_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: False
+  - name: "{{ monasca_events_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
+  - name: "{{ monasca_alarm_state_transitions_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
+  - name: "{{ monasca_alarm_notifications_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
+  - name: "{{ monasca_alarm_notifications_retry_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
+  - name: "{{ monasca_periodic_notifications_topic }}"
+    partitions: "{{ monasca_default_topic_partitions }}"
+    replication_factor: "{{ monasca_default_topic_replication_factor }}"
+    enabled: "{{ monasca_enable_alerting_pipeline | bool }}"
+
 # NOTE(dszumski): Due to the way monasca-notification is currently
 # implemented it is not recommended to change this period.
 monasca_periodic_notifications_period: 60

ansible/roles/monasca/handlers/main.yml

@@ -14,21 +14,6 @@
   when:
     - kolla_action != "config"
 
-- name: Restart monasca-log-transformer container
-  vars:
-    service_name: "monasca-log-transformer"
-    service: "{{ monasca_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-
 - name: Restart monasca-log-persister container
   vars:
     service_name: "monasca-log-persister"

ansible/roles/monasca/tasks/bootstrap.yml

@@ -61,6 +61,47 @@
   delegate_to: "{{ groups['influxdb'][0] }}"
   when: monasca_influxdb_name not in monasca_influxdb_database.stdout_lines
 
-# NOTE(dszumski): The Monasca APIs write logs and messages to Kafka. Since
-# Kafka has automatic topic generation enabled by default we don't need to
-# create topics here.
+# NOTE(dszumski): Although we can take advantage of automatic topic
+# creation in Kafka, creating the topics manually allows unique settings
+# to be used per topic, rather than the defaults. It also avoids an edge
+# case where services on multiple nodes may race to create topics, and
+# paves the way for enabling things like compression on a per topic basis.
+- name: List monasca kafka topics
+  become: true
+  command: >
+    docker exec kafka /opt/kafka/bin/kafka-topics.sh
+    --zookeeper localhost
+    --list
+  register: kafka_topics
+  run_once: True
+  delegate_to: "{{ groups['kafka'][0] }}"
+
+- name: Create monasca kafka topics if they don't exist
+  become: true
+  command: >
+    docker exec kafka /opt/kafka/bin/kafka-topics.sh
+    --create
+    --topic {{ item.name }}
+    --partitions {{ item.partitions }}
+    --replication-factor {{ item.replication_factor }}
+    --zookeeper localhost
+  run_once: True
+  delegate_to: "{{ groups['kafka'][0] }}"
+  when:
+    - item.name not in kafka_topics.stdout_lines
+    - item.enabled | bool
+  with_items: "{{ monasca_all_topics }}"
+
+- name: Remove monasca kafka topics for disabled services
+  become: true
+  command: >
+    docker exec kafka /opt/kafka/bin/kafka-topics.sh
+    --delete
+    --topic "{{ item.name }}"
+    --zookeeper localhost
+  run_once: True
+  delegate_to: "{{ groups['kafka'][0] }}"
+  when:
+    - item.name in kafka_topics.stdout_lines
+    - not item.enabled | bool
+  with_items: "{{ monasca_all_topics }}"