Merge pull request #227 from stfc/kibana_logstash

Add Kibana and Logstash to ChatOps

Author: AlexCK-STFC

chatops_deployment/ansible/group_vars/monitoring/vars.yml

@@ -1,3 +1,4 @@
+---
 grafana_client_id: "{{ vault_grafana_client_id }}"
 grafana_client_secret: "{{ vault_grafana_client_secret }}"
 grafana_admin_password: "{{ vault_grafana_admin_password }}"
@@ -8,3 +9,5 @@ alertmanager_password: "{{ vault_alertmanager_password }}"
 alertmanager_version: "0.28.1"
 prometheus_version: "3.2.1"
 elastic_password: "{{ vault_elastic_password }}"
+kibana_system_password: "{{ vault_kibana_system_password }}"
+logstash_system_password: "{{ vault_logstash_system_password }}"

chatops_deployment/ansible/roles/elastic/handlers/main.yml

@@ -4,3 +4,15 @@
   ansible.builtin.systemd_service:
     name: elasticsearch.service
     state: restarted
+
+- name: Restart Kibana
+  become: true
+  ansible.builtin.systemd_service:
+    name: kibana.service
+    state: restarted
+
+- name: Restart Logstash
+  become: true
+  ansible.builtin.systemd_service:
+    name: logstash.service
+    state: restarted

chatops_deployment/ansible/roles/elastic/tasks/kibana.yml

@@ -0,0 +1,63 @@
+---
+- name: Install prerequisite packages
+  become: true
+  ansible.builtin.apt:
+    pkg:
+      - apt-transport-https
+      - software-properties-common
+      - wget
+    update_cache: true
+
+- name: Create key directory
+  become: true
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
+- name: Add Elasticsearch key and repository to apt
+  become: true
+  block:
+    - name: Add key
+      ansible.builtin.get_url:
+        url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+        dest: /etc/apt/keyrings/elasticsearch.asc
+        mode: "0755"
+
+    - name: Add repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/keyrings/elasticsearch.asc] https://artifacts.elastic.co/packages/9.x/apt stable main"
+        state: present
+
+- name: Install Kibana
+  become: true
+  ansible.builtin.apt:
+    name: kibana
+    state: latest # noqa: package-latest
+    update_cache: true
+
+- name: Template kibana config
+  become: true
+  ansible.builtin.template:
+    src: kibana.yml.j2
+    dest: "/etc/kibana/kibana.yml"
+    owner: root
+    group: kibana
+    mode: "0640"
+  notify:
+    - Restart Kibana
+
+- name: Copy certificate and key
+  become: true
+  ansible.builtin.copy:
+    src: "./SSL/{{ item }}"
+    dest: "/etc/kibana/{{ item }}"
+    owner: root
+    group: kibana
+    mode: "0440"
+  notify:
+    - Restart Kibana
+  loop:
+    - kibana.key
+    - kibana.crt
+    - elasticsearch.crt

chatops_deployment/ansible/roles/elastic/tasks/logstash.yml

@@ -0,0 +1,63 @@
+---
+- name: Install prerequisite packages
+  become: true
+  ansible.builtin.apt:
+    pkg:
+      - apt-transport-https
+      - software-properties-common
+      - wget
+    update_cache: true
+
+- name: Create key directory
+  become: true
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
+- name: Add Elasticsearch key and repository to apt
+  become: true
+  block:
+    - name: Add key
+      ansible.builtin.get_url:
+        url: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+        dest: /etc/apt/keyrings/elasticsearch.asc
+        mode: "0755"
+
+    - name: Add repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/keyrings/elasticsearch.asc] https://artifacts.elastic.co/packages/9.x/apt stable main"
+        state: present
+
+- name: Install Logstash
+  become: true
+  ansible.builtin.apt:
+    name: logstash
+    state: latest # noqa: package-latest
+    update_cache: true
+
+- name: Template logstash config
+  become: true
+  ansible.builtin.template:
+    src: logstash.conf.j2
+    dest: "/etc/logstash/logstash.conf"
+    owner: root
+    group: logstash
+    mode: "0640"
+  notify:
+    - Restart Logstash
+
+- name: Copy certificate and key
+  become: true
+  ansible.builtin.copy:
+    src: "./{{ env }}_ssl/{{ item }}"
+    dest: "/etc/logstash/{{ item }}"
+    owner: root
+    group: logstash
+    mode: "0440"
+  notify:
+    - Restart Logstash
+  loop:
+    - logstash.key
+    - logstash.crt
+    - elasticsearch.crt

chatops_deployment/ansible/roles/elastic/tasks/main.yml

@@ -3,3 +3,13 @@
   ansible.builtin.import_tasks: elasticsearch.yml
   tags:
     - elasticsearch
+
+- name: Install Kibana
+  ansible.builtin.import_tasks: kibana.yml
+  tags:
+    - kibana
+
+- name: Install Logstash
+  ansible.builtin.import_tasks: logstash.yml
+  tags:
+    - logstash

chatops_deployment/ansible/roles/elastic/templates/kibana.yml.j2

@@ -0,0 +1,12 @@
+server.host: {{ inventory_hostname }}
+server.port: 5601
+server.publicBaseUrl: https://kibana.{{ domain }}:443
+elasticsearch.hosts: ["https://localhost:9200"]
+elasticsearch.username: kibana_system
+elasticsearch.password: "{{ kibana_system_password }}"
+
+server.ssl:
+  enabled: true
+  key: /etc/kibana/kibana.key
+  certificate: /etc/kibana/kibana.crt
+elasticsearch.ssl.certificateAuthorities: /etc/kibana/elasticsearch.crt

chatops_deployment/ansible/roles/elastic/templates/logstash.conf.j2

@@ -0,0 +1,37 @@
+input {
+  beats {
+    port => 5044
+    host => "0.0.0.0"
+    ssl_enabled => true
+    ssl_certificate => "/etc/logstash/logstash.crt"
+    ssl_key => "/etc/logstash/logstash.key"
+  }
+}
+
+filter {
+  if [service][name] == "chatops" {
+    grok {
+      match => {
+        "message" => [
+          "\[%{TIMESTAMP_ISO8601:timestamp}\] %{LOGLEVEL:level} in %{DATA:module}: %{GREEDYDATA:log_message}"
+        ]
+      }
+    }
+
+    date {
+      match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
+      target => "@timestamp"
+      remove_field => ["timestamp"]
+    }
+  }
+}
+
+output {
+  elasticsearch {
+  hosts => ["https://127.0.0.1:9200"]
+  user => "elastic"
+  password => "{{ elastic_password }}"
+  ssl_enabled => true
+  ssl_certificate_authorities => "/etc/logstash/elasticsearch.crt"
+  }
+}

chatops_deployment/ansible/roles/haproxy/tasks/certbot.yml

@@ -41,7 +41,7 @@
   become: true
   ansible.builtin.command: |
     certbot certonly --standalone --non-interactive --agree-tos --expand --domains \
-    {{ domain }},chatops.{{ domain }},prometheus.{{ domain }},grafana.{{ domain }},alertmanager.{{ domain }} \
+    {{ domain }},chatops.{{ domain }},prometheus.{{ domain }},grafana.{{ domain }},alertmanager.{{ domain }},kibana.{{ domain }} \
     -m cloud-support@stfc.ac.uk
   register: generate_cert
   changed_when: generate_cert.rc == 0

chatops_deployment/ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -47,10 +47,12 @@ acl grafana hdr_sub(host) -i grafana.{{ domain }}
 acl prometheus hdr_sub(host) -i prometheus.{{ domain }}
 acl alertmanager hdr_sub(host) -i alertmanager.{{ domain }}
 acl chatops_sub hdr_sub(host) -i chatops.{{ domain }}
+acl kibana hdr_sub(host) -i kibana.{{ domain }}
 acl chatops hdr_sub(host) -i {{ domain }}
 
 use_backend GRAFANA if grafana
 use_backend PROMETHEUS if prometheus
+use_backend KIBANA if kibana
 use_backend ALERTMANAGER if alertmanager
 use_backend CHATOPS if chatops || chatops_sub
 
@@ -89,3 +91,8 @@ server chatops_{{ loop.index }} {{ address }}:3000 check backup
 server chatops_{{ loop.index }} {{ address }}:3000 check
 {% endif %}
 {% endfor %}
+
+backend KIBANA
+{% for address in groups['elastic'] %}
+server elastic_{{ loop.index }} {{ address }}:5601 check
+{% endfor %}