Extend support for logging 5.9 and above versions

Signed-off-by: Jeevan-Darapu <[email protected]>

Author: Jeevaann

examples/all.yaml

@@ -159,8 +159,10 @@ ocp_cluster_logging: false
 cluster_log_forwarder: false
 cluster_logging_channel: ""
 elastic_search_channel: ""
+loki_channel: ""
 elasticsearch_clf_cs: ""  # brew.registry.redhat.io/rh-osbs/iib:111110
 clusterlogging_clf_cs: "" # brew.registry.redhat.io/rh-osbs/iib:11111
+loki_clf_cs: ""  # brew.registry.redhat.io/rh-osbs/iib:111112
 log_label: ""
 elasticsearch_url: ""
 syslog_url: ""

examples/ocp_cluster_logging_vars.yml

@@ -3,14 +3,16 @@ ocp_cluster_logging: false
 cluster_log_forwarder: false
 cluster_logging_channel: ""
 elastic_search_channel: ""
+loki_channel: ""
 elasticsearch_clf_cs: ""  # brew.registry.redhat.io/rh-osbs/iib:111110
 clusterlogging_clf_cs: "" # brew.registry.redhat.io/rh-osbs/iib:11111
+loki_clf_cs: ""  # brew.registry.redhat.io/rh-osbs/iib:111112
 log_label: ""
 elasticsearch_url: ""
 syslog_url: ""
 fluentd_url: ""
 kafka_url: ""
-kafka_path: "" # Location of kafka on external vm ex. /root/kafka/kafka_2.13-2.7.0/bin 
+kafka_path: "" # Location of kafka on external vm ex. /usr/local/kafka/bin
 loki_url: ""
 cloudwatch_secret: ""
 aws_region: ""

group_vars/all.yml

@@ -46,14 +46,16 @@ ocp_cluster_logging: false
 cluster_log_forwarder: false
 cluster_logging_channel: ""
 elastic_search_channel: ""
+loki_channel: ""
 elasticsearch_clf_cs: ""  # brew.registry.redhat.io/rh-osbs/iib:111110
 clusterlogging_clf_cs: "" # brew.registry.redhat.io/rh-osbs/iib:11111
+loki_clf_cs: ""  # brew.registry.redhat.io/rh-osbs/iib:111112
 log_label: ""
 elasticsearch_url: ""
 syslog_url: ""
 fluentd_url: ""
 kafka_url: ""
-kafka_path: "" # Location of kafka on external vm ex. /root/kafka/kafka_2.13-2.7.0/bin 
+kafka_path: "" # Location of kafka on external vm ex. /usr/local/kafka/bin
 loki_url: ""
 cloudwatch_secret: ""
 aws_region: ""

playbooks/roles/ocp-cluster-logging/files/clf-cleanup.yml

@@ -16,27 +16,27 @@
     register: icsp_cnt
 
   - name: Check if the CatalogSource exists
-    shell:  oc get CatalogSource -n openshift-marketplace | grep "cluster-logging\|elasticsearch" |wc -l
-    register: catalog_src_cnt
+    shell: oc get CatalogSource -n openshift-marketplace | grep "cluster-logging\|elasticsearch\|loki" | awk 'NR==1{print $1}''NR==2{print $1}''NR==3{print $1}' | xargs
+    register: catalog_src
 
   - name: Delete ImageContentSourcePolicy if it exists
     shell: oc delete ImageContentSourcePolicy brew-registry
     when: icsp_cnt.stdout|int == 1
 
   - name: Delete CatalogSource if it exists
-    shell: oc delete CatalogSource elasticsearch cluster-logging -n openshift-marketplace
-    when: catalog_src_cnt.stdout|int == 2
+    shell: oc delete CatalogSource -n openshift-marketplace  {{ catalog_src.stdout }}
+    when: catalog_src.stdout|length > 0
 
   - name: Set disableAllDefaultSources to false
     shell: |
       oc patch operatorhub.config.openshift.io/cluster -p='{"spec":{"disableAllDefaultSources":false}}' --type=merge
 
 # Check ClusterLogForwarder instance and delete if it exist
-- name: Check if the ClutserLogForwarder instance exist
+- name: Check if the ClusterLogForwarder instance exist
   shell: oc get ClusterLogForwarder -n openshift-logging | wc -l
   register: check_clf
 
-- name: Delete ClutserLogForwarder instance if it exists
+- name: Delete ClusterLogForwarder instance if it exists
   shell: oc delete ClusterLogForwarder instance -n openshift-logging
   when: check_clf.stdout|int != 0
 
@@ -49,6 +49,24 @@
   shell: oc delete ClusterLogging instance -n openshift-logging
   when: check_cl.stdout|int != 0
 
+# Check Lokistack instance and delete if it exists
+- name: Check if the lokistack instance exists
+  shell: oc get lokistack -n openshift-logging | grep lokistack | awk 'NR==1{print $1}'
+  register: check_lokistack
+
+- name: Delete lokistack instance if it exists
+  shell: oc delete lokistack {{ check_lokistack.stdout }} -n openshift-logging
+  when: check_lokistack.stdout|length > 0
+
+# Check and delete service account for log collection
+- name: Check the service account exists
+  shell: oc get serviceaccount -n openshift-logging | grep logcollector | awk 'NR==1{print $1}'
+  register: log_sa
+
+- name: Delete the service account
+  shell: oc delete serviceaccount {{ log_sa.stdout }} -n openshift-logging
+  when: log_sa.stdout|length > 0
+
 # Check and delete Elasticsearch subscription if it exists
 - name: Check if the Elasticsearch subscription exists
   shell: oc get subscription -n openshift-operators-redhat | grep elasticsearch-operator | wc -l
@@ -85,3 +103,37 @@
   shell: oc delete clusterserviceversion {{ cluster_logging_csv.stdout }} -n openshift-logging 
   when: cluster_logging_csv.stdout|length > 0
 
+# Check and delete lokistack subscription if it exists
+- name: Check if the Loki subscription exists
+  shell: oc get subscription -n openshift-operators-redhat | grep loki-operator | wc -l
+  register: loki_subs
+
+- name: Delete Loki subscription if it exists
+  shell: oc delete subscription loki-operator -n openshift-operators-redhat
+  when: loki_subs.stdout|int != 0
+
+# Check and delete Loki operator if it exists
+- name: Check if the Loki operator exists
+  shell:  oc get csv  -n openshift-operators-redhat | grep loki-operator | awk 'NR==1{print $1}'
+  register: loki_csv
+
+- name: Delete Loki operator if it exists
+  shell: oc delete clusterserviceversion {{ loki_csv.stdout }} -n openshift-operators-redhat
+  when: loki_csv.stdout|length > 0
+
+# Delete the Operator groups
+- name: Check CLO operator group
+  shell: oc get operatorgroup -n openshift-logging | grep cluster-logging | awk 'NR==1{print $1}'
+  register: clo_og
+
+- name: Delete CLO operator group
+  shell: oc delete operatorgroup {{ clo_og.stdout }} -n openshift-logging
+  when: clo_og.stdout|length > 0
+
+- name: Check Loki operator group
+  shell: oc get operatorgroup -n openshift-operators-redhat | grep openshift-operators-redhat | awk 'NR==1{print $1}'
+  register: loki_og
+
+- name: Delete Loki operator group
+  shell: oc delete operatorgroup {{ loki_og.stdout }} -n openshift-operators-redhat
+  when: loki_og.stdout|length > 0

playbooks/roles/ocp-cluster-logging/files/clusterlogforwarder.yml

@@ -23,9 +23,21 @@
     src: "{{ role_path }}/templates/clf-instance.yml.j2"
     dest: "{{ role_path }}/files/clf-instance.yml"
   delegate_to: localhost
+  when: clo_version | float < 6.0
+
+- name: Generating ClusterLogForwarder file
+  template:
+    src: "{{ role_path }}/templates/new-clf-instance.yml.j2"
+    dest: "{{ role_path }}/files/new-clf-instance.yml"
+  delegate_to: localhost
+  when: clo_version | float >= 6.0
 
 # Creating ClusterLogForwarder custom resource
 - include_tasks: "{{ role_path }}/files/clf-instance.yml"
+  when: clo_version | float < 6.0
+
+- include_tasks: "{{ role_path }}/files/new-clf-instance.yml"
+  when: clo_version | float >= 6.0
 
 # Check if the pods are in good state
 - name: Check the logging pods are in good state
@@ -65,23 +77,28 @@
     - "loki"
     - "cloudwatch"
     - "kibana-ldap"
+    - "lokistack"
 
 - set_fact: 
     syslog_server_logfile: "/var/log/messages"
     external_server_logs_path: "/root/clf_logs"
 
+- name: Pause for 2 minutes to get new logs
+  pause:
+    minutes: 2
+
 - name: Fetch the logs from external instances 
   block:
     # Save the logs on external Kafka system and fetch on bastion
     - block:
       - name: Save the logs on Kafka server
         shell: |
           mkdir -p {{ external_server_logs_path }}/kafka
-          {{ kafka_path }}/kafka-console-consumer.sh --bootstrap-server {{ kafka_host }}:9092 --topic {{ log_labels }}-audit --max-messages 10 > {{ external_server_logs_path }}/kafka/audit.txt
-          {{ kafka_path }}/kafka-console-consumer.sh --bootstrap-server {{ kafka_host }}:9092 --topic {{ log_labels }}-infrastructure --max-messages 10 > {{ external_server_logs_path }}/kafka/infrastructure.txt
-          {{ kafka_path }}/kafka-console-consumer.sh --bootstrap-server {{ kafka_host }}:9092 --topic {{ log_labels }}-application --max-messages 10 > {{ external_server_logs_path }}/kafka/application.txt
-        async: 30
-        poll: 5
+          {{ kafka_path }}/kafka-console-consumer.sh --bootstrap-server {{ kafka_host }}:9092 --topic {{ app_log_label }} --max-messages 10 > {{ external_server_logs_path }}/kafka/application.txt
+          {{ kafka_path }}/kafka-console-consumer.sh --bootstrap-server {{ kafka_host }}:9092 --topic {{ audit_log_label }} --max-messages 10 > {{ external_server_logs_path }}/kafka/audit.txt
+          {{ kafka_path }}/kafka-console-consumer.sh --bootstrap-server {{ kafka_host }}:9092 --topic {{ infra_log_label }} --max-messages 10 > {{ external_server_logs_path }}/kafka/infrastructure.txt
+        async: 120
+        poll: 10
 
       - name: Copy the logs file from Kafka to bastion
         fetch:
@@ -95,6 +112,26 @@
       delegate_to: kafka
       when: kafka_server_url is defined
 
+    #Make lokistack.sh file executable
+    - name: Give executable permissions for lokistack.sh file
+      file:
+        path: "{{ role_path }}/files/lokistack.sh"
+        mode: '0555'
+        state: file
+
+    # Check if lokistack exists
+    - name: Check the Lokistack
+      shell: oc get lokistack -n openshift-logging | grep lokistack | awk 'NR==1{print $1}'
+      register: lokistack_present
+
+    # Save the logs on external lokistack and fetch on bastion
+    - name: Save the logs for lokistack instance
+      shell: |
+        echo "{{ lokistack_present.stdout }}"
+        mkdir -p {{ cl_log_dir }}/lokistack
+        {{ role_path }}/files/lokistack.sh {{ lokistack_present.stdout }}
+      when: lokistack_present.stdout|length > 0
+
     # Save the logs on external Syslog system and fecth on bastion
     - block:
       - name: Save the logs on external Syslog instance
@@ -123,22 +160,40 @@
     # Fetch logs from Elasticsearch
     - name: Fetch Logs from Elasticsearch
       shell: |
-        curl -XGET "{{ elasticsearch_server_url }}/infra*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.logs":"{{ log_labels }}-infrastructure"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/infrastructure.txt
-        curl -XGET "{{ elasticsearch_server_url }}/audit*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.logs":"{{ log_labels }}-audit"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/audit.txt
-        curl -XGET "{{ elasticsearch_server_url }}/app*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.logs":"{{ log_labels }}-application"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/application.txt
-      when: elasticsearch_server_url is defined
+        curl -XGET "{{ elasticsearch_server_url }}/infra*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.label":"{{ infra_log_label }}"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/infrastructure.txt
+        curl -XGET "{{ elasticsearch_server_url }}/audit*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.label":"{{ audit_log_label }}"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/audit.txt
+        curl -XGET "{{ elasticsearch_server_url }}/app*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.label":"{{ app_log_label }}"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/application.txt
+      when: 
+        - elasticsearch_server_url is defined
+        - clo_version | float >= 6.0
+
+    - name: Fetch Logs from Elasticsearch
+      shell: |
+        curl -XGET "{{ elasticsearch_server_url }}/infra*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.logs":"{{ infra_log_label }}"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/infrastructure.txt
+        curl -XGET "{{ elasticsearch_server_url }}/audit*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.logs":"{{ audit_log_label }}"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/audit.txt
+        curl -XGET "{{ elasticsearch_server_url }}/app*/_search" -H 'Content-Type: application/json' -d '{ "query": { "bool": { "must": [ { "match":{"openshift.labels.logs":"{{ app_log_label }}"} } ] } } }' > {{ cl_log_dir }}/elasticsearch/application.txt
+      when:
+        - elasticsearch_server_url is defined
+        - clo_version | float <= 5.9
 
     # Fetch logs from Loki
     - name: Fetch logs from Loki
       shell: |
         curl -G -s "{{ loki_server_url }}/api/prom/query" --data-urlencode 'query={log_type="infrastructure"}' > {{ cl_log_dir }}/loki/infrastructure.txt
         curl -G -s "{{ loki_server_url }}/api/prom/query" --data-urlencode 'query={log_type="audit"}' > {{ cl_log_dir }}/loki/audit.txt
         curl -G -s "{{ loki_server_url }}/api/prom/query" --data-urlencode 'query={log_type="application"}' > {{ cl_log_dir }}/loki/application.txt
+      async: 120
+      poll: 10
       when: loki_server_url is defined
 
     # Deleting CLF Custom Resource instance because Fluentd and CloudWatch stores the logs on their system
     - name: Delete ClusterLogForwarder
       shell: oc delete ClusterLogForwarder instance -n openshift-logging
+      when: clo_version | float <= 5.9
+
+    - name: Delete ClusterLogForwarder
+      shell: oc delete obsclf collector -n openshift-logging
+      when: clo_version | float >= 6.0
 
     - name: Check the logging pods are restarting
       shell: oc get pods -n openshift-logging --no-headers | awk '{if ($3 == "Terminating" ) print $1}' | wc -l
@@ -216,8 +271,19 @@
 
     - name: Elasticsearch clean up
       shell: |
-        curl -X POST "{{ elasticsearch_server_url }}/audit*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.logs":"{{ log_labels }}-audit" } }}'
-        curl -X POST "{{ elasticsearch_server_url }}/app*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.logs":"{{ log_labels }}-application" } }}'
-        curl -X POST "{{ elasticsearch_server_url }}/infra*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.logs":"{{ log_labels }}-infrastructure" } }}'
-      when: elasticsearch_server_url is defined
+        curl -X POST "{{ elasticsearch_server_url }}/audit*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.label":"{{ audit_log_label }}" } }}'
+        curl -X POST "{{ elasticsearch_server_url }}/app*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.label":"{{ app_log_label }}" } }}'
+        curl -X POST "{{ elasticsearch_server_url }}/infra*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.label":"{{ infra_log_label }}" } }}'
+      when: 
+        - elasticsearch_server_url is defined
+        - clo_version | float >= 6.0
+
+    - name: Elasticsearch clean up
+      shell: |
+        curl -X POST "{{ elasticsearch_server_url }}/audit*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.logs":"{{ audit_log_label }}" } }}'
+        curl -X POST "{{ elasticsearch_server_url }}/app*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.logs":"{{ app_log_label }}" } }}'
+        curl -X POST "{{ elasticsearch_server_url }}/infra*/_delete_by_query?pretty" -H 'Content-Type: application/json' -d '{ "query": { "match": { "openshift.labels.logs":"{{ infra_log_label }}" } }}'
+      when:
+        - elasticsearch_server_url is defined
+        - clo_version | float <= 5.9
   ignore_errors: yes