allow email-notifications on the bastion host

docs/email-notifications.md

@@ -0,0 +1,120 @@
+# Helper Node email notifications installation
+
+This quickstart will get postfix up and running on the bastion host so that you can test OpenShift's monitoring stack Alertmanager notifications.
+
+~~~
+ansible-playbook -e @vars.yaml tasks/main.yml -v -e postfix_install=true
+~~~
+
+How to update the alertmanager configuration?
+
+1) extract the alertmanager configuration => 
+
+~~~
+oc get secret alertmanager-main -o go-template='{{ index .data "alertmanager.yaml"|base64decode}}' -n openshift-monitoring > alertmanager.yaml
+~~~
+
+2) Edit the `alertmanager.yaml` file with your required changes.
+
+
+3) update the alertmanager configuration =>
+
+~~~
+oc create secret generic alertmanager-main --from-file=alertmanager.yaml --dry-run=client -o=yaml -n openshift-monitoring | oc replace secret --filename=-
+~~~
+
+---
+# Helper Node Alertmanager notification configuration
+
+1) How to send the alerts to the multiple receivers in RHOCP 4? - Red Hat Alertmanager is not sending the critical alerts Alertmanager was configured to send the critical alerts but it's not working Alertmanager is not sending the alerts to all the routes It was configured.
+* [link](https://access.redhat.com/solutions/6612991) <= sending alerts to multiple receivers
+
+2) Send dummy alerts to alertmanager in OpenShift 4 - A receiver has to be tested, for instance, with a Critical alert. A real critical alert cannot be forced in production.
+* [link](https://access.redhat.com/solutions/6828481) <= sending dummy alerts to alertmanager
+
+3) To test the email receiver by generating a critical alert, run the following. **Please note**, it is imperative to read the links in the previous steps 1,2.
+
+~~~
+oc exec alertmanager-main-0 -n openshift-monitoring -- amtool alert add --alertmanager.url http://localhost:9093 alertname=myalarm  --start="2022-03-18T00:00:00-00:00" severity=critical
+~~~
+
+4) To verify that you received an alert, on the bastion host run the following.
+
+~~~
+cat /var/spool/mail/incoming
+~~~
+
+---
+
+Working example of the `alertmanager.yaml` configuration. Replace `bastion.ocp4.example.com` with the actual hostname that you used in the `vars.yaml` => `helper.name`
+
+* Entries that should be left as-is:
+  * smtp_smarthost: 'smtp.ocp4.example.com:25'
+  * smarthost: 'smtp.ocp4.example.com:25'
+
+~~~
+global:
+  resolve_timeout: 5m
+  smtp_from: [email protected]
+  smtp_smarthost: 'smtp.ocp4.example.com:25'
+  smtp_hello: [email protected]
+  smtp_require_tls: false
+  smtp_auth_username: 'user'
+  smtp_auth_password: 'password'
+inhibit_rules:
+  - equal:
+      - namespace
+      - alertname
+    source_matchers:
+      - severity = critical
+    target_matchers:
+      - severity =~ warning|info
+  - equal:
+      - namespace
+      - alertname
+    source_matchers:
+      - severity = warning
+    target_matchers:
+      - severity = info
+  - equal:
+      - namespace
+    source_matchers:
+      - alertname = InfoInhibitor
+    target_matchers:
+      - severity = info
+receivers:
+  - name: Critical
+    email_configs:
+      - to: [email protected]
+        from: [email protected]
+        smarthost: 'smtp.ocp4.example.com:25'
+        hello: ocp4.example.com
+        require_tls: false
+  - name: Default
+  - name: 'null'
+  - name: Watchdog
+  - name: Warning
+    email_configs:
+      - to: [email protected]
+route:
+  group_by:
+    - namespace
+  group_interval: 5m
+  group_wait: 30s
+  receiver: Default
+  repeat_interval: 12h
+  routes:
+    - matchers:
+        - alertname = Watchdog
+      receiver: Watchdog
+    - matchers:
+        - alertname = InfoInhibitor
+      receiver: 'null'
+    - receiver: Critical
+      continue: true
+      matchers:
+        - severity = critical
+    - receiver: Warning
+      matchers:
+        - severity = warning
+~~~

handlers/main.yml

@@ -30,3 +30,6 @@
     name: nfs-server
     state: restarted
 
+- name: Check postfix
+  command: postfix check
+  when: postfix_check

tasks/main.yml

@@ -24,7 +24,9 @@
     when: ssh_gen_key
 
   - name: set setup facts
-    include: set_facts_.yaml
+    # ERROR! [DEPRECATED]: ansible.builtin.include has been removed. Use include_tasks or import_tasks instead. This feature was removed from ansible-core in a release after 2023-05-16. Please update your playbooks.
+    # CentOS Stream 8
+    import_tasks: set_facts_.yaml
 
   - name: Install needed packages
     package:
@@ -657,8 +659,13 @@
       state: stopped
     when: (dhcp.dns is defined and dhcp.dns != helper.ipaddr) or (dns.lb_ipaddr is defined and dns.lb_ipaddr != helper.ipaddr)
 
+  - name: Install postfix to receive OpenShift alerts
+    include_tasks: postfix.yaml
+    when: (postfix_install and ansible_facts['os_family'] == "RedHat" and ansible_facts['distribution_major_version'] >= "7") or
+          (postfix_install and ansible_facts['os_family'] == "CentOS" and ansible_facts['distribution_major_version'] >= "7")
+    tags: postfix
+
   - name:  Information about this install
     debug:
       msg:
-        - "Please run /usr/local/bin/helpernodecheck for information"
-
+        - "Please run /usr/local/bin/helpernodecheck for information"

tasks/postfix.yaml

@@ -0,0 +1,191 @@
+---
+# Distribution defaults are different from the postfix defaults, hence need to
+# reinstall the package
+- name: Remove postfix package(s) to restore config postfix_packages
+  block:
+    # It is assumed that the only package providing config files that might
+    # be modified is postfix - if this is not so, then additional
+    # packages need to be added to this check
+    - name: Get status of package(s) postfix_packages
+      command: rpm -V {{ item }}  # noqa command-instead-of-module
+      loop: "{{ postfix_packages }}"
+      register: postfix_packages_status
+      failed_when: false
+      changed_when: false
+
+    - name: Remove package(s) to reset original confs postfix_packages
+      package:
+        name: "{{ postfix_packages }}"
+        state: absent
+      register: postfix_erased
+      when: postfix_packages_status.results |
+        rejectattr('stdout', 'match', '^package .* is not installed') |
+        selectattr('stdout', 'search', ' /etc/postfix/.*[.]cf($|\n)') |
+        list | length > 0
+
+- name: Remove configuration for replacement
+  block:
+    # It is assumed that the only package providing config files that might
+    # be modified is postfix - if this is not so, then additional
+    # packages need to be added to this check
+    - name: Get status of config files
+      command: rpm -V --nomtime {{ item }}  # noqa command-instead-of-module
+      loop: "{{ postfix_packages }}"
+      register: postfix_packages_status
+      failed_when: false
+      changed_when: false
+
+    - name: Remove config files - make empty
+      copy:
+        dest: "{{ item }}"
+        content: "\n"  # to make postconf -e happy
+        mode: preserve
+      loop: "{{ mod_cf_files }}"
+      vars:
+        mod_cf_files: "{{ postfix_packages_status.results |
+          map(attribute='stdout') | map('default', '') |
+          map('regex_findall', ' (/etc/postfix/.*[.]cf)(?:$|\n)') | select |
+          flatten | list }}"
+
+- name: Install Postfix & mailx
+  package:
+    name: "{{ postfix_packages }}"
+    state: present
+
+- name: Get default database type from postconf
+  command: postconf -h default_database_type
+  changed_when: false
+  register: postfix_register_dbtype
+
+- name: Set postfix_default_database_type
+  set_fact:
+    postfix_default_database_type: "{{ postfix_register_dbtype.stdout | trim }}"
+
+- name: get service facts
+  ansible.builtin.service_facts:
+
+- name: Enable Postfix
+  ansible.builtin.systemd_service:
+    state: started
+    name: postfix
+    enabled: yes
+  when: ansible_facts['services']['postfix.service'] is defined
+
+- name: Get current config
+  command: postconf
+  register: postfix_register_config
+  changed_when: false
+  when: postfix_conf | dict2items | rejectattr('key', 'match', '^previous$')
+    | list | items2dict | d({}) | length > 0
+
+- name: Check given config against current config
+  changed_when: false
+  when:
+    - postfix_conf | dict2items | rejectattr('key', 'match', '^previous$')
+      | list | items2dict | d({}) | length > 0
+    - postfix_register_config is defined
+    - postfix_register_config.stdout_lines is defined
+  set_fact:
+    postfix_has_config_changed: |-
+      {% for key, val in postfix_conf.items() %}
+      {%   if key != 'previous' %}
+      {%     set space = ' ' if val else '' %}
+      {%     set itemstr = '{} ={}{}'.format(key, space, val) %}
+      {%     if itemstr not in postfix_register_config.stdout_lines %}
+      True itemstr {{ itemstr }} is not in postfix_register_config
+      {%     endif %}
+      {%   endif %}
+      {% endfor %}
+
+- name: Configure additional files
+  copy:
+    content: "{{ file['content'] }}"
+    dest: /etc/postfix/{{ file['name'] }}
+    owner: root
+    group: root
+    mode: '0640'
+  loop: "{{ postfix_files }}"
+  register: postfix_postmap_files
+  no_log: true
+  loop_control:
+    loop_var: file
+  notify:
+    - Check postfix
+
+- name: Postmap files
+  command: postmap {{ result["dest"] | quote }}
+  when:
+    - result["changed"]
+    - result["file"]["postmap"] | d(false)
+  no_log: true
+  changed_when: true
+  loop: "{{ postfix_postmap_files['results'] }}"
+  loop_control:
+    loop_var: result
+
+- name: Add smtp users
+  block:
+  - name: Add the users to send & receive email notifications from OpenShift's Alertmanager 
+    ansible.builtin.user:
+      name: "{{ item }}"
+      shell: /usr/sbin/nologin
+      password: '!'
+    with_items:
+      - openshift
+      - incoming
+
+- name: Apply changes
+  when: postfix_has_config_changed | d("") is search("True")
+  block:
+    - name: Gather facts for ansible_date_time
+      setup:
+        filter:
+          - ansible_date_time
+      when: postfix_backup | bool
+
+    - name: Backup configuration
+      copy:
+        remote_src: true
+        src: /etc/postfix/main.cf
+        dest: /etc/postfix/main.cf.{{ postfix_backup_multiple |
+          ternary(ansible_date_time.iso8601, "backup") }}
+        mode: "0644"
+      when: postfix_backup 
+
+    - name: Ensure Last modified header is absent
+      lineinfile:
+        path: /etc/postfix/main.cf
+        regexp: '# Last modified:'
+        state: absent
+
+    - name: Configure Postfix
+      command: postconf -e {{ item.key | quote }}={{ item.value | quote }}
+      notify:
+        - Check postfix
+      with_dict: "{{ postfix_conf }}"
+      when:
+        - item.key not in ['previous']
+        - postfix_has_config_changed
+          | d("") is search("True itemstr " ~ item.key)
+      changed_when:
+        - item.key not in ['previous']
+        - postfix_has_config_changed
+          | d("") is search("True itemstr " ~ item.key)
+
+- name: Restart postfix, in all cases
+  ansible.builtin.service:
+    name: postfix
+    state: restarted
+
+- ansible.builtin.wait_for:
+    host: "smtp.{{ dns.clusterid }}.{{ dns.domain | lower }}"
+    port: 25
+
+- name: Sending an email using Ansible
+  community.general.mail:
+    host: "smtp.{{ dns.clusterid }}.{{ dns.domain | lower }}"
+    port: 25
+    sender: "openshift@{{ helper.name }}.{{ dns.clusterid }}.{{ dns.domain | lower }}"
+    to: "incoming@{{ helper.name }}.{{ dns.clusterid }}.{{ dns.domain | lower }}"
+    subject: Email By Ansible
+    body: Test successful

tasks/set_facts_.yaml

@@ -130,3 +130,44 @@
         - rpcbind
         - nfs-server
   when: ansible_distribution_major_version >= "8"
+
+- block:
+  - set_fact:
+      postfix_conf: 
+        myorigin: "$mydomain"
+        inet_interfaces: "all"
+        inet_protocols: "all"
+        mydestination: "$myhostname, localhost.$mydomain, localhost"
+        mynetworks_style: "subnet"
+        mynetworks: "127.0.0.0/8 192.168.7.0/24"
+        relay_domains: ""
+
+  - set_fact:
+      # Additional config maps/files, e.g.:
+      # // for all options see link https://github.com/linux-system-roles/postfix //
+      # // this addition to the ocp4-helpernode is the hassle-free option. //
+      # postfix_files:
+      #   - name: 'sasl_passwd'
+      #     content: 'smtp.example.com [email protected]:myFirstPassword
+      #     postmap: true
+      #   - name: 'sender_canonical_maps'
+      #     content: '/.+/    [email protected]'
+      postfix_files: []
+
+  # Whether to run 'postfix check' before it's started
+  - set_fact:
+      postfix_check: true
+
+  # Whether to make backup of current config
+  - set_fact:
+      postfix_backup: false
+
+  # List of default rpm packages to install.
+  - set_fact:
+      postfix_packages:
+        - postfix
+        - mailx
+
+  - set_fact:
+      postfix_smtp_services:
+        - smtp