Merge pull request #28 from kishen-v/patch-os

Introduce tasks to patch OS with latest updates

Author: k8s-ci-robot

kubetest2-tf/data/k8s-ansible/patch-nodes.yaml

@@ -0,0 +1,15 @@
+- name: Update nodes with latest patches and updates
+  hosts:
+    - masters
+    - workers
+  roles:
+    - role: update-node-os
+
+- name: Reboot Kubernetes nodes one-by-one
+  hosts:
+    - masters
+    - workers
+  serial: 1
+  become: yes
+  roles:
+    - role: reboot-sequentially

kubetest2-tf/data/k8s-ansible/roles/reboot-sequentially/tasks/main.yaml

@@ -0,0 +1,61 @@
+- block:
+    - name: Resolve Kubernetes node name from inventory IP
+      shell: |
+        kubectl get nodes -o jsonpath="{range .items[*]}{.metadata.name} {.status.addresses[?(@.type=='InternalIP')].address}{'\n'}{end}" --kubeconfig {{ kubeconfig_path }} |\
+        grep {{ inventory_hostname }} | awk '{print $1}'
+      register: node_name
+      delegate_to: "{{ groups['masters'][0] }}"
+
+    - name: Cordon the kubernetes node
+      shell: |
+        kubectl cordon {{ node_name.stdout }}
+      register: drain_output
+      changed_when: "'already cordoned' not in drain_output.stdout"
+      delegate_to: "{{ groups['masters'][0] }}"
+
+    - name: Check and wait if there are any running jobs that need to complete before draining.
+      shell: |
+        kubectl get pods -n test-pods \
+        --kubeconfig {{ kubeconfig_path }} \
+        --field-selector spec.nodeName={{ node_name.stdout }},status.phase=Running \
+        -o go-template={% raw %}'{{range .items}}{{if or (not .metadata.ownerReferences) (ne (index .metadata.ownerReferences 0).kind "DaemonSet")}}{{.metadata.name}}{{"\n"}} {{end}}{{end}}'{% endraw %} \
+        | wc -l
+      register: running_pod_count
+      retries: 360
+      delay: 30
+      until: running_pod_count.stdout | int == 0
+      delegate_to: "{{ groups['masters'][0] }}"
+
+    - name: Drain Kubernetes Node
+      shell: |
+        kubectl drain {{ node_name.stdout }} --ignore-daemonsets --delete-emptydir-data --kubeconfig {{ kubeconfig_path }}
+      register: drain_output
+      changed_when: "'already cordoned' not in drain_output.stdout"
+      delegate_to: "{{ groups['masters'][0] }}"
+
+    - name: Wait for all pods to be evicted
+      shell: |
+        kubectl get pods -n test-pods --field-selector spec.nodeName={{ node_name.stdout }},status.phase=Running -o go-template='{% raw %}{{range .items}}{{if or (not .metadata.ownerReferences) (ne (index .metadata.ownerReferences 0).kind "DaemonSet")}}{{.metadata.name}}{{"\\n"}}{{end}}{{end}}{% endraw %}' | wc -l
+      register: pods_remaining
+      until: pods_remaining.stdout | int == 0
+      retries: 10
+      delay: 15
+      delegate_to: "{{ groups['masters'][0] }}"
+
+    - name: Reboot node
+      reboot:
+
+    - name: Wait for node to become Ready
+      shell: |
+        kubectl get node {{ node_name.stdout }} --kubeconfig {{ kubeconfig_path }} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+      register: node_status
+      until: node_status.stdout == "True"
+      retries: 20
+      delay: 15
+      delegate_to: "{{ groups['masters'][0] }}"
+
+    - name: Uncordon the node
+      shell: kubectl uncordon {{ node_name.stdout }} --kubeconfig {{ kubeconfig_path }}
+      delegate_to: "{{ groups['masters'][0] }}"
+
+  when: reboot_check is defined and reboot_check.rc == 1

kubetest2-tf/data/k8s-ansible/roles/update-node-os/tasks/main.yaml

@@ -0,0 +1,11 @@
+- name: Update packages and kernel to latest available versions
+  package:
+    name: '*'
+    state: latest
+  when: ansible_pkg_mgr in ['yum', 'dnf']
+
+- name: Check if reboot required
+  shell: needs-restarting -r
+  register: reboot_check
+  ignore_errors: yes
+  when: ansible_distribution in ['CentOS', 'RedHat']