Add node recovery playbook and improve deployment automation

New playbook:
- recover-node.yml: Automates recovery of reimaged/failed nodes
  - Cleans stale K3s node secrets and registrations
  - Handles Longhorn disk UUID mismatches
  - Re-registers nodes with cluster

Role improvements:
- base: Add defaults, enhance iSCSI config, fix hostname persistence
- k3s-prereq: Auto-detect NVMe boot vs eMMC, handle post-migration layout
- k3s-agent: Add service checks, wait for Ready state

Playbook improvements:
- bootstrap.yml: Add pre-flight checks and post-bootstrap verification

Author: jfreed-dev

CHANGELOG.md

@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.8] - 2025-12-26
+
+### Added
+
+- Node recovery playbook (`playbooks/recover-node.yml`)
+  - Cleans up stale K3s node password secrets
+  - Removes stale Longhorn disk entries
+  - Re-registers nodes with cluster
+  - Configures Longhorn storage automatically
+  - Usage: `ansible-playbook playbooks/recover-node.yml --limit node3`
+
+### Changed
+
+- Improved `base` role
+  - Added role defaults for packages and kernel modules
+  - Enhanced iSCSI configuration with socket and initiator setup
+  - Fixed hostname persistence in /etc/hostname
+
+- Improved `k3s-prereq` role
+  - Auto-detects NVMe boot vs eMMC boot scenarios
+  - Handles post-migration partition layout (nvme0n1p1=root, nvme0n1p2=longhorn)
+  - Better idempotency for storage configuration
+
+- Improved `k3s-agent` role
+  - Added service existence check before restart
+  - Waits for node to reach Ready state
+  - Creates node password directory
+
+- Improved `bootstrap.yml` playbook
+  - Added pre-flight checks (memory, architecture, NVMe)
+  - Post-bootstrap verification of iSCSI and storage
+  - Better logging of configuration state
+
 ## [1.1.7] - 2025-12-26
 
 ### Changed
@@ -136,6 +169,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Comprehensive installation guide (INSTALL.md)
 - Implementation documentation (docs/IMPLEMENTATION.md)
 
+[1.1.8]: https://github.com/jfreed-dev/turing-ansible-cluster/compare/v1.1.7...v1.1.8
 [1.1.7]: https://github.com/jfreed-dev/turing-ansible-cluster/compare/v1.1.6...v1.1.7
 [1.1.6]: https://github.com/jfreed-dev/turing-ansible-cluster/compare/v1.1.5...v1.1.6
 [1.1.5]: https://github.com/jfreed-dev/turing-ansible-cluster/compare/v1.1.4...v1.1.5

ansible/playbooks/bootstrap.yml

@@ -1,6 +1,12 @@
 ---
 # Bootstrap nodes with base packages and prerequisites
 # Usage: ansible-playbook -i inventories/server/hosts.yml playbooks/bootstrap.yml
+#
+# This playbook prepares nodes for K3s installation by:
+# - Installing required packages (open-iscsi, nfs-common, etc.)
+# - Configuring kernel modules and sysctl settings
+# - Setting up NVMe storage for Longhorn (if applicable)
+# - Configuring hostname and /etc/hosts
 
 - name: Bootstrap cluster nodes
   hosts: k3s_cluster
@@ -13,7 +19,7 @@
   pre_tasks:
     - name: Display target information
       ansible.builtin.debug:
-        msg: "Bootstrapping {{ inventory_hostname }} ({{ ansible_host }})"
+        msg: "Bootstrapping {{ inventory_hostname }} ({{ ansible_host }}) - NVMe: {{ has_nvme | default(false) }}"
 
     - name: Wait for nodes to be reachable
       ansible.builtin.wait_for_connection:
@@ -22,16 +28,65 @@
     - name: Gather facts
       ansible.builtin.setup:
 
+    # Pre-flight checks
+    - name: Check minimum memory (2GB required)
+      ansible.builtin.assert:
+        that: ansible_memtotal_mb >= 2048
+        fail_msg: "Node {{ inventory_hostname }} has only {{ ansible_memtotal_mb }}MB RAM. Minimum 2048MB required."
+        success_msg: "Memory check passed: {{ ansible_memtotal_mb }}MB"
+
+    - name: Check architecture
+      ansible.builtin.assert:
+        that: ansible_architecture == 'aarch64'
+        fail_msg: "Expected aarch64 architecture, got {{ ansible_architecture }}"
+        success_msg: "Architecture check passed: {{ ansible_architecture }}"
+
+    - name: Check NVMe device exists (if configured)
+      ansible.builtin.stat:
+        path: "{{ nvme_device | default('/dev/nvme0n1') }}"
+      register: bootstrap_nvme_check
+      when: has_nvme | default(false)
+
+    - name: Warn if NVMe configured but not present
+      ansible.builtin.debug:
+        msg: "WARNING: has_nvme=true but {{ nvme_device | default('/dev/nvme0n1') }} not found!"
+      when:
+        - has_nvme | default(false)
+        - not bootstrap_nvme_check.stat.exists | default(false)
+
   roles:
     - base
     - k3s-prereq
 
   post_tasks:
+    - name: Verify iSCSI is running
+      ansible.builtin.command: systemctl is-active iscsid
+      register: bootstrap_iscsid_status
+      changed_when: false
+      failed_when: false
+
+    - name: Display iSCSI status
+      ansible.builtin.debug:
+        msg: "iSCSI status: {{ bootstrap_iscsid_status.stdout }}"
+
+    - name: Verify Longhorn directory exists
+      ansible.builtin.stat:
+        path: "{{ longhorn_path }}"
+      register: bootstrap_longhorn_dir
+
+    - name: Display storage configuration
+      ansible.builtin.debug:
+        msg: |
+          Storage configuration for {{ inventory_hostname }}:
+          - Longhorn path: {{ longhorn_path }} (exists: {{ bootstrap_longhorn_dir.stat.exists }})
+          - NVMe configured: {{ has_nvme | default(false) }}
+          - Boot device: {{ ansible_mounts | selectattr('mount', 'equalto', '/') | map(attribute='device') | first }}
+
     - name: Reboot if required
       ansible.builtin.reboot:
         reboot_timeout: 300
       when: reboot_required | default(false)
 
     - name: Bootstrap complete
       ansible.builtin.debug:
-        msg: "{{ inventory_hostname }} bootstrap complete"
+        msg: "{{ inventory_hostname }} bootstrap complete - ready for K3s installation"