Add badfish support for iDRAC cleanup operations (#745)

This commit adds support for using badfish container to perform iDRAC
cleanup operations on Dell hardware. Badfish is used to clear the iDRAC
job queue and reset the iDRAC service to improve stability during boot
operations. It does not replace redfish_command or URI modules, which
continue to be used for standard Redfish operations.

Changes:
- Created new 'badfish' Ansible role with install.yml and call.yml tasks
- Added 'reset_idrac' parameter to control badfish-based iDRAC cleanup
  operations
- Integrated badfish container installation into bastion bootstrap process
- Updated boot-iso/dell.yml to use badfish for:
  - Clearing iDRAC job queue (always executed)
  - Resetting iDRAC service (when reset_idrac is enabled)
  - Waiting for iDRAC to be available after reset
- Replaced fixed pause with wait_for module to verify host power down
- All badfish operations use quay.io/quads/badfish container image

The badfish role provides a reusable call.yml task file that accepts
badfish_host, badfish_user, badfish_password, and badfish_args parameters,
making it easy to call badfish commands from other roles.

The 'reset_idrac' parameter pulls and uses the badfish container to
perform iDRAC cleanup operations, which helps resolve issues with stuck
job queues and improves iDRAC stability during virtual media boot
operations.

AI Model: Claude Sonnet 4.5

Author: cjeanner

ansible/roles/badfish/tasks/call.yml

@@ -0,0 +1,80 @@
+---
+# badfish call tasks
+# Reusable task file to run badfish container with various options
+#
+# Required variables:
+#   badfish_host: BMC address or hostname
+#   badfish_user: BMC username
+#   badfish_password: BMC password
+#   badfish_args: List of badfish command and its arguments (e.g., ['--power-on'] or ['--mount-virtual-media', 'http://example.com/image.iso'])
+#
+# Optional variables:
+#   badfish_dns: DNS server IP (for VPN environments)
+#   badfish_no_log: Set to false to log command output (default: false)
+#   delay: Delay between retries in seconds (default: omit)
+#   retries: Number of retries (default: omit)
+
+- name: Validate required badfish parameters
+  ansible.builtin.assert:
+    that:
+      - badfish_host is defined
+      - badfish_host | length > 0
+      - badfish_user is defined
+      - badfish_user | length > 0
+      - badfish_password is defined
+      - badfish_password | length > 0
+      - badfish_args is defined
+      - badfish_args | length > 0
+    fail_msg: "Missing or empty required badfish parameters. Required: badfish_host, badfish_user, badfish_password, badfish_args"
+    quiet: true
+
+- name: Build badfish command
+  vars:
+    _badfish_podman_cmd: >-
+      {{
+        ['podman', 'run', '--rm'] +
+        (['--dns', badfish_dns] if badfish_dns is defined else [])
+      }}
+    _badfish_badfish_cmd: >-
+      {{
+        ['-H', badfish_host, '-u', badfish_user, '-p', badfish_password] +
+        (badfish_args | default([]))
+      }}
+    _badfish_full_cmd: "{{ _badfish_podman_cmd + ['quay.io/quads/badfish'] + _badfish_badfish_cmd }}"
+  ansible.builtin.set_fact:
+    badfish_command_list: "{{ _badfish_full_cmd }}"
+
+- name: Display badfish command
+  ansible.builtin.debug:
+    msg: "Running badfish command: {{ badfish_command_list | join(' ') }}"
+
+- name: Run badfish container
+  ansible.builtin.command:
+    argv: "{{ badfish_command_list }}"
+  no_log: "{{ badfish_no_log | default(false) }}"
+  register: badfish_result
+  failed_when: badfish_result.rc != 0
+  delay: "{{ delay | default(omit) }}"
+  retries: "{{ retries | default(omit) }}"
+  until: not badfish_result.failed
+
+- name: Display badfish stdout
+  ansible.builtin.debug:
+    msg: "badfish stdout: {{ badfish_result.stdout }}"
+  when: badfish_result.stdout is defined
+
+- name: Display badfish stderr
+  ansible.builtin.debug:
+    msg: "badfish stderr: {{ badfish_result.stderr }}"
+  when: badfish_result.stderr is defined
+
+- name: Fail if badfish reports an error in stderr
+  ansible.builtin.fail:
+    msg: "badfish reported an error: {{ badfish_result.stderr }}"
+  when:
+    - not ignore_errors | default(false) | bool
+    - badfish_result.stderr is defined and 'ERROR' in badfish_result.stderr
+
+- name: Set fact with badfish result for external access
+  ansible.builtin.set_fact:
+    badfish_command_result: "{{ badfish_result }}"

ansible/roles/badfish/tasks/install.yml

@@ -0,0 +1,12 @@
+---
+# badfish install tasks
+# Pulls the badfish container image from quay.io
+
+- name: Ensure podman package is installed
+  ansible.builtin.package:
+    name: podman
+    state: present
+
+- name: Pull badfish container image
+  ansible.builtin.command:
+    cmd: podman pull quay.io/quads/badfish

ansible/roles/bastion-install/defaults/main.yml

@@ -14,3 +14,7 @@ rh_crucible_url: https://github.com/perftool-incubator/crucible
 
 # Since the use of tc on the bastion machine is rare, we disable rebooting the bastion machine by default
 bastion_install_tc_reboot: false
+
+# Reset iDRAC service using badfish container (pulls and uses badfish container
+# to clear job queue and reset iDRAC service)
+reset_idrac: false

ansible/roles/bastion-install/tasks/main.yml

@@ -48,6 +48,12 @@
     disable_gpg_check: yes
   when: ansible_facts['distribution_major_version'] is version('9', '>=')
 
+- name: Install badfish container image
+  include_role:
+    name: badfish
+    tasks_from: install
+  when: reset_idrac | bool
+
 - name: Install python
   pip:
     name: python-hpilo

ansible/roles/boot-iso/defaults/main.yml

@@ -9,3 +9,7 @@ http_store_port: 8081
 # For a scale out it indicates how many worker nodes are already deployed
 # and should not be included in the scale out from the inventory.
 offset: 0
+
+# Reset iDRAC service using badfish container (pulls and uses badfish container
+# to clear job queue and reset iDRAC service)
+reset_idrac: false

ansible/roles/boot-iso/tasks/dell.yml

@@ -2,41 +2,79 @@
 # Dell tasks for booting an iso
 
 - name: Set Virtual Media ISO
-  set_fact:
+  ansible.builtin.set_fact:
     _virtual_media_iso: "{{ virtual_media_iso | default(hostvars[item]['boot_iso']) }}"
 
+- name: "Dell - Clear iDrac job queue for {{ item }} (badfish)"
+  ansible.builtin.include_role:
+    name: badfish
+    tasks_from: call
+  vars:
+    badfish_host: "{{ hostvars[item]['bmc_address'] }}"
+    badfish_user: "{{ hostvars[item]['bmc_user'] }}"
+    badfish_password: "{{ hostvars[item]['bmc_password'] }}"
+    badfish_args:
+      - "--clear-jobs"
+      - "--force"
+    ignore_errors: true
+  when: reset_idrac | bool
+
 - name: "Dell - Power down machine prior to booting iso for {{ item }}"
-  shell: |
-    ipmitool -I lanplus -H {{ hostvars[item]['bmc_address'] }} -U {{ hostvars[item]['bmc_user'] }} -P {{ hostvars[item]['bmc_password'] }} chassis power off
+  ansible.builtin.command:
+    cmd: >-
+      ipmitool -I lanplus -H "{{ hostvars[item]['bmc_address'] }}"
+      -U "{{ hostvars[item]['bmc_user'] }}"
+      -P "{{ hostvars[item]['bmc_password'] }}" chassis power off
   ignore_errors: true
   register: ipmi_poweroff
 
-- name: "Dell - Pause for power down for {{ item }}"
-  pause:
-    seconds: 10
+- name: "Dell - Reset iDRAC for {{ item }} (badfish)"
+  ansible.builtin.include_role:
+    name: badfish
+    tasks_from: call
+  vars:
+    badfish_host: "{{ hostvars[item]['bmc_address'] }}"
+    badfish_user: "{{ hostvars[item]['bmc_user'] }}"
+    badfish_password: "{{ hostvars[item]['bmc_password'] }}"
+    badfish_args:
+      - "--racreset"
+    ignore_errors: true
+  when: reset_idrac | bool
+
+- name: "Dell - Wait for power down for {{ item }}"
+  ansible.builtin.wait_for:
+    port: 22
+    delay: 2
+    state: stopped
+    host: "{{ hostvars[item]['ansible_host'] | default(hostvars[item]['inventory_hostname']) }}"
+    timeout: 60
   when: not ipmi_poweroff.failed
 
-- name: Dell - Set OneTimeBoot VirtualCD
-  uri:
-    url: "https://{{ hostvars[item]['bmc_address'] }}/redfish/v1/Managers/iDRAC.Embedded.1/Actions/Oem/EID_674_Manager.ImportSystemConfiguration"
+- name: "Ensure iDRAC reset order is passed for {{ item }}"
+  when: reset_idrac | bool
+  ansible.builtin.pause:
+    seconds: 30
+
+- name: "Dell - Wait for iDRAC to be available for {{ item }}"
+  ansible.builtin.uri:
+    url: "https://{{ hostvars[item]['bmc_address'] }}/redfish/v1"
     user: "{{ hostvars[item]['bmc_user'] }}"
     password: "{{ hostvars[item]['bmc_password'] }}"
-    method: POST
+    method: GET
     headers:
       content-type: application/json
       Accept: application/json
-    body:
-      {
-        "ShareParameters": { "Target": "ALL" },
-        "ImportBuffer": '<SystemConfiguration><Component FQDD="iDRAC.Embedded.1"><Attribute Name="ServerBoot.1#BootOnce">Enabled</Attribute><Attribute Name="ServerBoot.1#FirstBootDevice">VCD-DVD</Attribute></Component></SystemConfiguration>',
-      }
-    body_format: json
-    validate_certs: no
-    status_code: 202
-    return_content: yes
+    validate_certs: false
+    status_code: [200, 201, 301, 302]
+  register: racreset_result
+  until: racreset_result.status in [200, 201, 301, 302]
+  retries: 60
+  delay: 5
+  failed_when: false
+  when: reset_idrac | bool  
 
 - name: "Dell - Check for Virtual Media for {{ item }}"
-  uri:
+  ansible.builtin.uri:
     url: "https://{{ hostvars[item]['bmc_address'] }}/redfish/v1/Managers/iDRAC.Embedded.1/VirtualMedia/CD"
     user: "{{ hostvars[item]['bmc_user'] }}"
     password: "{{ hostvars[item]['bmc_password'] }}"
@@ -50,12 +88,15 @@
     status_code: 200
     return_content: yes
   register: check_virtual_media
+  retries: 10
+  delay: 10
+  until: check_virtual_media.status == 200
 
 - name: Block to rescue incase of stuck virtual media
   when: check_virtual_media.json.Image
   block:
   - name: "Dell - Eject any CD Virtual Media for {{ item }}"
-    uri:
+    ansible.builtin.uri:
       url: "https://{{ hostvars[item]['bmc_address'] }}/redfish/v1/Managers/iDRAC.Embedded.1/VirtualMedia/CD/Actions/VirtualMedia.EjectMedia"
       user: "{{ hostvars[item]['bmc_user'] }}"
       password: "{{ hostvars[item]['bmc_password'] }}"
@@ -72,15 +113,17 @@
   rescue:
     # Use racadm to address the failed redfish unmount of old virtual media
     - name: "Force mount of a existing image for {{ item }}"
-      raw: racadm remoteimage -c -u "" -p "" -l http://{{ http_store_host }}:{{ http_store_port }}/{{ _virtual_media_iso }}
+      ansible.builtin.raw: >-
+        racadm remoteimage -c -u "" -p "" -l http://{{ http_store_host }}:{{ http_store_port }}/{{ _virtual_media_iso }}
       delegate_to: "{{ hostvars[item]['bmc_address'] }}"
       vars:
         ansible_user: "{{ hostvars[item]['bmc_user'] }}"
         ansible_password: "{{ hostvars[item]['bmc_password'] }}"
         ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
 
     - name: "Force unmount of the existing image for {{ item }}"
-      raw: racadm remoteimage -d
+      ansible.builtin.raw: >-
+        racadm remoteimage -d
       delegate_to: "{{ hostvars[item]['bmc_address'] }}"
       vars:
         ansible_user: "{{ hostvars[item]['bmc_user'] }}"
@@ -106,6 +149,25 @@
   retries: 10
   delay: 30
 
+- name: Dell - Set OneTimeBoot VirtualCD
+  ansible.builtin.uri:
+    url: "https://{{ hostvars[item]['bmc_address'] }}/redfish/v1/Managers/iDRAC.Embedded.1/Actions/Oem/EID_674_Manager.ImportSystemConfiguration"
+    user: "{{ hostvars[item]['bmc_user'] }}"
+    password: "{{ hostvars[item]['bmc_password'] }}"
+    method: POST
+    headers:
+      content-type: application/json
+      Accept: application/json
+    body:
+      {
+        "ShareParameters": { "Target": "ALL" },
+        "ImportBuffer": '<SystemConfiguration><Component FQDD="iDRAC.Embedded.1"><Attribute Name="ServerBoot.1#BootOnce">Enabled</Attribute><Attribute Name="ServerBoot.1#FirstBootDevice">VCD-DVD</Attribute></Component></SystemConfiguration>',
+      }
+    body_format: json
+    validate_certs: no
+    status_code: 202
+    return_content: yes
+
 - name: "DELL - Power ON for {{ item }}"
   community.general.redfish_command:
     category: Systems

ansible/vars/all.sample.yml

@@ -71,6 +71,10 @@ setup_bastion_registry: false
 # Use in conjunction with ipv6 based clusters
 use_bastion_registry: false
 
+# Reset iDRAC service using badfish container (pulls and uses badfish container
+# to clear job queue and reset iDRAC service)
+# reset_idrac: false
+
 ################################################################################
 # OCP node vars
 ################################################################################