Handle bootc last, and always apply configuration changes

* Moves bootc handling into the ssh module, since a different module/manager
  like flightctl would do things differently than what we do via SSH

* Apply bootc changes at the end, after configuration changes and services
  have been handled on the exporter hosts

* If bootc update is in progress don't restart services, since that will
  happen naturally on the host after bootc reboot, and it means too much
  pressure for some sidekicks.

Author: mangelajo

internal/exporter/host/host.go

@@ -127,44 +127,6 @@ func (e *ExporterHostSyncer) filterExporterInstances(hostName string, exporterIn
 	return exporterInstances
 }
 
-// handleBootcUpgrade handles bootc upgrade checking and execution
-func (e *ExporterHostSyncer) handleBootcUpgrade(hostSsh ssh.HostManager) (bool, error) {
-	// Check if bootc upgrade service is already running
-	statusCmd, _ := hostSsh.RunHostCommand("systemctl is-active bootc-fetch-apply-updates.service bootc-fetch-apply-updates.timer")
-	if statusCmd != nil {
-		statuses := strings.Fields(statusCmd.Stdout)
-		if len(statuses) == 2 &&
-			(statuses[0] == "active" || statuses[0] == "activating" ||
-				statuses[1] == "active" || statuses[1] == "activating") {
-			fmt.Printf("    ⚠️  Bootc upgrade in progress, skipping exporter instances for this host\n")
-			return true, nil // skip = true
-		}
-	}
-
-	// Check booted image
-	bootcStdout, err := hostSsh.RunHostCommand("[ -f /run/ostree-booted ] && bootc upgrade --check")
-	if err == nil && bootcStdout != nil && bootcStdout.ExitCode == 0 && bootcStdout.Stdout != "" {
-		if strings.HasPrefix(bootcStdout.Stdout, "No changes") {
-			if e.dryRun {
-				fmt.Printf("    ✅ Bootc image is up to date\n")
-			}
-		} else if e.dryRun {
-			fmt.Printf("    📄 Would upgrade bootc image\n")
-		} else {
-			// Trigger bootc upgrade timer now. Assuming it uses manual activation (e.g. OnActiveSec=0, RandomizedDelaySec=1h, RemainAfterElapse=false)
-			_, err := hostSsh.RunHostCommand("systemctl restart bootc-fetch-apply-updates.timer")
-			if err != nil {
-				return false, fmt.Errorf("error triggering bootc upgrade service: %w", err)
-			}
-			fmt.Printf("    ✅ Bootc upgrade started, skipping exporter instances for this host\n")
-			return true, nil // skip = true
-		}
-	} else {
-		fmt.Printf("    ℹ️ Not a bootc managed host\n")
-	}
-	return false, nil // skip = false
-}
-
 // processExporterInstance processes a single exporter instance
 func (e *ExporterHostSyncer) processExporterInstance(exporterInstance *api.ExporterInstance, hostSsh ssh.HostManager) error {
 	if isDead, deadAnnotation := isExporterInstanceDead(exporterInstance); isDead {
@@ -222,8 +184,9 @@ func (e *ExporterHostSyncer) calculateBackoffDelay(attempts int) time.Duration {
 	return delay
 }
 
-// processExporterInstances processes exporter instances and adds failures to global retry queue
-func (e *ExporterHostSyncer) processExporterInstances(exporterInstances []*api.ExporterInstance, hostSsh ssh.HostManager, hostName string, retryQueue *[]RetryItem) {
+// processExporterInstancesAndBootc processes exporter instances and adds failures to global retry queue
+func (e *ExporterHostSyncer) processExporterInstancesAndBootc(exporterInstances []*api.ExporterInstance, hostSsh ssh.HostManager, hostName string, retryQueue *[]RetryItem) {
+
 	for _, exporterInstance := range exporterInstances {
 		if err := e.processExporterInstance(exporterInstance, hostSsh); err != nil {
 			fmt.Printf("    ❌ Failed to process %s: %v\n", exporterInstance.Name, err)
@@ -237,6 +200,19 @@ func (e *ExporterHostSyncer) processExporterInstances(exporterInstances []*api.E
 			})
 		}
 	}
+
+	if err := hostSsh.HandleBootcUpgrade(e.dryRun); err != nil {
+		// For other errors, just log them and continue
+		fmt.Printf("    ⚠️  Bootc upgrade error: %v\n", err)
+		*retryQueue = append(*retryQueue, RetryItem{
+			ExporterInstance: nil,
+			HostSSH:          hostSsh,
+			HostName:         hostName,
+			Attempts:         1,
+			LastError:        err,
+			LastAttemptTime:  time.Now(),
+		})
+	}
 }
 
 // processGlobalRetryQueue processes the global retry queue with exponential backoff
@@ -272,18 +248,28 @@ func (e *ExporterHostSyncer) processGlobalRetryQueue(retryQueue []RetryItem) err
 
 		// Second pass: retry items that are ready
 		for _, retryItem := range itemsToRetry {
-			fmt.Printf("🔄 Retrying %s on %s (attempt %d/%d)...\n",
-				retryItem.ExporterInstance.Name, retryItem.HostName, retryItem.Attempts+1, e.retryConfig.MaxAttempts)
-
-			if err := e.processExporterInstance(retryItem.ExporterInstance, retryItem.HostSSH); err != nil {
-				// Still failed, increment attempts and add to next retry queue
-				retryItem.Attempts++
-				retryItem.LastError = err
-				retryItem.LastAttemptTime = time.Now()
-				nextRetryQueue = append(nextRetryQueue, retryItem)
-				fmt.Printf("❌ Retry failed for %s on %s: %v\n", retryItem.ExporterInstance.Name, retryItem.HostName, err)
+			if retryItem.ExporterInstance == nil {
+				fmt.Printf("🔄 Retrying bootc upgrade on %s (attempt %d/%d)...\n",
+					retryItem.HostName, retryItem.Attempts+1, e.retryConfig.MaxAttempts)
+				if err := retryItem.HostSSH.HandleBootcUpgrade(e.dryRun); err != nil {
+					fmt.Printf("❌ Retry failed for bootc upgrade on %s: %v\n", retryItem.HostName, err)
+				} else {
+					fmt.Printf("✅ Retry succeeded for bootc upgrade on %s\n", retryItem.HostName)
+				}
 			} else {
-				fmt.Printf("✅ Retry succeeded for %s on %s\n", retryItem.ExporterInstance.Name, retryItem.HostName)
+				fmt.Printf("🔄 Retrying instance %s on %s (attempt %d/%d)...\n",
+					retryItem.ExporterInstance.Name, retryItem.HostName, retryItem.Attempts+1, e.retryConfig.MaxAttempts)
+
+				if err := e.processExporterInstance(retryItem.ExporterInstance, retryItem.HostSSH); err != nil {
+					// Still failed, increment attempts and add to next retry queue
+					retryItem.Attempts++
+					retryItem.LastError = err
+					retryItem.LastAttemptTime = time.Now()
+					nextRetryQueue = append(nextRetryQueue, retryItem)
+					fmt.Printf("❌ Retry failed for %s on %s: %v\n", retryItem.ExporterInstance.Name, retryItem.HostName, err)
+				} else {
+					fmt.Printf("✅ Retry succeeded for %s on %s\n", retryItem.ExporterInstance.Name, retryItem.HostName)
+				}
 			}
 		}
 
@@ -359,14 +345,8 @@ func (e *ExporterHostSyncer) SyncExporterHosts() error {
 			fmt.Printf("    ✅ Connection: %s\n", status)
 		}
 
-		if skip, err := e.handleBootcUpgrade(hostSsh); err != nil {
-			return err
-		} else if skip {
-			continue
-		}
-
 		// Process each exporter instance and add failures to global retry queue
-		e.processExporterInstances(exporterInstances, hostSsh, host.Name, &retryQueue)
+		e.processExporterInstancesAndBootc(exporterInstances, hostSsh, host.Name, &retryQueue)
 	}
 
 	// Second pass: retry all failed instances globally

internal/exporter/ssh/ssh.go

@@ -19,12 +19,24 @@ import (
 	"golang.org/x/crypto/ssh/agent"
 )
 
+// BootcStatus represents the status of bootc upgrade
+type BootcStatus int
+
+const (
+	BOOTC_UP_TO_DATE BootcStatus = iota
+	BOOTC_UPDATING
+	BOOTC_WILL_UPDATE
+	BOOTC_NOT_MANAGED
+)
+
 type HostManager interface {
 	Status() (string, error)
 	NeedsUpdate() (bool, error)
 	Diff() (string, error)
 	Apply(exporterConfig *v1alpha1.ExporterConfigTemplate, dryRun bool) error
 	RunHostCommand(command string) (*CommandResult, error)
+	GetBootcStatus() BootcStatus
+	HandleBootcUpgrade(dryRun bool) error
 }
 
 // CommandResult represents the result of running a command via SSH
@@ -202,6 +214,18 @@ func (m *SSHHostManager) Apply(exporterConfig *v1alpha1.ExporterConfigTemplate,
 		return fmt.Errorf("failed to reconcile exporter config file: %w", err)
 	}
 
+	if m.GetBootcStatus() == BOOTC_UPDATING {
+		if dryRun {
+			fmt.Printf("            📄 Bootc upgrade in progress, would skip exporter service restarts/container updates\n")
+		} else {
+			fmt.Printf("            ⚠️ Bootc upgrade in progress, skipping exporter service restarts/container updates\n")
+			return nil
+		}
+	}
+
+	// Only if bootc is not updating, we restart/start services and pull containers
+	// otherwise it's too much pressure on the system
+
 	if changedExporterConfig || changedContainer || changedService {
 		if !dryRun {
 			// Apply the changes: reload systemd, enable service and restart the exporter
@@ -386,6 +410,59 @@ func (m *SSHHostManager) reconcileFile(path string, content string, dryRun bool)
 	return true, nil
 }
 
+// GetBootcStatus checks the bootc status and returns the appropriate BootcStatus enum
+func (m *SSHHostManager) GetBootcStatus() BootcStatus {
+	// Check if bootc upgrade service is already running
+	statusCmd, _ := m.RunHostCommand("systemctl is-active bootc-fetch-apply-updates.service bootc-fetch-apply-updates.timer")
+	if statusCmd != nil {
+		statuses := strings.Fields(statusCmd.Stdout)
+		if len(statuses) == 2 &&
+			(statuses[0] == "active" || statuses[0] == "activating" ||
+				statuses[1] == "active" || statuses[1] == "activating") {
+			return BOOTC_UPDATING
+		}
+	}
+
+	// Check booted image
+	bootcStdout, err := m.RunHostCommand("[ -f /run/ostree-booted ] && bootc upgrade --check")
+	if err == nil && bootcStdout != nil && bootcStdout.ExitCode == 0 && bootcStdout.Stdout != "" {
+		if strings.HasPrefix(bootcStdout.Stdout, "No changes") {
+			return BOOTC_UP_TO_DATE
+		} else {
+			return BOOTC_WILL_UPDATE
+		}
+	}
+
+	return BOOTC_NOT_MANAGED
+}
+
+// HandleBootcUpgrade handles bootc upgrade checking and execution
+func (m *SSHHostManager) HandleBootcUpgrade(dryRun bool) error {
+	status := m.GetBootcStatus()
+
+	switch status {
+	case BOOTC_UPDATING:
+		fmt.Printf("    ⚠️  Bootc upgrade in progress\n")
+	case BOOTC_UP_TO_DATE:
+		fmt.Printf("    ✅ Bootc image is up to date\n")
+	case BOOTC_WILL_UPDATE:
+		if dryRun {
+			fmt.Printf("    📄 Would upgrade bootc image\n")
+		} else {
+			// Trigger bootc upgrade timer now. Assuming it uses manual activation (e.g. OnActiveSec=0, RandomizedDelaySec=1h, RemainAfterElapse=false)
+			_, err := m.RunHostCommand("systemctl restart bootc-fetch-apply-updates.timer")
+			if err != nil {
+				return fmt.Errorf("error triggering bootc upgrade service: %w", err)
+			}
+			fmt.Printf("    ✅ Bootc upgrade started\n")
+			return nil
+		}
+	case BOOTC_NOT_MANAGED:
+		fmt.Printf("    ℹ️ Not a bootc managed host\n")
+	}
+	return nil
+}
+
 func (m *SSHHostManager) createSshClient() (*ssh.Client, error) {
 
 	port := 22

internal/exporter/ssh/ssh_test.go

@@ -387,3 +387,11 @@ func TestSanitizeDiff(t *testing.T) {
 		})
 	}
 }
+
+func TestBootcStatusConstants(t *testing.T) {
+	// Test that all BootcStatus constants are defined correctly
+	assert.Equal(t, BootcStatus(0), BOOTC_UP_TO_DATE)
+	assert.Equal(t, BootcStatus(1), BOOTC_UPDATING)
+	assert.Equal(t, BootcStatus(2), BOOTC_WILL_UPDATE)
+	assert.Equal(t, BootcStatus(3), BOOTC_NOT_MANAGED)
+}