Merge pull request #195 from pfeifferj/feat-circuit-breaker-logging

feat: improve logging for circuitbreaker

Author: pfeifferj

docs/troubleshooting.md

@@ -104,7 +104,7 @@ kubectl patch ibmnodeclass YOUR-NODECLASS --type='merge' \
 
 **Step 2c: Test Connectivity from Node**
 ```bash
-# Attach floating IP for debugging 
+# Attach floating IP for debugging
 
 # Then SSH and test
 ssh -i ~/.ssh/eb root@FLOATING_IP
@@ -190,7 +190,7 @@ kubectl get ibmnodeclass YOUR-NODECLASS -o yaml
 ```bash
 # Verify instances are created in correct resource group
 ibmcloud is instances --output json | \
-  jq '.[] | select(.name | contains("nodepool")) | 
+  jq '.[] | select(.name | contains("nodepool")) |
   {name: .name, resource_group: .resource_group.id}'
 
 # Should match the resource group in IBMNodeClass

pkg/cloudprovider/circuitbreaker.go

@@ -212,11 +212,20 @@ func (cb *CircuitBreaker) RecordFailure(nodeClass, region string, err error) {
 	if recentFailures >= cb.config.FailureThreshold {
 		if cb.state != CircuitBreakerOpen {
 			cb.transitionToOpen()
+			// Include recent failure details for better debugging
+			recentErrors := make([]string, 0, len(cb.failures))
+			for _, f := range cb.failures {
+				if f.Timestamp.After(time.Now().Add(-cb.config.FailureWindow)) {
+					recentErrors = append(recentErrors, fmt.Sprintf("%s: %s", f.Timestamp.Format("15:04:05"), f.Error))
+				}
+			}
 			cb.logger.Error(fmt.Errorf("circuit breaker opened due to failure threshold exceeded"), "Circuit breaker OPENED",
 				"failures", recentFailures,
 				"threshold", cb.config.FailureThreshold,
 				"nodeClass", nodeClass,
-				"region", region)
+				"region", region,
+				"recentErrors", recentErrors,
+				"recoveryTimeout", cb.config.RecoveryTimeout)
 		}
 	} else if cb.state == CircuitBreakerHalfOpen {
 		// Failure in half-open state - go back to open

pkg/cloudprovider/cloudprovider.go

@@ -298,7 +298,7 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim)
 			"nodeClass", nodeClass.Name,
 			"region", nodeClass.Spec.Region)
 		c.recorder.Publish(ibmevents.NodeClaimCircuitBreakerBlocked(nodeClaim, cbErr.Error()))
-		return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("circuit breaker blocked provisioning: %w", cbErr))
+		return nil, fmt.Errorf("provisioning temporarily blocked by circuit breaker: %w", cbErr)
 	}
 
 	// Get the appropriate instance provider based on NodeClass configuration
@@ -311,7 +311,12 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim)
 
 	node, err := instanceProvider.Create(ctx, nodeClaim)
 	if err != nil {
-		log.Error(err, "Failed to create instance")
+		// Log the actual error details for better troubleshooting
+		log.Error(err, "Failed to create instance",
+			"nodeClass", nodeClass.Name,
+			"region", nodeClass.Spec.Region,
+			"zone", nodeClass.Spec.Zone,
+			"instanceTypes", lo.Map(compatible, func(it *cloudprovider.InstanceType, _ int) string { return it.Name }))
 		c.circuitBreaker.RecordFailure(nodeClass.Name, nodeClass.Spec.Region, err)
 		return nil, fmt.Errorf("creating instance, %w", err)
 	}

pkg/cloudprovider/cloudprovider_test.go

@@ -576,7 +576,7 @@ func TestCloudProvider_Create_CircuitBreakerEventPublishing(t *testing.T) {
 
 	// Verify error
 	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "circuit breaker blocked provisioning")
+	assert.Contains(t, err.Error(), "provisioning temporarily blocked by circuit breaker")
 
 	// Verify the correct event was published
 	assert.Len(t, eventRecorder.events, 1)

pkg/providers/common/types/bootstrap_test.go

@@ -248,7 +248,7 @@ func TestParseKubeconfigWithSpacing(t *testing.T) {
 clusters:
 - cluster:
     certificate-authority-data:    ` + base64.StdEncoding.EncodeToString([]byte("test-ca-data")) + `
-    server:   https://test.example.com:6443   
+    server:   https://test.example.com:6443
   name: test-cluster`
 
 	endpoint, caData, err := ParseKubeconfig(kubeconfigWithSpacing)