performance improvements

Author: joejstuart

cmd/validate/vsa.go

@@ -375,8 +375,8 @@ func validateImagesFromRekor(ctx context.Context, cmd *cobra.Command, data struc
 				continue
 			}
 
-			// Call the validation function
-			validationResult, err := validate(ctx, comp.ContainerImage, data.policy, rekorRetriever, data.publicKey)
+			// Call the validation function with content retrieval
+			validationResult, vsaContent, err := vsa.ValidateVSAWithContent(ctx, comp.ContainerImage, data.policy, rekorRetriever, data.publicKey)
 			if err != nil {
 				err = fmt.Errorf("validation failed for %s: %w", comp.ContainerImage, err)
 				results <- result{err: err, component: comp, validationResult: nil, vsaComponents: nil}
@@ -386,18 +386,14 @@ func validateImagesFromRekor(ctx context.Context, cmd *cobra.Command, data struc
 				continue
 			}
 
-			// Extract actual components from VSA attestation data
+			// Extract actual components from VSA attestation data (no redundant retrieval)
 			var vsaComponents []applicationsnapshot.Component
-			if validationResult != nil {
-				// Try to retrieve VSA data to extract actual components
-				vsaContent, err := rekorRetriever.RetrieveVSAData(ctx)
-				if err == nil {
-					predicate, err := vsa.ParseVSAContent(vsaContent)
-					if err == nil && predicate.Results != nil {
-						// Use actual components from VSA attestation if available
-						vsaComponents = predicate.Results.Components
-						logrus.Debugf("Extracted %d actual components from VSA attestation for %s", len(vsaComponents), comp.ContainerImage)
-					}
+			if validationResult != nil && vsaContent != "" {
+				predicate, err := vsa.ParseVSAContent(vsaContent)
+				if err == nil && predicate.Results != nil {
+					// Use actual components from VSA attestation if available
+					vsaComponents = predicate.Results.Components
+					logrus.Debugf("Extracted %d actual components from VSA attestation for %s", len(vsaComponents), comp.ContainerImage)
 				}
 			}
 

internal/validate/vsa/rekor_retriever.go

@@ -27,6 +27,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/rekor"
 	"github.com/sigstore/rekor/pkg/generated/client"
@@ -725,10 +726,10 @@ func (rc *rekorClient) worker(ctx context.Context, uuidChan <-chan string, resul
 			// Continue processing
 		}
 
-		// Fetch the log entry
-		entry, err := rc.GetLogEntryByUUID(ctx, uuid)
+		// Fetch the log entry with retry logic
+		entry, err := rc.GetLogEntryByUUIDWithRetry(ctx, uuid, 3)
 		if err != nil {
-			log.Debugf("Worker %d: Failed to fetch log entry for UUID %s: %v", workerID, uuid, err)
+			log.Debugf("Worker %d: Failed to fetch log entry for UUID %s after retries: %v", workerID, uuid, err)
 			select {
 			case resultChan <- fetchResult{entry: nil, err: err}:
 			case <-ctx.Done():
@@ -748,8 +749,8 @@ func (rc *rekorClient) worker(ctx context.Context, uuidChan <-chan string, resul
 
 // getWorkerCount returns the number of workers to use for parallel operations
 func (rc *rekorClient) getWorkerCount() int {
-	// Default to 8 workers
-	defaultWorkers := 8
+	// Default to 4 workers (optimized for Rekor rate limits)
+	defaultWorkers := 4
 
 	// Check environment variable
 	if workerStr := os.Getenv("EC_REKOR_WORKERS"); workerStr != "" {
@@ -840,6 +841,65 @@ func (rc *rekorClient) GetLogEntryByUUID(ctx context.Context, uuid string) (*mod
 	return nil, fmt.Errorf("log entry not found for UUID: %s", uuid)
 }
 
+// GetLogEntryByUUIDWithRetry fetches a log entry with exponential backoff retry logic
+func (rc *rekorClient) GetLogEntryByUUIDWithRetry(ctx context.Context, uuid string, maxRetries int) (*models.LogEntryAnon, error) {
+	var lastErr error
+
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		if attempt > 0 {
+			// Exponential backoff: 100ms, 200ms, 400ms
+			backoff := time.Duration(100*attempt) * time.Millisecond
+			log.Debugf("Retrying GetLogEntryByUUID for UUID %s (attempt %d/%d) after %v", uuid, attempt+1, maxRetries+1, backoff)
+
+			select {
+			case <-time.After(backoff):
+				// Continue with retry
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		}
+
+		entry, err := rc.GetLogEntryByUUID(ctx, uuid)
+		if err == nil {
+			if attempt > 0 {
+				log.Debugf("GetLogEntryByUUID succeeded for UUID %s on attempt %d", uuid, attempt+1)
+			}
+			return entry, nil
+		}
+
+		lastErr = err
+
+		// Don't retry on certain types of errors (e.g., not found, authentication)
+		if isNonRetryableError(err) {
+			log.Debugf("Non-retryable error for UUID %s: %v", uuid, err)
+			break
+		}
+	}
+
+	return nil, fmt.Errorf("failed to fetch log entry for UUID %s after %d attempts: %w", uuid, maxRetries+1, lastErr)
+}
+
+// isNonRetryableError determines if an error should not be retried
+func isNonRetryableError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+	// Don't retry on authentication, authorization, or not found errors
+	nonRetryablePatterns := []string{
+		"401", "403", "404", "not found", "unauthorized", "forbidden",
+	}
+
+	for _, pattern := range nonRetryablePatterns {
+		if strings.Contains(strings.ToLower(errStr), pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // RekorVSADataRetriever implements VSADataRetriever for Rekor-based VSA retrieval
 type RekorVSADataRetriever struct {
 	rekorRetriever *RekorVSARetriever

internal/validate/vsa/validation.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/google/go-containerregistry/pkg/name"
 	ssldsse "github.com/secure-systems-lab/go-securesystemslib/dsse"
@@ -279,27 +280,39 @@ func compareRules(vsaRuleResults map[string][]RuleResult, requiredRules map[stri
 	return result
 }
 
+// ValidationResultWithContent contains both validation result and VSA content
+type ValidationResultWithContent struct {
+	*ValidationResult
+	VSAContent string
+}
+
 // ValidateVSA is the main validation function called by the command
 func ValidateVSA(ctx context.Context, imageRef string, policy policy.Policy, retriever VSADataRetriever, publicKey string) (*ValidationResult, error) {
+	result, _, err := ValidateVSAWithContent(ctx, imageRef, policy, retriever, publicKey)
+	return result, err
+}
+
+// ValidateVSAWithContent returns both validation result and VSA content to avoid redundant retrieval
+func ValidateVSAWithContent(ctx context.Context, imageRef string, policy policy.Policy, retriever VSADataRetriever, publicKey string) (*ValidationResult, string, error) {
 	// Extract digest from image reference
 	ref, err := name.ParseReference(imageRef)
 	if err != nil {
-		return nil, fmt.Errorf("invalid image reference: %w", err)
+		return nil, "", fmt.Errorf("invalid image reference: %w", err)
 	}
 
 	digest := ref.Identifier()
 
 	// Retrieve VSA data using the provided retriever
 	vsaContent, err := retriever.RetrieveVSAData(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve VSA data: %w", err)
+		return nil, "", fmt.Errorf("failed to retrieve VSA data: %w", err)
 	}
 
 	// Verify signature if public key is provided
 	signatureVerified := false
 	if publicKey != "" {
 		if vsaContent == "" {
-			return nil, fmt.Errorf("signature verification not supported for this VSA retriever")
+			return nil, "", fmt.Errorf("signature verification not supported for this VSA retriever")
 		}
 		if err := verifyVSASignature(vsaContent, publicKey); err != nil {
 			// For now, log the error but don't fail the validation
@@ -314,7 +327,7 @@ func ValidateVSA(ctx context.Context, imageRef string, policy policy.Policy, ret
 	// Parse the VSA content to extract violations and successes
 	predicate, err := ParseVSAContent(vsaContent)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse VSA content: %w", err)
+		return nil, "", fmt.Errorf("failed to parse VSA content: %w", err)
 	}
 
 	// Create policy resolver and discover available rules
@@ -335,7 +348,7 @@ func ValidateVSA(ctx context.Context, imageRef string, policy policy.Policy, ret
 		ruleDiscovery := evaluator.NewRuleDiscoveryService()
 		rules, nonAnnotatedRules, err := ruleDiscovery.DiscoverRulesWithNonAnnotated(ctx, policySources)
 		if err != nil {
-			return nil, fmt.Errorf("failed to discover rules from policy sources: %w", err)
+			return nil, "", fmt.Errorf("failed to discover rules from policy sources: %w", err)
 		}
 
 		// Combine rules for filtering
@@ -356,7 +369,7 @@ func ValidateVSA(ctx context.Context, imageRef string, policy policy.Policy, ret
 	if vsaPolicyResolver != nil {
 		requiredRules, err = vsaPolicyResolver.GetRequiredRules(ctx, digest)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get required rules from policy: %w", err)
+			return nil, "", fmt.Errorf("failed to get required rules from policy: %w", err)
 		}
 	} else {
 		// If no policy resolver is available, consider all rules in VSA as required
@@ -370,15 +383,37 @@ func ValidateVSA(ctx context.Context, imageRef string, policy policy.Policy, ret
 	result := compareRules(vsaRuleResults, requiredRules, digest)
 	result.SignatureVerified = signatureVerified
 
-	return result, nil
+	return result, vsaContent, nil
 }
 
-// extractPackageFromCode extracts the package name from a rule code
+// packageCache caches package name extractions to avoid repeated string operations
+var packageCache = make(map[string]string)
+var packageCacheMutex sync.RWMutex
+
+// extractPackageFromCode extracts the package name from a rule code with caching
 func extractPackageFromCode(code string) string {
+	// Check cache first
+	packageCacheMutex.RLock()
+	if cached, exists := packageCache[code]; exists {
+		packageCacheMutex.RUnlock()
+		return cached
+	}
+	packageCacheMutex.RUnlock()
+
+	// Extract package name
+	var packageName string
 	if idx := strings.Index(code, "."); idx != -1 {
-		return code[:idx]
+		packageName = code[:idx]
+	} else {
+		packageName = code
 	}
-	return code
+
+	// Cache the result
+	packageCacheMutex.Lock()
+	packageCache[code] = packageName
+	packageCacheMutex.Unlock()
+
+	return packageName
 }
 
 // verifyVSASignature verifies the signature of a VSA file using cosign's DSSE verification