Skip to content

Commit 8513f9e

Browse files
committed
fix: continue processing all slices even if one fails
- Don't return early when a slice fails - continue processing remaining slices - Mark partition as successful if any slices succeed (even if some fail) - Only mark partition as failed if ALL slices fail - Track failed slice dates for better error reporting - Fixes summary showing 0 successful when some slices succeeded
1 parent b3ede27 commit 8513f9e

File tree

1 file changed

+34
-10
lines changed

1 file changed

+34
-10
lines changed

cmd/archiver.go

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,6 +1064,10 @@ func (a *Archiver) processPartitionWithSplit(partition PartitionInfo, program *t
10641064
totalBytes := int64(0)
10651065
successCount := 0
10661066
skipCount := 0
1067+
failCount := 0
1068+
var firstError error
1069+
var failedSliceDates []string
1070+
10671071
for i, timeRange := range ranges {
10681072
// Check for cancellation
10691073
select {
@@ -1102,17 +1106,17 @@ func (a *Archiver) processPartitionWithSplit(partition PartitionInfo, program *t
11021106
}
11031107

11041108
if sliceResult.Error != nil {
1105-
// Only log errors in debug mode
1109+
// Track errors but continue processing other slices
1110+
failCount++
1111+
if firstError == nil {
1112+
firstError = sliceResult.Error
1113+
}
1114+
failedSliceDates = append(failedSliceDates, timeRange.Start.Format("2006-01-02"))
1115+
// Log errors in debug mode
11061116
if a.config.Debug {
11071117
a.logger.Error(fmt.Sprintf(" ❌ Error processing slice %s: %v", timeRange.Start.Format("2006-01-02"), sliceResult.Error))
11081118
}
1109-
// Only return if it's a critical error (not "no rows")
1110-
if !sliceResult.Skipped {
1111-
return sliceResult
1112-
}
1113-
}
1114-
1115-
if sliceResult.Skipped {
1119+
} else if sliceResult.Skipped {
11161120
skipCount++
11171121
if a.config.Debug {
11181122
a.logger.Debug(fmt.Sprintf(" No data for %s, skipping", timeRange.Start.Format("2006-01-02")))
@@ -1125,15 +1129,35 @@ func (a *Archiver) processPartitionWithSplit(partition PartitionInfo, program *t
11251129

11261130
// Only log in debug mode - in TUI mode this corrupts the display
11271131
if a.config.Debug {
1128-
a.logger.Info(fmt.Sprintf(" Split partition complete: %d files created, %d skipped", successCount, skipCount))
1132+
if failCount > 0 {
1133+
a.logger.Info(fmt.Sprintf(" Split partition complete: %d files created, %d skipped, %d failed", successCount, skipCount, failCount))
1134+
} else {
1135+
a.logger.Info(fmt.Sprintf(" Split partition complete: %d files created, %d skipped", successCount, skipCount))
1136+
}
11291137
}
11301138

1131-
// Handle case where all slices were skipped
1139+
// Determine partition result based on slice outcomes
11321140
if successCount > 0 {
1141+
// At least some slices succeeded - mark partition as successful
11331142
result.BytesWritten = totalBytes
11341143
result.Compressed = true
11351144
result.Uploaded = true
1145+
// If some slices failed, log a warning but don't fail the partition
1146+
if failCount > 0 {
1147+
result.SkipReason = fmt.Sprintf("%d slice(s) failed: %s", failCount, strings.Join(failedSliceDates, ", "))
1148+
if a.config.Debug {
1149+
a.logger.Warn(fmt.Sprintf(" ⚠️ Partition %s completed with %d failed slice(s) out of %d total", partition.TableName, failCount, len(ranges)))
1150+
}
1151+
}
1152+
} else if failCount > 0 {
1153+
// All slices failed - mark partition as failed
1154+
result.Error = firstError
1155+
result.BytesWritten = 0
1156+
result.Compressed = false
1157+
result.Uploaded = false
1158+
result.Stage = "Failed"
11361159
} else {
1160+
// All slices were skipped
11371161
result.BytesWritten = 0
11381162
result.Compressed = false
11391163
result.Uploaded = false

0 commit comments

Comments
 (0)