Skip to content

Commit 3ea569d

Browse files
committed
fix: count skipped slices as successful and save cache for empty slices
- Save cache metadata when slices are skipped due to no data - Count skipped slices as successful operations in partition results - Prevents re-checking empty slices on subsequent runs - Fixes summary showing 0 successful when slices were skipped - Ensures cache is persisted even when slices have no data
1 parent f04627e commit 3ea569d

File tree

1 file changed

+20
-6
lines changed

1 file changed

+20
-6
lines changed

cmd/archiver.go

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,17 +1137,25 @@ func (a *Archiver) processPartitionWithSplit(partition PartitionInfo, program *t
11371137
}
11381138

11391139
// Determine partition result based on slice outcomes
1140-
if successCount > 0 {
1141-
// At least some slices succeeded - mark partition as successful
1140+
// Count skipped slices as successful operations (they were processed successfully, just had no data)
1141+
totalSuccessful := successCount + skipCount
1142+
1143+
if totalSuccessful > 0 {
1144+
// At least some slices succeeded or were skipped - mark partition as successful
11421145
result.BytesWritten = totalBytes
1143-
result.Compressed = true
1144-
result.Uploaded = true
1146+
result.Compressed = successCount > 0 // Only true if we actually uploaded files
1147+
result.Uploaded = successCount > 0 // Only true if we actually uploaded files
1148+
11451149
// If some slices failed, log a warning but don't fail the partition
11461150
if failCount > 0 {
11471151
result.SkipReason = fmt.Sprintf("%d slice(s) failed: %s", failCount, strings.Join(failedSliceDates, ", "))
11481152
if a.config.Debug {
11491153
a.logger.Warn(fmt.Sprintf(" ⚠️ Partition %s completed with %d failed slice(s) out of %d total", partition.TableName, failCount, len(ranges)))
11501154
}
1155+
} else if skipCount > 0 && successCount == 0 {
1156+
// All slices were skipped (no data)
1157+
result.Skipped = true
1158+
result.SkipReason = "All slices skipped (no data in time ranges)"
11511159
}
11521160
} else if failCount > 0 {
11531161
// All slices failed - mark partition as failed
@@ -1157,12 +1165,12 @@ func (a *Archiver) processPartitionWithSplit(partition PartitionInfo, program *t
11571165
result.Uploaded = false
11581166
result.Stage = "Failed"
11591167
} else {
1160-
// All slices were skipped
1168+
// This shouldn't happen, but handle it gracefully
11611169
result.BytesWritten = 0
11621170
result.Compressed = false
11631171
result.Uploaded = false
11641172
result.Skipped = true
1165-
result.SkipReason = "All slices skipped (no data in time ranges)"
1173+
result.SkipReason = "No slices processed"
11661174
}
11671175

11681176
return result
@@ -1410,6 +1418,12 @@ func (a *Archiver) processSinglePartitionSlice(partition PartitionInfo, _ *tea.P
14101418
if tempFilePath != "" {
14111419
cleanupTempFile(tempFilePath)
14121420
}
1421+
// Save cache metadata to indicate this slice was checked and had no data
1422+
// This prevents re-checking empty slices on subsequent runs
1423+
cache.setFileMetadataWithETagAndStartTime(partition.TableName, objectKey, 0, 0, "", "", false, sliceStartTime)
1424+
if err := cache.save(a.config.CacheScope); err != nil {
1425+
a.logger.Debug(fmt.Sprintf(" ⚠️ Failed to save cache metadata for empty slice: %v", err))
1426+
}
14131427
result.Duration = time.Since(sliceStartTime)
14141428
return result
14151429
}

0 commit comments

Comments
 (0)