Skip to content

Commit 444d20e

Browse files
mattl-netflixmattl-netflix
committed
Return missing files from snapshot verification continuing to marginalize BackupVerificationResult. Return all results valid or otherwise.
1 parent 373069a commit 444d20e

File tree

4 files changed

+73
-43
lines changed

4 files changed

+73
-43
lines changed

‎priam/src/main/java/com/netflix/priam/backup/BackupVerification.java‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
*/
1414
package com.netflix.priam.backup;
1515

16+
import com.google.common.collect.ImmutableMap;
17+
import com.google.common.collect.ImmutableSet;
1618
import com.netflix.priam.backupv2.IMetaProxy;
1719
import com.netflix.priam.utils.DateUtil;
1820
import com.netflix.priam.utils.DateUtil.DateRange;
@@ -75,9 +77,9 @@ public Optional<BackupVerificationResult> verifyLatestBackup(
7577
for (BackupMetadata backupMetadata :
7678
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
7779
if (backupMetadata.getLastValidated() == null || force) {
78-
Optional<BackupVerificationResult> result = verifyBackup(metaProxy, backupMetadata);
79-
if (result.isPresent()) {
80-
return result;
80+
BackupVerificationResult result = verifyBackup(metaProxy, backupMetadata);
81+
if (result.valid) {
82+
return Optional.of(result);
8183
}
8284
} else {
8385
updateLatestResult(backupMetadata);
@@ -88,26 +90,28 @@ public Optional<BackupVerificationResult> verifyLatestBackup(
8890
return Optional.empty();
8991
}
9092

91-
public List<BackupMetadata> verifyBackupsInRange(
93+
public ImmutableMap<BackupMetadata, ImmutableSet<String>> findMissingBackupFilesInRange(
9294
BackupVersion backupVersion, DateRange dateRange) throws IllegalArgumentException {
9395
IMetaProxy metaProxy = getMetaProxy(backupVersion);
94-
List<BackupMetadata> results = new ArrayList<>();
96+
ImmutableMap.Builder<BackupMetadata, ImmutableSet<String>> mapBuilder =
97+
ImmutableMap.builder();
9598
for (BackupMetadata backupMetadata :
9699
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
97-
if (backupMetadata.getLastValidated() != null
98-
|| verifyBackup(metaProxy, backupMetadata).isPresent()) {
99-
results.add(backupMetadata);
100-
}
100+
List<String> missingFiles =
101+
backupMetadata.getLastValidated() == null
102+
? verifyBackup(metaProxy, backupMetadata).filesInMetaOnly
103+
: new ArrayList<>();
104+
mapBuilder.put(backupMetadata, ImmutableSet.copyOf(missingFiles));
101105
}
102-
return results;
106+
return mapBuilder.build();
103107
}
104108

105109
/** returns the latest valid backup verification result if we have found one within the SLO * */
106110
public Optional<Instant> getLatestVerfifiedBackupTime() {
107111
return latestResult == null ? Optional.empty() : Optional.of(latestResult.snapshotInstant);
108112
}
109113

110-
private Optional<BackupVerificationResult> verifyBackup(
114+
private BackupVerificationResult verifyBackup(
111115
IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) {
112116
Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation());
113117
metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount());
@@ -119,9 +123,8 @@ private Optional<BackupVerificationResult> verifyBackup(
119123
Date now = new Date(DateUtil.getInstant().toEpochMilli());
120124
latestBackupMetaData.setLastValidated(now);
121125
backupStatusMgr.update(latestBackupMetaData);
122-
return Optional.of(result);
123126
}
124-
return Optional.empty();
127+
return result;
125128
}
126129

127130
private void updateLatestResult(BackupMetadata backupMetadata) {

‎priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
import com.netflix.priam.utils.DateUtil.DateRange;
3131
import java.time.Instant;
3232
import java.time.temporal.ChronoUnit;
33-
import java.util.List;
33+
import java.util.Map;
34+
import java.util.Set;
35+
import java.util.stream.Collectors;
3436
import javax.inject.Inject;
3537
import javax.inject.Singleton;
3638
import org.slf4j.Logger;
@@ -84,21 +86,28 @@ public void execute() throws Exception {
8486
Instant slo =
8587
now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS);
8688
DateRange dateRange = new DateRange(slo, now);
87-
List<BackupMetadata> verifiedBackups =
88-
backupVerification.verifyBackupsInRange(
89-
BackupVersion.SNAPSHOT_META_SERVICE, dateRange);
89+
Set<BackupMetadata> verifiedBackups =
90+
backupVerification
91+
.findMissingBackupFilesInRange(
92+
BackupVersion.SNAPSHOT_META_SERVICE, dateRange)
93+
.entrySet()
94+
.stream()
95+
.filter(entry -> entry.getValue().isEmpty())
96+
.map(Map.Entry::getKey)
97+
.collect(Collectors.toSet());
9098

9199
verifiedBackups
92100
.stream()
93-
.filter(result -> result.getLastValidated().toInstant().isAfter(now))
101+
.filter(metadata -> metadata.getLastValidated().toInstant().isAfter(now))
94102
.forEach(
95-
result -> {
103+
metadata -> {
96104
logger.info(
97105
"Sending {} message for backup: {}",
98106
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
99-
result.getSnapshotLocation());
107+
metadata.getSnapshotLocation());
100108
backupNotificationMgr.notify(
101-
result.getSnapshotLocation(), result.getStart().toInstant());
109+
metadata.getSnapshotLocation(),
110+
metadata.getStart().toInstant());
102111
});
103112

104113
if (verifiedBackups.isEmpty()) {

‎priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java‎

Lines changed: 38 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package com.netflix.priam.backup;
1919

20+
import com.google.common.truth.Truth;
2021
import com.google.inject.Guice;
2122
import com.google.inject.Injector;
2223
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
@@ -31,6 +32,7 @@
3132
import java.nio.file.Paths;
3233
import java.time.Instant;
3334
import java.time.temporal.ChronoUnit;
35+
import java.util.AbstractCollection;
3436
import java.util.Date;
3537
import java.util.List;
3638
import java.util.Optional;
@@ -109,16 +111,23 @@ public void noBackup() throws Exception {
109111

110112
@Test
111113
public void noBackupDateRange() throws Exception {
112-
List<BackupMetadata> backupVerificationResults =
113-
backupVerification.verifyBackupsInRange(
114-
BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now()));
115-
Assert.assertFalse(backupVerificationResults.size() > 0);
114+
long foundBackups =
115+
backupVerification
116+
.findMissingBackupFilesInRange(
117+
BackupVersion.SNAPSHOT_BACKUP,
118+
new DateRange(Instant.now(), Instant.now()))
119+
.entrySet()
120+
.size();
121+
Truth.assertThat(foundBackups).isEqualTo(0L);
116122

117-
backupVerificationResults =
118-
backupVerification.verifyBackupsInRange(
119-
BackupVersion.SNAPSHOT_META_SERVICE,
120-
new DateRange(Instant.now(), Instant.now()));
121-
Assert.assertFalse(backupVerificationResults.size() > 0);
123+
foundBackups =
124+
backupVerification
125+
.findMissingBackupFilesInRange(
126+
BackupVersion.SNAPSHOT_META_SERVICE,
127+
new DateRange(Instant.now(), Instant.now()))
128+
.entrySet()
129+
.size();
130+
Truth.assertThat(foundBackups).isEqualTo(0L);
122131
}
123132

124133
private void setUp() throws Exception {
@@ -184,12 +193,16 @@ public void verifyBackupVersion1() throws Exception {
184193
public void verifyBackupVersion1DateRange() throws Exception {
185194
setUp();
186195
// Verify for backup version 1.0
187-
List<BackupMetadata> backupVerificationResults =
188-
backupVerification.verifyBackupsInRange(
189-
BackupVersion.SNAPSHOT_BACKUP,
190-
new DateRange(backupDate + "," + backupDateEnd));
191-
Assert.assertTrue(!backupVerificationResults.isEmpty());
192-
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
196+
long missingFilesCount =
197+
backupVerification
198+
.findMissingBackupFilesInRange(
199+
BackupVersion.SNAPSHOT_BACKUP,
200+
new DateRange(backupDate + "," + backupDateEnd))
201+
.values()
202+
.stream()
203+
.filter(AbstractCollection::isEmpty)
204+
.count();
205+
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
193206
List<BackupMetadata> backupMetadata =
194207
backupStatusMgr.getLatestBackupMetadata(
195208
BackupVersion.SNAPSHOT_BACKUP,
@@ -260,12 +273,16 @@ public void verifyBackupVersion2() throws Exception {
260273
public void verifyBackupVersion2DateRange() throws Exception {
261274
setUp();
262275
// Verify for backup version 2.0
263-
List<BackupMetadata> backupVerificationResults =
264-
backupVerification.verifyBackupsInRange(
265-
BackupVersion.SNAPSHOT_META_SERVICE,
266-
new DateRange(backupDate + "," + backupDateEnd));
267-
Assert.assertTrue(!backupVerificationResults.isEmpty());
268-
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
276+
long missingFilesCount =
277+
backupVerification
278+
.findMissingBackupFilesInRange(
279+
BackupVersion.SNAPSHOT_META_SERVICE,
280+
new DateRange(backupDate + "," + backupDateEnd))
281+
.values()
282+
.stream()
283+
.filter(AbstractCollection::isEmpty)
284+
.count();
285+
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
269286
List<BackupMetadata> backupMetadata =
270287
backupStatusMgr.getLatestBackupMetadata(
271288
BackupVersion.SNAPSHOT_META_SERVICE,

‎priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,8 @@ public void testRestoreMode(@Mocked InstanceState state) throws Exception {
163163
Truth.assertThat(badVerifications.count()).isEqualTo(0);
164164
new Verifications() {
165165
{
166-
backupVerification.verifyBackupsInRange((BackupVersion) any, (DateRange) any);
166+
backupVerification.findMissingBackupFilesInRange(
167+
(BackupVersion) any, (DateRange) any);
167168
maxTimes = 0;
168169
}
169170

0 commit comments

Comments
 (0)