Skip to content

Commit 6f1a435

Browse files
mattl-netflixmattl-netflix
committed
Return missing files from snapshot verification continuing to marginalize BackupVerificationResult. Return all results valid or otherwise.
1 parent 373069a commit 6f1a435

File tree

7 files changed

+151
-178
lines changed

7 files changed

+151
-178
lines changed

‎priam/src/main/java/com/netflix/priam/backup/BackupVerification.java‎

Lines changed: 13 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
*/
1414
package com.netflix.priam.backup;
1515

16+
import com.google.common.collect.ImmutableMap;
17+
import com.google.common.collect.ImmutableSet;
1618
import com.netflix.priam.backupv2.IMetaProxy;
1719
import com.netflix.priam.utils.DateUtil;
1820
import com.netflix.priam.utils.DateUtil.DateRange;
@@ -68,46 +70,29 @@ public IMetaProxy getMetaProxy(BackupVersion backupVersion) {
6870
return null;
6971
}
7072

71-
public Optional<BackupVerificationResult> verifyLatestBackup(
73+
public ImmutableMap<BackupMetadata, ImmutableSet<String>> findMissingBackupFilesInRange(
7274
BackupVersion backupVersion, boolean force, DateRange dateRange)
7375
throws IllegalArgumentException {
7476
IMetaProxy metaProxy = getMetaProxy(backupVersion);
77+
ImmutableMap.Builder<BackupMetadata, ImmutableSet<String>> mapBuilder =
78+
ImmutableMap.builder();
7579
for (BackupMetadata backupMetadata :
7680
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
77-
if (backupMetadata.getLastValidated() == null || force) {
78-
Optional<BackupVerificationResult> result = verifyBackup(metaProxy, backupMetadata);
79-
if (result.isPresent()) {
80-
return result;
81-
}
82-
} else {
83-
updateLatestResult(backupMetadata);
84-
return Optional.of(latestResult);
85-
}
81+
List<String> missingFiles =
82+
backupMetadata.getLastValidated() == null || force
83+
? verifyBackup(metaProxy, backupMetadata).filesInMetaOnly
84+
: new ArrayList<>();
85+
mapBuilder.put(backupMetadata, ImmutableSet.copyOf(missingFiles));
8686
}
87-
latestResult = null;
88-
return Optional.empty();
89-
}
90-
91-
public List<BackupMetadata> verifyBackupsInRange(
92-
BackupVersion backupVersion, DateRange dateRange) throws IllegalArgumentException {
93-
IMetaProxy metaProxy = getMetaProxy(backupVersion);
94-
List<BackupMetadata> results = new ArrayList<>();
95-
for (BackupMetadata backupMetadata :
96-
backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange)) {
97-
if (backupMetadata.getLastValidated() != null
98-
|| verifyBackup(metaProxy, backupMetadata).isPresent()) {
99-
results.add(backupMetadata);
100-
}
101-
}
102-
return results;
87+
return mapBuilder.build();
10388
}
10489

10590
/** returns the latest valid backup verification result if we have found one within the SLO * */
10691
public Optional<Instant> getLatestVerfifiedBackupTime() {
10792
return latestResult == null ? Optional.empty() : Optional.of(latestResult.snapshotInstant);
10893
}
10994

110-
private Optional<BackupVerificationResult> verifyBackup(
95+
private BackupVerificationResult verifyBackup(
11196
IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) {
11297
Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation());
11398
metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount());
@@ -119,9 +104,8 @@ private Optional<BackupVerificationResult> verifyBackup(
119104
Date now = new Date(DateUtil.getInstant().toEpochMilli());
120105
latestBackupMetaData.setLastValidated(now);
121106
backupStatusMgr.update(latestBackupMetaData);
122-
return Optional.of(result);
123107
}
124-
return Optional.empty();
108+
return result;
125109
}
126110

127111
private void updateLatestResult(BackupMetadata backupMetadata) {

‎priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
import com.netflix.priam.utils.DateUtil.DateRange;
3131
import java.time.Instant;
3232
import java.time.temporal.ChronoUnit;
33-
import java.util.List;
33+
import java.util.Map;
34+
import java.util.Set;
35+
import java.util.stream.Collectors;
3436
import javax.inject.Inject;
3537
import javax.inject.Singleton;
3638
import org.slf4j.Logger;
@@ -84,21 +86,28 @@ public void execute() throws Exception {
8486
Instant slo =
8587
now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS);
8688
DateRange dateRange = new DateRange(slo, now);
87-
List<BackupMetadata> verifiedBackups =
88-
backupVerification.verifyBackupsInRange(
89-
BackupVersion.SNAPSHOT_META_SERVICE, dateRange);
89+
Set<BackupMetadata> verifiedBackups =
90+
backupVerification
91+
.findMissingBackupFilesInRange(
92+
BackupVersion.SNAPSHOT_META_SERVICE, false /* force */, dateRange)
93+
.entrySet()
94+
.stream()
95+
.filter(entry -> entry.getValue().isEmpty())
96+
.map(Map.Entry::getKey)
97+
.collect(Collectors.toSet());
9098

9199
verifiedBackups
92100
.stream()
93-
.filter(result -> result.getLastValidated().toInstant().isAfter(now))
101+
.filter(metadata -> metadata.getLastValidated().toInstant().isAfter(now))
94102
.forEach(
95-
result -> {
103+
metadata -> {
96104
logger.info(
97105
"Sending {} message for backup: {}",
98106
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
99-
result.getSnapshotLocation());
107+
metadata.getSnapshotLocation());
100108
backupNotificationMgr.notify(
101-
result.getSnapshotLocation(), result.getStart().toInstant());
109+
metadata.getSnapshotLocation(),
110+
metadata.getStart().toInstant());
102111
});
103112

104113
if (verifiedBackups.isEmpty()) {

‎priam/src/main/java/com/netflix/priam/resources/BackupServlet.java‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
*/
1717
package com.netflix.priam.resources;
1818

19+
import com.google.common.collect.ImmutableMap;
20+
import com.google.common.collect.ImmutableSet;
1921
import com.netflix.priam.backup.*;
2022
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
2123
import com.netflix.priam.config.IBackupRestoreConfig;
2224
import com.netflix.priam.config.IConfiguration;
2325
import com.netflix.priam.scheduler.PriamScheduler;
2426
import com.netflix.priam.utils.DateUtil;
2527
import com.netflix.priam.utils.DateUtil.DateRange;
28+
import com.netflix.priam.utils.GsonJsonSerializer;
2629
import java.time.Instant;
2730
import java.time.temporal.ChronoUnit;
2831
import java.util.*;
@@ -215,19 +218,12 @@ public Response snapshotsByDate(@PathParam("date") String date) throws Exception
215218
@Produces(MediaType.APPLICATION_JSON)
216219
public Response validateSnapshotByDate(
217220
@PathParam("daterange") String daterange,
218-
@DefaultValue("false") @QueryParam("force") boolean force)
219-
throws Exception {
221+
@DefaultValue("false") @QueryParam("force") boolean force) {
220222
DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
221-
Optional<BackupVerificationResult> result =
222-
backupVerification.verifyLatestBackup(
223+
ImmutableMap<BackupMetadata, ImmutableSet<String>> result =
224+
backupVerification.findMissingBackupFilesInRange(
223225
BackupVersion.SNAPSHOT_BACKUP, force, dateRange);
224-
if (!result.isPresent()) {
225-
return Response.noContent()
226-
.entity("No valid meta found for provided time range")
227-
.build();
228-
}
229-
230-
return Response.ok(result.get().toString()).build();
226+
return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build();
231227
}
232228

233229
/*

‎priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package com.netflix.priam.resources;
1919

20+
import com.google.common.collect.ImmutableMap;
21+
import com.google.common.collect.ImmutableSet;
2022
import com.netflix.priam.backup.*;
2123
import com.netflix.priam.backupv2.BackupTTLTask;
2224
import com.netflix.priam.backupv2.BackupV2Service;
@@ -121,19 +123,12 @@ public Response info(@PathParam("date") String date) {
121123
@Path("/validate/{daterange}")
122124
public Response validateV2SnapshotByDate(
123125
@PathParam("daterange") String daterange,
124-
@DefaultValue("false") @QueryParam("force") boolean force)
125-
throws Exception {
126+
@DefaultValue("false") @QueryParam("force") boolean force) {
126127
DateUtil.DateRange dateRange = new DateUtil.DateRange(daterange);
127-
Optional<BackupVerificationResult> result =
128-
backupVerification.verifyLatestBackup(
128+
ImmutableMap<BackupMetadata, ImmutableSet<String>> result =
129+
backupVerification.findMissingBackupFilesInRange(
129130
BackupVersion.SNAPSHOT_META_SERVICE, force, dateRange);
130-
if (!result.isPresent()) {
131-
return Response.noContent()
132-
.entity("No valid meta found for provided time range")
133-
.build();
134-
}
135-
136-
return Response.ok(result.get().toString()).build();
131+
return Response.ok(GsonJsonSerializer.getGson().toJson(result)).build();
137132
}
138133

139134
@GET

‎priam/src/test/java/com/netflix/priam/backup/TestBackupVerification.java‎

Lines changed: 57 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717

1818
package com.netflix.priam.backup;
1919

20+
import com.google.common.collect.ImmutableMap;
21+
import com.google.common.collect.ImmutableSet;
22+
import com.google.common.truth.Truth;
2023
import com.google.inject.Guice;
2124
import com.google.inject.Injector;
2225
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
@@ -31,9 +34,7 @@
3134
import java.nio.file.Paths;
3235
import java.time.Instant;
3336
import java.time.temporal.ChronoUnit;
34-
import java.util.Date;
35-
import java.util.List;
36-
import java.util.Optional;
37+
import java.util.*;
3738
import mockit.Mock;
3839
import mockit.MockUp;
3940
import org.apache.commons.io.FileUtils;
@@ -91,34 +92,43 @@ public void cleanup() {
9192
}
9293

9394
@Test
94-
public void noBackup() throws Exception {
95-
Optional<BackupVerificationResult> backupVerificationResultOptinal =
96-
backupVerification.verifyLatestBackup(
95+
public void noBackup() {
96+
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
97+
backupVerification.findMissingBackupFilesInRange(
9798
BackupVersion.SNAPSHOT_BACKUP,
9899
false,
99100
new DateRange(Instant.now(), Instant.now()));
100-
Assert.assertFalse(backupVerificationResultOptinal.isPresent());
101+
Truth.assertThat(missingFiles).isEmpty();
101102

102-
backupVerificationResultOptinal =
103-
backupVerification.verifyLatestBackup(
103+
missingFiles =
104+
backupVerification.findMissingBackupFilesInRange(
104105
BackupVersion.SNAPSHOT_META_SERVICE,
105106
false,
106107
new DateRange(Instant.now(), Instant.now()));
107-
Assert.assertFalse(backupVerificationResultOptinal.isPresent());
108+
Truth.assertThat(missingFiles).isEmpty();
108109
}
109110

110111
@Test
111112
public void noBackupDateRange() throws Exception {
112-
List<BackupMetadata> backupVerificationResults =
113-
backupVerification.verifyBackupsInRange(
114-
BackupVersion.SNAPSHOT_BACKUP, new DateRange(Instant.now(), Instant.now()));
115-
Assert.assertFalse(backupVerificationResults.size() > 0);
113+
long foundBackups =
114+
backupVerification
115+
.findMissingBackupFilesInRange(
116+
BackupVersion.SNAPSHOT_BACKUP,
117+
false /* force */,
118+
new DateRange(Instant.now(), Instant.now()))
119+
.entrySet()
120+
.size();
121+
Truth.assertThat(foundBackups).isEqualTo(0L);
116122

117-
backupVerificationResults =
118-
backupVerification.verifyBackupsInRange(
119-
BackupVersion.SNAPSHOT_META_SERVICE,
120-
new DateRange(Instant.now(), Instant.now()));
121-
Assert.assertFalse(backupVerificationResults.size() > 0);
123+
foundBackups =
124+
backupVerification
125+
.findMissingBackupFilesInRange(
126+
BackupVersion.SNAPSHOT_META_SERVICE,
127+
false /* force */,
128+
new DateRange(Instant.now(), Instant.now()))
129+
.entrySet()
130+
.size();
131+
Truth.assertThat(foundBackups).isEqualTo(0L);
122132
}
123133

124134
private void setUp() throws Exception {
@@ -152,13 +162,12 @@ private void setUp() throws Exception {
152162
public void verifyBackupVersion1() throws Exception {
153163
setUp();
154164
// Verify for backup version 1.0
155-
Optional<BackupVerificationResult> backupVerificationResultOptinal =
156-
backupVerification.verifyLatestBackup(
165+
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
166+
backupVerification.findMissingBackupFilesInRange(
157167
BackupVersion.SNAPSHOT_BACKUP,
158168
false,
159169
new DateRange(backupDate + "," + backupDate));
160-
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
161-
Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant);
170+
Truth.assertThat(missingFiles.values().stream().allMatch(Set::isEmpty)).isTrue();
162171
Optional<BackupMetadata> backupMetadata =
163172
backupStatusMgr
164173
.getLatestBackupMetadata(
@@ -184,12 +193,17 @@ public void verifyBackupVersion1() throws Exception {
184193
public void verifyBackupVersion1DateRange() throws Exception {
185194
setUp();
186195
// Verify for backup version 1.0
187-
List<BackupMetadata> backupVerificationResults =
188-
backupVerification.verifyBackupsInRange(
189-
BackupVersion.SNAPSHOT_BACKUP,
190-
new DateRange(backupDate + "," + backupDateEnd));
191-
Assert.assertTrue(!backupVerificationResults.isEmpty());
192-
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
196+
long missingFilesCount =
197+
backupVerification
198+
.findMissingBackupFilesInRange(
199+
BackupVersion.SNAPSHOT_BACKUP,
200+
false /* force */,
201+
new DateRange(backupDate + "," + backupDateEnd))
202+
.values()
203+
.stream()
204+
.filter(AbstractCollection::isEmpty)
205+
.count();
206+
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
193207
List<BackupMetadata> backupMetadata =
194208
backupStatusMgr.getLatestBackupMetadata(
195209
BackupVersion.SNAPSHOT_BACKUP,
@@ -211,14 +225,12 @@ public void verifyBackupVersion1DateRange() throws Exception {
211225
public void verifyBackupVersion2() throws Exception {
212226
setUp();
213227
// Verify for backup version 2.0
214-
Optional<BackupVerificationResult> backupVerificationResultOptinal =
215-
backupVerification.verifyLatestBackup(
228+
ImmutableMap<BackupMetadata, ImmutableSet<String>> missingFiles =
229+
backupVerification.findMissingBackupFilesInRange(
216230
BackupVersion.SNAPSHOT_META_SERVICE,
217231
false,
218232
new DateRange(backupDate + "," + backupDate));
219-
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
220-
Assert.assertEquals(Instant.EPOCH, backupVerificationResultOptinal.get().snapshotInstant);
221-
Assert.assertEquals("some_random", backupVerificationResultOptinal.get().remotePath);
233+
Truth.assertThat(missingFiles.values().stream().allMatch(Set::isEmpty)).isTrue();
222234

223235
Optional<BackupMetadata> backupMetadata =
224236
backupStatusMgr
@@ -230,21 +242,6 @@ public void verifyBackupVersion2() throws Exception {
230242
Assert.assertTrue(backupMetadata.isPresent());
231243
Assert.assertNotNull(backupMetadata.get().getLastValidated());
232244

233-
// Retry the verification, it should not try and re-verify
234-
backupVerificationResultOptinal =
235-
backupVerification.verifyLatestBackup(
236-
BackupVersion.SNAPSHOT_META_SERVICE,
237-
false,
238-
new DateRange(backupDate + "," + backupDate));
239-
Assert.assertTrue(backupVerificationResultOptinal.isPresent());
240-
Assert.assertEquals(
241-
DateUtil.parseInstant(backupDate),
242-
backupVerificationResultOptinal.get().snapshotInstant);
243-
Assert.assertNotEquals("some_random", backupVerificationResultOptinal.get().remotePath);
244-
Assert.assertEquals(
245-
location.subpath(1, location.getNameCount()).toString(),
246-
backupVerificationResultOptinal.get().remotePath);
247-
248245
backupMetadata =
249246
backupStatusMgr
250247
.getLatestBackupMetadata(
@@ -260,12 +257,17 @@ public void verifyBackupVersion2() throws Exception {
260257
public void verifyBackupVersion2DateRange() throws Exception {
261258
setUp();
262259
// Verify for backup version 2.0
263-
List<BackupMetadata> backupVerificationResults =
264-
backupVerification.verifyBackupsInRange(
265-
BackupVersion.SNAPSHOT_META_SERVICE,
266-
new DateRange(backupDate + "," + backupDateEnd));
267-
Assert.assertTrue(!backupVerificationResults.isEmpty());
268-
Assert.assertTrue(backupVerificationResults.size() == numFakeBackups);
260+
long missingFilesCount =
261+
backupVerification
262+
.findMissingBackupFilesInRange(
263+
BackupVersion.SNAPSHOT_META_SERVICE,
264+
false /* force */,
265+
new DateRange(backupDate + "," + backupDateEnd))
266+
.values()
267+
.stream()
268+
.filter(AbstractCollection::isEmpty)
269+
.count();
270+
Truth.assertThat(missingFilesCount).isEqualTo(numFakeBackups);
269271
List<BackupMetadata> backupMetadata =
270272
backupStatusMgr.getLatestBackupMetadata(
271273
BackupVersion.SNAPSHOT_META_SERVICE,

0 commit comments

Comments
 (0)