Skip to content

Commit 8440040

Browse files
mattl-netflixmattl-netflix
authored
Include peers_v2 in backups omit it from restores. (#1112)
1 parent eecb780 commit 8440040

File tree

4 files changed

+94
-33
lines changed

4 files changed

+94
-33
lines changed

‎priam/src/main/java/com/netflix/priam/backup/AbstractBackup.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ protected final void initiateBackup(
6767
File backupDir = new File(columnFamilyDir, monitoringFolder);
6868
if (isAReadableDirectory(backupDir)) {
6969
String columnFamilyName = getColumnFamily(backupDir);
70-
if (backupRestoreUtil.isFiltered(keyspaceDir.getName(), columnFamilyName)) {
70+
if (backupRestoreUtil.shouldOmitFromBackup(keyspaceDir.getName(), columnFamilyName)) {
7171
// Clean the backup/snapshot directory else files will keep getting
7272
// accumulated.
7373
SystemUtils.cleanupDir(backupDir.getAbsolutePath(), null);

‎priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@
2020
import com.google.common.collect.ImmutableMap;
2121
import com.netflix.priam.backupv2.IMetaProxy;
2222
import com.netflix.priam.utils.DateUtil;
23+
import org.apache.commons.io.FileUtils;
24+
import org.apache.commons.lang3.StringUtils;
25+
26+
import javax.inject.Inject;
27+
import javax.inject.Provider;
2328
import java.nio.file.Path;
2429
import java.util.*;
2530
import java.util.regex.Pattern;
2631
import java.util.stream.Collectors;
27-
import javax.inject.Inject;
28-
import javax.inject.Provider;
29-
import org.apache.commons.io.FileUtils;
30-
import org.apache.commons.lang3.StringUtils;
3132

3233
/** Helper methods applicable to both backup and restore */
3334
public class BackupRestoreUtil {
@@ -41,10 +42,11 @@ public class BackupRestoreUtil {
4142
Arrays.asList(
4243
"local",
4344
"peers",
44-
"peers_v2",
4545
"hints",
4646
"compactions_in_progress",
4747
"LocationInfo"));
48+
private static final Map<String, List<String>> EXTRA_TABLES_TO_EXCLUDE_FROM_RESTORE =
49+
ImmutableMap.of("system", Arrays.asList("peers_v2"));
4850

4951
@Inject
5052
public BackupRestoreUtil(String configIncludeFilter, String configExcludeFilter) {
@@ -113,7 +115,7 @@ public static Map<String, List<String>> getFilter(String inputFilter)
113115
* @param columnFamilyDir name of the columnfamily directory in consideration
114116
* @return true if directory should be filter from processing; otherwise, false.
115117
*/
116-
public final boolean isFiltered(String keyspace, String columnFamilyDir) {
118+
public final boolean shouldOmitFromBackup(String keyspace, String columnFamilyDir) {
117119
if (StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(columnFamilyDir)) return false;
118120
String columnFamilyName = columnFamilyDir.split("-")[0];
119121
if (FILTER_COLUMN_FAMILY.containsKey(keyspace)
@@ -130,4 +132,11 @@ public final boolean isFiltered(String keyspace, String columnFamilyDir) {
130132
|| includeFilter.get(keyspace).contains(columnFamilyName)));
131133
return false;
132134
}
135+
136+
public final boolean shouldOmitFromRestore(String keyspace, String columnFamilyDir) {
137+
String columnFamilyName = columnFamilyDir.split("-")[0];
138+
return shouldOmitFromBackup(keyspace, columnFamilyDir) ||
139+
(EXTRA_TABLES_TO_EXCLUDE_FROM_RESTORE.containsKey(keyspace)
140+
&& EXTRA_TABLES_TO_EXCLUDE_FROM_RESTORE.get(keyspace).contains(columnFamilyName));
141+
}
133142
}

‎priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,7 @@ private List<Future<Path>> download(
110110
List<Future<Path>> futureList = new ArrayList<>();
111111
while (fsIterator.hasNext()) {
112112
AbstractBackupPath temp = fsIterator.next();
113-
if (backupRestoreUtil.isFiltered(
114-
temp.getKeyspace(), temp.getColumnFamily())) { // is filtered?
113+
if (backupRestoreUtil.shouldOmitFromRestore(temp.getKeyspace(), temp.getColumnFamily())) {
115114
logger.info(
116115
"Bypassing restoring file \"{}\" as it is part of the keyspace.columnfamily filter list. Its keyspace:cf is: {}:{}",
117116
temp.newRestoreFile(),

‎priam/src/test/groovy/com.netflix.priam/backup/TestBackupRestoreUtil.groovy‎

Lines changed: 77 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -24,38 +24,91 @@ import spock.lang.Unroll
2424
*/
2525
@Unroll
2626
class TestBackupRestoreUtil extends Specification {
27-
def "IsFilter for KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter is #result"() {
27+
def "shouldOmitFromBackup for KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter is #result"() {
2828
expect:
29-
new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).isFiltered(keyspace, columnfamily) == result
29+
new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).shouldOmitFromBackup(keyspace, columnfamily) == result
3030

3131
where:
32-
configIncludeFilter | configExcludeFilter | keyspace | columnfamily || result
33-
null | null | "defg" | "gh" || false
34-
"abc.*" | null | "abc" | "cd" || false
35-
"abc.*" | null | "ab" | "cd" || true
36-
null | "abc.de" | "abc" | "def" || false
37-
null | "abc.de" | "abc" | "de" || true
38-
"abc.*,def.*" | null | "abc" | "cd" || false
39-
"abc.*,def.*" | null | "def" | "ab" || false
40-
"abc.*,def.*" | null | "ab" | "cd" || true
41-
"abc.*,def.*" | null | "df" | "ab" || true
42-
null | "abc.de,fg.hi" | "abc" | "def" || false
43-
null | "abc.de,fg.hi" | "abc" | "de" || true
44-
null | "abc.de,fg.hi" | "fg" | "hijk" || false
45-
null | "abc.de,fg.hi" | "fg" | "hi" || true
46-
"abc.*" | "ab.ab" | "ab" | "cd" || true
47-
"abc.*" | "ab.ab" | "ab" | "ab" || true
48-
"abc.*" | "abc.ab" | "abc" | "ab" || true
49-
"abc.*" | "abc.ab" | "abc" | "cd" || false
50-
"abc.cd" | "abc.*" | "abc" | "cd" || true
51-
"abc.*" | "abc.*" | "abc" | "cd" || true
52-
"abc.*,def.*" | "abc.*" | "def" | "ab" || false
32+
configIncludeFilter | configExcludeFilter | keyspace | columnfamily || result
33+
null | null | "defg" | "gh" || false
34+
"abc.*" | null | "abc" | "cd" || false
35+
"abc.*" | null | "ab" | "cd" || true
36+
null | "abc.de" | "abc" | "def" || false
37+
null | "abc.de" | "abc" | "de" || true
38+
"abc.*,def.*" | null | "abc" | "cd" || false
39+
"abc.*,def.*" | null | "def" | "ab" || false
40+
"abc.*,def.*" | null | "ab" | "cd" || true
41+
"abc.*,def.*" | null | "df" | "ab" || true
42+
null | "abc.de,fg.hi" | "abc" | "def" || false
43+
null | "abc.de,fg.hi" | "abc" | "de" || true
44+
null | "abc.de,fg.hi" | "fg" | "hijk" || false
45+
null | "abc.de,fg.hi" | "fg" | "hi" || true
46+
"abc.*" | "ab.ab" | "ab" | "cd" || true
47+
"abc.*" | "ab.ab" | "ab" | "ab" || true
48+
"abc.*" | "abc.ab" | "abc" | "ab" || true
49+
"abc.*" | "abc.ab" | "abc" | "cd" || false
50+
"abc.cd" | "abc.*" | "abc" | "cd" || true
51+
"abc.*" | "abc.*" | "abc" | "cd" || true
52+
"abc.*,def.*" | "abc.*" | "def" | "ab" || false
53+
null | null | "system" | "local" || true
54+
null | null | "system" | "peers" || true
55+
null | null | "system" | "hints" || true
56+
null | null | "system" | "compactions_in_progress" || true
57+
null | null | "system" | "LocationInfo" || true
58+
null | null | "system" | "peers_v2" || false
5359
}
5460

5561

5662
def "Expected exception KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter"() {
5763
when:
58-
new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).isFiltered(keyspace, columnfamily)
64+
new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).shouldOmitFromBackup(keyspace, columnfamily)
65+
66+
then:
67+
thrown(ExcpectedException)
68+
69+
where:
70+
configIncludeFilter | configExcludeFilter | keyspace | columnfamily || ExcpectedException
71+
null | "def" | "defg" | null || IllegalArgumentException
72+
"abc" | null | null | "cd" || IllegalArgumentException
73+
}
74+
75+
def "shouldOmitFromRestore for KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter is #result"() {
76+
expect:
77+
new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).shouldOmitFromRestore(keyspace, columnfamily) == result
78+
79+
where:
80+
configIncludeFilter | configExcludeFilter | keyspace | columnfamily || result
81+
null | null | "defg" | "gh" || false
82+
"abc.*" | null | "abc" | "cd" || false
83+
"abc.*" | null | "ab" | "cd" || true
84+
null | "abc.de" | "abc" | "def" || false
85+
null | "abc.de" | "abc" | "de" || true
86+
"abc.*,def.*" | null | "abc" | "cd" || false
87+
"abc.*,def.*" | null | "def" | "ab" || false
88+
"abc.*,def.*" | null | "ab" | "cd" || true
89+
"abc.*,def.*" | null | "df" | "ab" || true
90+
null | "abc.de,fg.hi" | "abc" | "def" || false
91+
null | "abc.de,fg.hi" | "abc" | "de" || true
92+
null | "abc.de,fg.hi" | "fg" | "hijk" || false
93+
null | "abc.de,fg.hi" | "fg" | "hi" || true
94+
"abc.*" | "ab.ab" | "ab" | "cd" || true
95+
"abc.*" | "ab.ab" | "ab" | "ab" || true
96+
"abc.*" | "abc.ab" | "abc" | "ab" || true
97+
"abc.*" | "abc.ab" | "abc" | "cd" || false
98+
"abc.cd" | "abc.*" | "abc" | "cd" || true
99+
"abc.*" | "abc.*" | "abc" | "cd" || true
100+
"abc.*,def.*" | "abc.*" | "def" | "ab" || false
101+
null | null | "system" | "local" || true
102+
null | null | "system" | "peers" || true
103+
null | null | "system" | "hints" || true
104+
null | null | "system" | "compactions_in_progress" || true
105+
null | null | "system" | "LocationInfo" || true
106+
null | null | "system" | "peers_v2" || true
107+
}
108+
109+
def "Expected exception KS #keyspace and CF #columnfamily with configuration include #configIncludeFilter and exclude #configExcludeFilter"() {
110+
when:
111+
new BackupRestoreUtil(configIncludeFilter, configExcludeFilter).shouldOmitFromRestore(keyspace, columnfamily)
59112

60113
then:
61114
thrown(ExcpectedException)

0 commit comments

Comments
 (0)