Skip to content

Commit 07307c9

Browse files
nizhikovnizhikov
authored
IGNITE-27380 DataStorageConfiguration#extraSnapshotPaths added (#12602)
1 parent 3534e59 commit 07307c9

File tree

9 files changed

+889
-31
lines changed

9 files changed

+889
-31
lines changed

‎modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,14 +216,26 @@ public class DataStorageConfiguration implements Serializable {
216216

217217
/**
218218
* Additional directories where index and partition files are stored.
219-
* User may want to use dedicated storage for cache is server has several physical disks.
219+
* User may want to use dedicated storage for cache if server has several physical disks.
220220
* Spreading load across several disks can improve performance.
221221
*
222+
* @see #getStoragePath()
222223
* @see CacheConfiguration#setStoragePaths(String...)
223224
*/
224225
@IgniteExperimental
225226
private String[] extraStoragePaths;
226227

228+
/**
229+
* Additional directories where snapshot files are stored.
230+
* User may want to use dedicated storage for cache if server has several physical disks.
231+
* Spreading snapshot across several disks can improve performance.
232+
*
233+
* @see IgniteConfiguration#getSnapshotPath()
234+
* @see CacheConfiguration#setStoragePaths(String...)
235+
*/
236+
@IgniteExperimental
237+
private String[] extraSnapshotPaths;
238+
227239
/** Checkpoint frequency. */
228240
private long checkpointFreq = DFLT_CHECKPOINT_FREQ;
229241

@@ -574,6 +586,14 @@ public String[] getExtraStoragePaths() {
574586
return extraStoragePaths;
575587
}
576588

589+
/**
590+
* @return Additional directories for snapshots.
591+
*/
592+
@IgniteExperimental
593+
public String[] getExtraSnapshotPaths() {
594+
return extraSnapshotPaths;
595+
}
596+
577597
/**
578598
* Sets a path to the root directory where the Persistent Store will persist data and indexes.
579599
* By default, the Persistent Store's files are located under Ignite work directory.
@@ -601,6 +621,20 @@ public DataStorageConfiguration setExtraStoragePaths(String... extraStoragePaths
601621
return this;
602622
}
603623

624+
/**
625+
* Sets a paths to the root directories where the snapshot files stored.
626+
* By default, {@link IgniteConfiguration#getSnapshotPath()} used.
627+
* Length of {@code extraSnapshotPaths} must be equal to the length of {@link #getExtraStoragePaths()}.
628+
*
629+
* @param extraSnapshotPaths Extra snapshot paths where snapshot files can be stored.
630+
* @return {@code this} for chaining.
631+
*/
632+
public DataStorageConfiguration setExtraSnapshotPaths(String... extraSnapshotPaths) {
633+
this.extraSnapshotPaths = extraSnapshotPaths;
634+
635+
return this;
636+
}
637+
604638
/**
605639
* Gets checkpoint frequency.
606640
*

‎modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1959,6 +1959,19 @@ private void initializeDataStorageConfiguration(IgniteConfiguration cfg) throws
19591959
throw new IgniteCheckedException("DataStorageConfiguration contains duplicates " +
19601960
"[storagePath=" + dsCfg.getStoragePath() + ", extraStoragePaths=" + extraStorages + ']');
19611961
}
1962+
1963+
List<String> extraSnapshotStorages = F.asList(dsCfg.getExtraSnapshotPaths());
1964+
1965+
if (!extraSnapshotStorages.isEmpty() && extraStorages.size() != extraSnapshotStorages.size()) {
1966+
throw new IgniteCheckedException("DataStorageConfiguration error. " +
1967+
"Size of extraSnapshotPaths must be equal to extraStoragePath " +
1968+
"[extraStoragePaths=" + extraStorages + ", extraSnapshotPaths=" + extraSnapshotStorages + ']');
1969+
}
1970+
1971+
if (extraSnapshotStorages.size() != new HashSet<>(extraSnapshotStorages).size()) {
1972+
throw new IgniteCheckedException("DataStorageConfiguration contains duplicates " +
1973+
"[extraSnapshotPaths=" + extraSnapshotStorages + ']');
1974+
}
19621975
}
19631976

19641977
if (cfg.getMemoryConfiguration() != null || cfg.getPersistentStoreConfiguration() != null)

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -403,10 +403,7 @@ protected NodeFileTree(IgniteConfiguration cfg, File root, String folderName, bo
403403
walCdc = rootRelative(DFLT_WAL_CDC_PATH);
404404
}
405405

406-
extraStorages = extraStorages(
407-
dsCfg,
408-
storagePath -> resolveDirectory(Path.of(storagePath, DB_DIR).toString())
409-
);
406+
extraStorages = extraStorages(dsCfg);
410407
}
411408

412409
/** @return Node storage directory. */
@@ -989,12 +986,15 @@ private File cacheStorage(@Nullable String storagePath, String cacheDirName) {
989986
* @return Node storages.
990987
* @see DataStorageConfiguration#setExtraStoragePaths(String...)
991988
*/
992-
private Map<String, File> extraStorages(@Nullable DataStorageConfiguration dsCfg, Function<String, File> resolver) {
989+
private Map<String, File> extraStorages(@Nullable DataStorageConfiguration dsCfg) {
993990
if (dsCfg == null || F.isEmpty(dsCfg.getExtraStoragePaths()))
994991
return Collections.emptyMap();
995992

996993
return Arrays.stream(dsCfg.getExtraStoragePaths())
997-
.collect(Collectors.toMap(Function.identity(), resolver));
994+
.collect(Collectors.toMap(
995+
Function.identity(),
996+
p -> resolveDirectory(Path.of(p, DB_DIR).toString()))
997+
);
998998
}
999999

10001000
/**
@@ -1129,7 +1129,7 @@ private Stream<File> filesInStorages(FileFilter filter) {
11291129
* @param cfg Configured directory path.
11301130
* @return Initialized directory.
11311131
*/
1132-
private File resolveDirectory(String cfg) {
1132+
protected File resolveDirectory(String cfg) {
11331133
File sharedDir = new File(cfg);
11341134

11351135
return sharedDir.isAbsolute()

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotFileTree.java‎

Lines changed: 43 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ public SnapshotFileTree(
134134
this.path = path;
135135
this.consId = consId;
136136

137-
Map<String, File> snpExtraStorages = snapshotExtraStorages(ft, cfg.getSnapshotPath());
137+
Map<String, File> snpExtraStorages = snapshotExtraStorages(ft, cfg);
138138

139139
if (snpExtraStorages.isEmpty())
140140
extraStorages.clear();
@@ -380,27 +380,54 @@ public File meta() {
380380
* This will distribute workload to all physical device on host.
381381
*
382382
* @param ft Node file tree.
383-
* @param snpDfltPath Snapshot default path.
383+
* @param cfg Ignite configuration.
384384
*/
385-
private Map<String, File> snapshotExtraStorages(NodeFileTree ft, String snpDfltPath) {
385+
private Map<String, File> snapshotExtraStorages(NodeFileTree ft, IgniteConfiguration cfg) {
386+
DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
387+
386388
// If path provided then create snapshot inside it, only.
387-
// Same rule applies if absolute path to the snapshot root dir configured.
388-
if (path != null || new File(snpDfltPath).isAbsolute())
389+
if (path != null || dsCfg == null)
390+
return Collections.emptyMap();
391+
392+
String[] extraStorages = dsCfg.getExtraStoragePaths();
393+
394+
if (extraStorages == null)
389395
return Collections.emptyMap();
390396

391-
Map<String, File> snpExtraStorages = new HashMap<>();
397+
String[] extraSnpPaths = dsCfg.getExtraSnapshotPaths();
398+
399+
boolean hasExtraSnpPaths = extraSnpPaths != null && extraSnpPaths.length > 0;
392400

393-
ft.extraStorages().forEach((cfgStoragePath, storagePath) -> {
394-
// In case we want to make snapshot in several folders the paths will be the following:
395-
// {storage_path}/db/{folder_name} - node cache storage.
396-
// {storage_path}/snapshots/{snp_name}/db/{folder_name} - snapshot cache storage.
397-
snpExtraStorages.put(
398-
cfgStoragePath,
399-
new File(storagePath.getParentFile().getParentFile(), Path.of(snpDfltPath, name, DB_DIR, folderName()).toString())
400-
);
401-
});
401+
String snpDfltPath = cfg.getSnapshotPath();
402+
403+
boolean snpDfltPathAbsolute = new File(snpDfltPath).isAbsolute();
404+
405+
// If absolute path to the snapshot root dir configured and no extra snapshot paths, then use snapshot root, only.
406+
if (snpDfltPathAbsolute && !hasExtraSnpPaths)
407+
return Collections.emptyMap();
408+
409+
Map<String, File> snpRoots = new HashMap<>();
410+
411+
String snpUniqSuffix = snpDfltPathAbsolute
412+
? Path.of(name, DB_DIR, folderName()).toString()
413+
: Path.of(snpDfltPath, name, DB_DIR, folderName()).toString();
414+
415+
for (int i = 0; i < extraStorages.length; i++) {
416+
File snpRoot = hasExtraSnpPaths
417+
// Extra snapshot paths configured as "root" related.
418+
// folderName can be different from local (NodeFileTree ft) one in case snapshot restored on smaller topology
419+
// and data from some node copied to local.
420+
// resolveDirectory returns in form {root}/{extra_snp_path}/{folder_name} so parent required.
421+
? ft.resolveDirectory(extraSnpPaths[i]).getParentFile()
422+
// In case we want to make snapshot in several folders the paths will be the following:
423+
// {storage_path}/db/{folder_name} - node cache storage.
424+
// {storage_path} - snapshot root.
425+
: ft.extraStorages.get(extraStorages[i]).getParentFile().getParentFile();
426+
427+
snpRoots.put(extraStorages[i], new File(snpRoot, snpUniqSuffix));
428+
}
402429

403-
return snpExtraStorages;
430+
return snpRoots;
404431
}
405432

406433
/**

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/CustomCacheStorageConfigurationSelfTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,48 @@ public void testDuplicatesStoragePathThrows() {
112112
);
113113
}
114114

115+
/** */
116+
@Test
117+
public void testIncorrectSnapshotPathsThrows() {
118+
assertThrows(
119+
log,
120+
() -> startGrid(new IgniteConfiguration().setDataStorageConfiguration(new DataStorageConfiguration()
121+
.setStoragePath(myPath.getAbsolutePath())
122+
.setExtraStoragePaths(myPath2.getAbsolutePath())
123+
.setExtraSnapshotPaths("snppath1", "snppath2"))),
124+
IgniteCheckedException.class,
125+
"DataStorageConfiguration error. Size of extraSnapshotPaths must be equal to extraStoragePath"
126+
);
127+
128+
assertThrows(
129+
log,
130+
() -> startGrid(new IgniteConfiguration().setDataStorageConfiguration(new DataStorageConfiguration()
131+
.setStoragePath(myPath.getAbsolutePath())
132+
.setExtraSnapshotPaths("snppath1"))),
133+
IgniteCheckedException.class,
134+
"DataStorageConfiguration error. Size of extraSnapshotPaths must be equal to extraStoragePath"
135+
);
136+
137+
assertThrows(
138+
log,
139+
() -> startGrid(new IgniteConfiguration().setDataStorageConfiguration(new DataStorageConfiguration()
140+
.setStoragePath(myPath.getAbsolutePath())
141+
.setExtraSnapshotPaths("snppath1", "snppath2"))),
142+
IgniteCheckedException.class,
143+
"DataStorageConfiguration error. Size of extraSnapshotPaths must be equal to extraStoragePath"
144+
);
145+
146+
assertThrows(
147+
log,
148+
() -> startGrid(new IgniteConfiguration().setDataStorageConfiguration(new DataStorageConfiguration()
149+
.setStoragePath(myPath.getAbsolutePath())
150+
.setExtraStoragePaths(myPath2.getAbsolutePath(), myPath3.getAbsolutePath())
151+
.setExtraSnapshotPaths("snppath1", "snppath1"))),
152+
IgniteCheckedException.class,
153+
"DataStorageConfiguration contains duplicates [extraSnapshotPaths="
154+
);
155+
}
156+
115157
/** */
116158
@Test
117159
public void testCacheUnknownStoragePathThrows() throws Exception {

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotCreationNonDefaultStoragePathTest.java‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,19 @@
3939
* Test snapshot can be created when {@link DataStorageConfiguration#setStoragePath(String)} used.
4040
*/
4141
public class SnapshotCreationNonDefaultStoragePathTest extends AbstractDataRegionRelativeStoragePathTest {
42+
/** */
43+
protected String[] extraSnpPaths = new String[] {
44+
STORAGE_PATH_2,
45+
IDX_PATH
46+
};
47+
48+
/** {@inheritDoc} */
49+
@Override protected void beforeTest() throws Exception {
50+
super.beforeTest();
51+
52+
U.delete(new File(U.defaultWorkDirectory()));
53+
}
54+
4255
/** {@inheritDoc} */
4356
@Override protected DataStorageConfiguration dataStorageConfiguration() {
4457
return new DataStorageConfiguration()
@@ -121,12 +134,14 @@ public void testRestoreOnSmallerTopology() throws Exception {
121134

122135
srv.context().cache().context().snapshotMgr().createSnapshot("mysnp2", fullPathSnp.getAbsolutePath(), false, false).get();
123136

124-
String grid1ConsId = consId(grid(1).configuration());
137+
NodeFileTree ft = grid(1).context().pdsFolderResolver().fileTree();
125138

126139
stopGrid(1);
127140

128141
resetBaselineTopology();
129142

143+
ft.allStorages().forEach(U::delete);
144+
130145
BiConsumer<String, String> check = (name, path) -> {
131146
for (CacheConfiguration<?, ?> ccfg : ccfgs())
132147
grid(0).destroyCache(ccfg.getName());
@@ -155,15 +170,16 @@ public void testRestoreOnSmallerTopology() throws Exception {
155170
"No snapshot metadatas found for the baseline nodes with consistent ids: "
156171
);
157172

158-
Path[] copyStoppedNodeData = new Path[] {
159-
Path.of(DFLT_SNAPSHOT_DIRECTORY, "mysnp"),
160-
Path.of(STORAGE_PATH_2, DFLT_SNAPSHOT_DIRECTORY, "mysnp"),
161-
Path.of(IDX_PATH, DFLT_SNAPSHOT_DIRECTORY, "mysnp")
162-
};
173+
Path[] copyStoppedNodeData = new Path[extraSnpPaths.length + 1];
174+
175+
copyStoppedNodeData[0] = Path.of(DFLT_SNAPSHOT_DIRECTORY, "mysnp");
176+
177+
for (int i = 0; i < extraSnpPaths.length; i++)
178+
copyStoppedNodeData[i + 1] = Path.of(extraSnpPaths[i], DFLT_SNAPSHOT_DIRECTORY, "mysnp");
163179

164180
for (Path copyPath : copyStoppedNodeData) {
165181
FileUtils.copyDirectory(
166-
Path.of(U.defaultWorkDirectory(), grid1ConsId, copyPath.toString()).toFile(),
182+
Path.of(U.defaultWorkDirectory(), ft.folderName(), copyPath.toString()).toFile(),
167183
Path.of(U.defaultWorkDirectory(), consId(grid(0).configuration()), copyPath.toString()).toFile()
168184
);
169185
}

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/filename/SnapshotExtraStoragesTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.ignite.internal.processors.cache.persistence.filename;
19+
20+
import org.apache.ignite.configuration.DataStorageConfiguration;
21+
22+
/**
23+
* Test snapshot can be created when {@link DataStorageConfiguration#setStoragePath(String)} used.
24+
*/
25+
public class SnapshotExtraStoragesTest extends SnapshotCreationNonDefaultStoragePathTest {
26+
/** {@inheritDoc} */
27+
@Override protected DataStorageConfiguration dataStorageConfiguration() {
28+
extraSnpPaths = new String[] {
29+
"snp_path",
30+
"snp_path2"
31+
};
32+
33+
return super.dataStorageConfiguration().setExtraSnapshotPaths(storagePath("snp_path"), storagePath("snp_path2"));
34+
}
35+
}

0 commit comments

Comments
 (0)