Skip to content

Commit 5901b7f

Browse files
nizhikovnizhikov
authored
IGNITE-26860 Add configOnly flag to dump (#12476)
1 parent 04a30a8 commit 5901b7f

File tree

13 files changed

+162
-40
lines changed

13 files changed

+162
-40
lines changed

‎modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import java.util.Collections;
2929
import java.util.HashMap;
3030
import java.util.HashSet;
31-
import java.util.Iterator;
3231
import java.util.List;
3332
import java.util.Map;
3433
import java.util.Set;
@@ -367,6 +366,7 @@ private static GridKernalContext standaloneKernalContext(SnapshotFileTree sft, I
367366
*/
368367
private GroupsConfigs groupsConfigs(Dump dump) {
369368
Map<Integer, List<String>> grpsToNodes = new HashMap<>();
369+
List<StoredCacheData> ccfgs = new ArrayList<>();
370370

371371
Set<Integer> grpIds = cfg.groupNames() != null
372372
? Arrays.stream(cfg.groupNames()).map(CU::cacheId).collect(Collectors.toSet())
@@ -377,29 +377,24 @@ private GroupsConfigs groupsConfigs(Dump dump) {
377377
: null;
378378

379379
for (SnapshotMetadata meta : dump.metadata()) {
380-
for (Integer grp : meta.partitions().keySet()) {
381-
if (grpIds == null || grpIds.contains(grp))
382-
grpsToNodes.computeIfAbsent(grp, key -> new ArrayList<>()).add(meta.folderName());
383-
}
384-
}
380+
for (Integer grp : meta.cacheGroupIds()) {
381+
if (grpIds != null && !grpIds.contains(grp))
382+
continue;
385383

386-
Iterator<Map.Entry<Integer, List<String>>> grpToNodesIter = grpsToNodes.entrySet().iterator();
387-
List<StoredCacheData> ccfgs = new ArrayList<>();
384+
// Read all group configs from single node.
385+
List<StoredCacheData> grpCaches = dump.configs(meta.folderName(), grp, cacheIds);
388386

389-
while (grpToNodesIter.hasNext()) {
390-
Map.Entry<Integer, List<String>> grpToNodes = grpToNodesIter.next();
387+
if (F.isEmpty(grpCaches))
388+
continue;
391389

392-
// Read all group configs from single node.
393-
List<StoredCacheData> grpCaches = dump.configs(F.first(grpToNodes.getValue()), grpToNodes.getKey(), cacheIds);
390+
if (!grpsToNodes.containsKey(grp)) {
391+
grpsToNodes.put(grp, new ArrayList<>());
394392

395-
if (grpCaches.isEmpty()) {
396-
// Remove whole group to skip files read.
397-
grpToNodesIter.remove();
393+
ccfgs.addAll(grpCaches);
394+
}
398395

399-
continue;
396+
grpsToNodes.get(grp).add(meta.folderName());
400397
}
401-
402-
ccfgs.addAll(grpCaches);
403398
}
404399

405400
// Optimize - skip whole cache if only one in group!

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,9 @@ private IgniteInternalFuture<SnapshotOperationResponse> initLocalFullSnapshot(
976976
if (!isPersistenceEnabled(cctx.gridConfig()) && req.snapshotPath() == null)
977977
ft.mkdirSnapshotsRoot();
978978

979+
if (req.configOnly() && !req.dump())
980+
throw new IgniteException("Config only flag supported only for dump");
981+
979982
Map<Integer, Set<Integer>> parts = new HashMap<>();
980983

981984
// Prepare collection of pairs group and appropriate cache partition to be snapshot.
@@ -988,7 +991,7 @@ private IgniteInternalFuture<SnapshotOperationResponse> initLocalFullSnapshot(
988991

989992
AffinityTopologyVersion topVer = grpCtx.affinity().lastVersion();
990993

991-
if (req.onlyPrimary()) {
994+
if (req.onlyPrimary() && !req.configOnly()) {
992995
Set<Integer> include = new HashSet<>(grpCtx.affinity().primaryPartitions(cctx.localNodeId(), topVer));
993996

994997
include.remove(INDEX_PARTITION);
@@ -1012,6 +1015,7 @@ private IgniteInternalFuture<SnapshotOperationResponse> initLocalFullSnapshot(
10121015
req.dump(),
10131016
req.compress(),
10141017
req.encrypt(),
1018+
req.configOnly(),
10151019
locSndrFactory.apply(req.snapshotFileTree())
10161020
);
10171021

@@ -1639,7 +1643,7 @@ private boolean cancelLocalSnapshotTask0(Function<AbstractSnapshotFutureTask<?>,
16391643

16401644
/** {@inheritDoc} */
16411645
@Override public IgniteFuture<Void> createDump(String name, @Nullable Collection<String> cacheGrpNames) {
1642-
return createSnapshot(name, null, cacheGrpNames, false, false, true, false, false, false);
1646+
return createSnapshot(name, null, cacheGrpNames, false, false, true, false, false, false, false);
16431647
}
16441648

16451649
/**
@@ -1882,7 +1886,7 @@ public IgniteFutureImpl<Void> createSnapshot(
18821886
boolean incremental,
18831887
boolean onlyPrimary
18841888
) {
1885-
return createSnapshot(name, snpPath, null, incremental, onlyPrimary, false, false, false, false);
1889+
return createSnapshot(name, snpPath, null, incremental, onlyPrimary, false, false, false, false, false);
18861890
}
18871891

18881892
/**
@@ -1902,6 +1906,7 @@ public IgniteFutureImpl<Void> createSnapshot(
19021906
* @param compress If {@code true} then compress partition files.
19031907
* @param encrypt If {@code true} then content of dump encrypted.
19041908
* @param inclDs If {@code true} then data structures caches will be included in dump.
1909+
* @param configOnly If {@code true} then only cache config and metadata included in snapshot.
19051910
* @return Future which will be completed when a process ends.
19061911
*/
19071912
public IgniteFutureImpl<Void> createSnapshot(
@@ -1913,7 +1918,8 @@ public IgniteFutureImpl<Void> createSnapshot(
19131918
boolean dump,
19141919
boolean compress,
19151920
boolean encrypt,
1916-
boolean inclDs
1921+
boolean inclDs,
1922+
boolean configOnly
19171923
) {
19181924
A.notNullOrEmpty(name, "Snapshot name cannot be null or empty.");
19191925
A.ensure(U.alphanumericUnderscore(name), "Snapshot name must satisfy the following name pattern: a-zA-Z0-9_");
@@ -1922,6 +1928,7 @@ public IgniteFutureImpl<Void> createSnapshot(
19221928
A.ensure(!(cacheGrpNames != null && !dump), "Cache group names filter supported only for dump");
19231929
A.ensure(!compress || dump, "Compression is supported only for dumps");
19241930
A.ensure(!inclDs || dump, "Data structures can't be written into snapshot");
1931+
A.ensure(!configOnly || dump, "Config only supported only for dump");
19251932

19261933
try {
19271934
cctx.kernalContext().security().authorize(ADMIN_SNAPSHOT);
@@ -1951,7 +1958,8 @@ public IgniteFutureImpl<Void> createSnapshot(
19511958
dump,
19521959
compress,
19531960
encrypt,
1954-
inclDs
1961+
inclDs,
1962+
configOnly
19551963
),
19561964
options(Collections.singletonList(crd)).withFailoverDisabled()
19571965
));
@@ -2072,7 +2080,8 @@ else if (grps.isEmpty())
20722080
onlyPrimary,
20732081
dump,
20742082
compress,
2075-
encrypt
2083+
encrypt,
2084+
configOnly
20762085
);
20772086

20782087
startSnpProc.start(snpFut0.rqId, snpOpReq);
@@ -2363,6 +2372,7 @@ public GridCloseableIterator<CacheDataRow> partitionRowIterator(
23632372
* @param dump {@code true} if cache group dump must be created.
23642373
* @param compress If {@code true} then compress partition files.
23652374
* @param encrypt If {@code true} then content of dump encrypted.
2375+
* @param configOnly If {@code true} then only cache config and metadata included in snapshot.
23662376
* @param snpSndr Factory which produces snapshot receiver instance.
23672377
* @return Snapshot operation task which should be registered on checkpoint to run.
23682378
*/
@@ -2375,6 +2385,7 @@ AbstractSnapshotFutureTask<?> registerSnapshotTask(
23752385
boolean dump,
23762386
boolean compress,
23772387
boolean encrypt,
2388+
boolean configOnly,
23782389
SnapshotSender snpSndr
23792390
) {
23802391
AbstractSnapshotFutureTask<?> task = registerTask(sft.name(), dump
@@ -2387,7 +2398,8 @@ AbstractSnapshotFutureTask<?> registerSnapshotTask(
23872398
snpSndr,
23882399
parts,
23892400
compress,
2390-
encrypt
2401+
encrypt,
2402+
configOnly
23912403
)
23922404
: new SnapshotFutureTask(
23932405
cctx,
@@ -4238,6 +4250,9 @@ private static class CreateSnapshotCallable implements IgniteCallable<Void> {
42384250
/** If {@code true} then data structures caches will be included in dump. */
42394251
private final boolean inclDs;
42404252

4253+
/** If {@code true} then only cache config and metadata included in snapshot. */
4254+
private final boolean configOnly;
4255+
42414256
/** Auto-injected grid instance. */
42424257
@IgniteInstanceResource
42434258
private transient IgniteEx ignite;
@@ -4251,6 +4266,7 @@ private static class CreateSnapshotCallable implements IgniteCallable<Void> {
42514266
* @param comprParts If {@code true} then compress partition files.
42524267
* @param encrypt If {@code true} then content of dump encrypted.
42534268
* @param inclDs If {@code true} then data structures caches will be included in dump.
4269+
* @param configOnly If {@code true} then only cache config and metadata included in snapshot.
42544270
*/
42554271
public CreateSnapshotCallable(
42564272
String snpName,
@@ -4260,7 +4276,8 @@ public CreateSnapshotCallable(
42604276
boolean dump,
42614277
boolean comprParts,
42624278
boolean encrypt,
4263-
boolean inclDs
4279+
boolean inclDs,
4280+
boolean configOnly
42644281
) {
42654282
this.snpName = snpName;
42664283
this.cacheGrpNames = cacheGrpNames;
@@ -4270,6 +4287,7 @@ public CreateSnapshotCallable(
42704287
this.comprParts = comprParts;
42714288
this.encrypt = encrypt;
42724289
this.inclDs = inclDs;
4290+
this.configOnly = configOnly;
42734291
}
42744292

42754293
/** {@inheritDoc} */
@@ -4286,7 +4304,8 @@ public CreateSnapshotCallable(
42864304
dump,
42874305
comprParts,
42884306
encrypt,
4289-
inclDs
4307+
inclDs,
4308+
configOnly
42904309
).get();
42914310
}
42924311

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperationRequest.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ public class SnapshotOperationRequest extends AbstractSnapshotOperationRequest {
8282
/** If {@code true} then content of dump encrypted. */
8383
private final boolean encrypt;
8484

85+
/** If {@code true} then only cache config and metadata included in snapshot. */
86+
private final boolean configOnly;
87+
8588
/**
8689
* @param reqId Request ID.
8790
* @param opNodeId Operational node ID.
@@ -95,6 +98,7 @@ public class SnapshotOperationRequest extends AbstractSnapshotOperationRequest {
9598
* @param dump If {@code true} then create dump.
9699
* @param compress If {@code true} then compress partition files.
97100
* @param encrypt If {@code true} then content of dump encrypted.
101+
* @param configOnly If {@code true} then only cache config and metadata included in snapshot.
98102
*/
99103
public SnapshotOperationRequest(
100104
UUID reqId,
@@ -108,7 +112,8 @@ public SnapshotOperationRequest(
108112
boolean onlyPrimary,
109113
boolean dump,
110114
boolean compress,
111-
boolean encrypt
115+
boolean encrypt,
116+
boolean configOnly
112117
) {
113118
super(reqId, snpName, snpPath, grps, incIdx, nodes);
114119

@@ -119,6 +124,7 @@ public SnapshotOperationRequest(
119124
this.dump = dump;
120125
this.compress = compress;
121126
this.encrypt = encrypt;
127+
this.configOnly = configOnly;
122128
}
123129

124130
/**
@@ -172,6 +178,11 @@ public boolean encrypt() {
172178
return encrypt;
173179
}
174180

181+
/** @return If {@code true} then only cache config and metadata included in snapshot. */
182+
public boolean configOnly() {
183+
return configOnly;
184+
}
185+
175186
/**
176187
* @return Flag indicating that the {@link DistributedProcessType#START_SNAPSHOT} phase has completed.
177188
*/

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,10 +112,19 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext<?, ?> cctx) {
112112

113113
SnapshotMetadata meta = opCtx.metadata();
114114

115-
Map<Integer, Set<Integer>> grps = F.isEmpty(opCtx.groups())
116-
? new HashMap<>(meta.partitions())
117-
: opCtx.groups().stream().map(CU::cacheId)
118-
.collect(Collectors.toMap(Function.identity(), grpId -> meta.partitions().getOrDefault(grpId, Collections.emptySet())));
115+
Map<Integer, Set<Integer>> grps;
116+
117+
if (F.isEmpty(opCtx.groups()))
118+
grps = new HashMap<>(meta.partitions());
119+
else {
120+
grps = opCtx.groups().stream()
121+
.map(CU::cacheId)
122+
.filter(meta.partitions()::containsKey) // Filter out groups for which there are no partitions in snapshot.
123+
.collect(Collectors.toMap(
124+
Function.identity(),
125+
grpId -> meta.partitions().get(grpId))
126+
);
127+
}
119128

120129
if (type() == SnapshotHandlerType.CREATE) {
121130
grps.entrySet().removeIf(e -> {

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,7 @@ public IgniteFutureImpl<Void> start(
408408
onlyPrimary,
409409
false,
410410
false,
411+
false,
411412
false
412413
);
413414

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/CreateDumpFutureTask.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.IOException;
2323
import java.io.Serializable;
2424
import java.nio.ByteBuffer;
25+
import java.util.Collections;
2526
import java.util.HashMap;
2627
import java.util.HashSet;
2728
import java.util.List;
@@ -92,6 +93,9 @@ public class CreateDumpFutureTask extends AbstractCreateSnapshotFutureTask imple
9293
/** If {@code true} then compress partition files. */
9394
private final boolean compress;
9495

96+
/** If {@code true} then only cache config and metadata included in snapshot. */
97+
private final boolean configOnly;
98+
9599
/** Dump transfer rate limiter. */
96100
private final BasicRateLimiter rateLimiter;
97101

@@ -146,6 +150,7 @@ public class CreateDumpFutureTask extends AbstractCreateSnapshotFutureTask imple
146150
* @param parts Parts to dump.
147151
* @param compress If {@code true} then compress partition files.
148152
* @param encrypt If {@code true} then content of dump encrypted.
153+
* @param configOnly If {@code true} then only cache config and metadata included in snapshot.
149154
*/
150155
public CreateDumpFutureTask(
151156
GridCacheSharedContext<?, ?> cctx,
@@ -157,7 +162,8 @@ public CreateDumpFutureTask(
157162
SnapshotSender snpSndr,
158163
Map<Integer, Set<Integer>> parts,
159164
boolean compress,
160-
boolean encrypt
165+
boolean encrypt,
166+
boolean configOnly
161167
) {
162168
super(
163169
cctx,
@@ -171,6 +177,7 @@ public CreateDumpFutureTask(
171177
this.ioFactory = compress ? new WriteOnlyZipFileIOFactory(ioFactory) : new BufferedFileIOFactory(ioFactory);
172178

173179
this.compress = compress;
180+
this.configOnly = configOnly;
174181
this.rateLimiter = rateLimiter;
175182
this.encKey = encrypt ? cctx.gridConfig().getEncryptionSpi().create() : null;
176183
this.encThLocBufs = encrypt ? new ConcurrentHashMap<>() : null;
@@ -199,6 +206,12 @@ public CreateDumpFutureTask(
199206

200207
/** {@inheritDoc} */
201208
@Override protected void processPartitions() throws IgniteCheckedException {
209+
if (configOnly) {
210+
parts.keySet().forEach(grpId -> processed.put(grpId, Collections.emptySet()));
211+
212+
return;
213+
}
214+
202215
super.processPartitions();
203216

204217
processed.values().forEach(parts -> parts.remove(INDEX_PARTITION));

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/AbstractSnapshotSelfTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -792,7 +792,7 @@ public static void doSnapshotCancellationTest(
792792
List<BlockingExecutor> execs = setBlockingSnapshotExecutor(srvs);
793793

794794
IgniteFuture<Void> fut = snp(startCli).createSnapshot(SNAPSHOT_NAME, null, null, false,
795-
false, dump, false, false, false);
795+
false, dump, false, false, false, false);
796796

797797
for (BlockingExecutor exec : execs)
798798
exec.waitForBlocked(30_000L);
@@ -830,6 +830,7 @@ protected static IgniteInternalFuture<?> startLocalSnapshotTask(
830830
false,
831831
false,
832832
false,
833+
false,
833834
snpSndr
834835
);
835836

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/EncryptedSnapshotTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ public void testSnapshotTaskIsBlockedWithoutMetastore() throws Exception {
296296
false,
297297
false,
298298
false,
299+
false,
299300
snp(ig).localSnapshotSenderFactory().apply(sft)
300301
).get(TIMEOUT),
301302
IgniteCheckedException.class,

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManagerSelfTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ public void testSnapshotLocalPartitionMultiCpWithLoad() throws Exception {
162162
false,
163163
false,
164164
false,
165+
false,
165166
new DelegateSnapshotSender(log, mgr.snapshotExecutorService(), mgr.localSnapshotSenderFactory().apply(sft)) {
166167
@Override public void sendPart0(File from, File to, @Nullable String storagePath, GroupPartitionId pair, Long length) {
167168
try {

‎modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/AbstractCacheDumpTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,7 @@ void createDump(IgniteEx ign, String name, @Nullable Collection<String> cacheGrp
526526
/** */
527527
void createDump(IgniteEx ign, String name, @Nullable Collection<String> cacheGrpNames, boolean comprParts) {
528528
ign.context().cache().context().snapshotMgr()
529-
.createSnapshot(name, null, cacheGrpNames, false, onlyPrimary, true, comprParts, encrypted, false).get();
529+
.createSnapshot(name, null, cacheGrpNames, false, onlyPrimary, true, comprParts, encrypted, false, false).get();
530530
}
531531

532532
/** */

0 commit comments

Comments
 (0)