Skip to content

Commit ae1cad6

Browse files
nizhikovnizhikov
authored
IGNITE-24390 Move WAL dirs to NodeFileTree (#11841)
1 parent 9e04bd9 commit ae1cad6

File tree

44 files changed

+355
-535
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+355
-535
lines changed

‎modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,7 @@ private void cleanPersistenceDir() throws Exception {
7070
SharedFileTree sft = new SharedFileTree(U.defaultWorkDirectory());
7171

7272
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false));
73-
U.delete(sft.db());
74-
U.delete(sft.marshaller());
75-
U.delete(sft.binaryMetaRoot());
73+
U.delete(sft.marshaller().getParentFile());
7674
}
7775

7876
/** */

‎modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/FoldersReuseCompatibilityTest.java‎

Lines changed: 6 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@
1818
package org.apache.ignite.compatibility.persistence;
1919

2020
import java.io.File;
21-
import java.io.IOException;
2221
import java.util.Arrays;
2322
import java.util.Set;
2423
import java.util.TreeSet;
24+
import java.util.function.Consumer;
2525
import org.apache.ignite.Ignite;
2626
import org.apache.ignite.IgniteCache;
2727
import org.apache.ignite.IgniteCheckedException;
@@ -224,36 +224,11 @@ private void assertNodeIndexesInFolder(Integer... indexes) throws IgniteCheckedE
224224
private void assertPdsDirsDefaultExist(String subDirName) throws IgniteCheckedException {
225225
NodeFileTree ft = nodeFileTree(subDirName);
226226

227-
assertTrue(ft.binaryMeta().exists() && ft.binaryMeta().isDirectory());
227+
Consumer<File> check = dir -> assertTrue(dir.exists() && dir.isDirectory());
228228

229-
assertDirectoryExist(PersistentStoreConfiguration.DFLT_WAL_STORE_PATH, subDirName);
230-
assertDirectoryExist(PersistentStoreConfiguration.DFLT_WAL_ARCHIVE_PATH, subDirName);
231-
assertDirectoryExist(PdsFolderResolver.DB_DEFAULT_FOLDER, subDirName);
229+
check.accept(ft.binaryMeta());
230+
check.accept(ft.wal());
231+
check.accept(ft.walArchive());
232+
check.accept(ft.nodeStorage());
232233
}
233-
234-
/**
235-
* Checks one folder existence
236-
*
237-
* @param subFolderNames subfolders array to touch
238-
* @throws IgniteCheckedException if IO error occur
239-
*/
240-
private void assertDirectoryExist(String... subFolderNames) throws IgniteCheckedException {
241-
File curFolder = new File(U.defaultWorkDirectory());
242-
243-
for (String name : subFolderNames) {
244-
curFolder = new File(curFolder, name);
245-
}
246-
247-
final String path;
248-
try {
249-
path = curFolder.getCanonicalPath();
250-
}
251-
catch (IOException e) {
252-
throw new IgniteCheckedException("Failed to convert path: [" + curFolder.getAbsolutePath() + "]", e);
253-
}
254-
255-
assertTrue("Directory " + Arrays.asList(subFolderNames).toString()
256-
+ " is expected to exist [" + path + "]", curFolder.exists() && curFolder.isDirectory());
257-
}
258-
259234
}

‎modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,9 +250,7 @@ private void checkDeleteLostSegmentLinks(List<Long> expBefore, List<Long> expAft
250250

251251
/** */
252252
private void checkLinks(IgniteEx srv, List<Long> expLinks) {
253-
FileWriteAheadLogManager wal0 = (FileWriteAheadLogManager)srv.context().cache().context().wal(true);
254-
255-
File[] links = wal0.walCdcDirectory().listFiles(WAL_SEGMENT_FILE_FILTER);
253+
File[] links = srv.context().pdsFolderResolver().fileTree().walCdc().listFiles(WAL_SEGMENT_FILE_FILTER);
256254

257255
assertEquals(expLinks.size(), links.length);
258256
Arrays.stream(links).map(File::toPath).map(FileWriteAheadLogManager::segmentIndex)

‎modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,8 @@ public String getWalPath() {
750750
* @return {@code this} for chaining.
751751
*/
752752
public DataStorageConfiguration setWalPath(String walStorePath) {
753+
A.notNull(walStorePath, "WAL");
754+
753755
this.walPath = walStorePath;
754756

755757
return this;
@@ -772,6 +774,8 @@ public String getWalArchivePath() {
772774
* @return {@code this} for chaining.
773775
*/
774776
public DataStorageConfiguration setWalArchivePath(String walArchivePath) {
777+
A.notNull(walArchivePath, "WAL archive");
778+
775779
this.walArchivePath = walArchivePath;
776780

777781
return this;
@@ -797,6 +801,8 @@ public String getCdcWalPath() {
797801
*/
798802
@IgniteExperimental
799803
public DataStorageConfiguration setCdcWalPath(String cdcWalPath) {
804+
A.notNull(cdcWalPath, "CDC WAL");
805+
800806
this.cdcWalPath = cdcWalPath;
801807

802808
return this;

‎modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,12 +78,6 @@ public class PersistentStoreConfiguration implements Serializable {
7878
/** Default wal always write full pages. */
7979
public static final boolean DFLT_WAL_ALWAYS_WRITE_FULL_PAGES = false;
8080

81-
/** Default wal directory. */
82-
public static final String DFLT_WAL_STORE_PATH = "db/wal";
83-
84-
/** Default wal archive directory. */
85-
public static final String DFLT_WAL_ARCHIVE_PATH = "db/wal/archive";
86-
8781
/** Default write throttling enabled. */
8882
public static final boolean DFLT_WRITE_THROTTLING_ENABLED = false;
8983

@@ -115,10 +109,10 @@ public class PersistentStoreConfiguration implements Serializable {
115109
private int walSegmentSize = DFLT_WAL_SEGMENT_SIZE;
116110

117111
/** Directory where WAL is stored (work directory) */
118-
private String walStorePath = DFLT_WAL_STORE_PATH;
112+
private String walStorePath = DataStorageConfiguration.DFLT_WAL_PATH;
119113

120114
/** WAL archive path. */
121-
private String walArchivePath = DFLT_WAL_ARCHIVE_PATH;
115+
private String walArchivePath = DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH;
122116

123117
/** Metrics enabled flag. */
124118
private boolean metricsEnabled = DFLT_METRICS_ENABLED;

‎modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java‎

Lines changed: 13 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import java.io.Serializable;
2323
import java.nio.file.Files;
2424
import java.nio.file.Path;
25-
import java.nio.file.Paths;
2625
import java.util.Arrays;
2726
import java.util.Comparator;
2827
import java.util.HashSet;
@@ -226,12 +225,6 @@ public class CdcMain implements Runnable {
226225
/** Logger. */
227226
private final IgniteLogger log;
228227

229-
/** Change Data Capture directory. */
230-
private Path cdcDir;
231-
232-
/** Database directory. */
233-
private File dbDir;
234-
235228
/** Ignite folders. */
236229
private NodeFileTree ft;
237230

@@ -325,10 +318,10 @@ public void runX() throws Exception {
325318
}
326319

327320
try (CdcFileLockHolder lock = lockPds()) {
328-
Files.createDirectories(cdcDir.resolve(STATE_DIR));
321+
Files.createDirectories(ft.walCdc().toPath().resolve(STATE_DIR));
329322

330323
if (log.isInfoEnabled()) {
331-
log.info("Change Data Capture [dir=" + cdcDir + ']');
324+
log.info("Change Data Capture [dir=" + ft.walCdc() + ']');
332325
log.info("Ignite node Binary meta [dir=" + ft.binaryMeta() + ']');
333326
log.info("Ignite node Marshaller [dir=" + ft.marshaller() + ']');
334327
}
@@ -340,7 +333,7 @@ public void runX() throws Exception {
340333
try {
341334
kctx.resource().injectGeneric(consumer.consumer());
342335

343-
state = createState(cdcDir.resolve(STATE_DIR));
336+
state = createState(ft.walCdc().toPath().resolve(STATE_DIR));
344337

345338
walState = state.loadWalState();
346339
typesState = state.loadTypesState();
@@ -379,7 +372,6 @@ protected CdcConsumerState createState(Path stateDir) {
379372
}
380373

381374
/**
382-
* @return Kernal instance.
383375
* @throws IgniteCheckedException If failed.
384376
*/
385377
private void startStandaloneKernal() throws IgniteCheckedException {
@@ -424,7 +416,7 @@ private void startStandaloneKernal() throws IgniteCheckedException {
424416
private void initMetrics() {
425417
mreg.objectMetric(BINARY_META_DIR, String.class, "Binary meta directory").value(ft.binaryMeta().getAbsolutePath());
426418
mreg.objectMetric(MARSHALLER_DIR, String.class, "Marshaller directory").value(ft.marshaller().getAbsolutePath());
427-
mreg.objectMetric(CDC_DIR, String.class, "CDC directory").value(cdcDir.toFile().getAbsolutePath());
419+
mreg.objectMetric(CDC_DIR, String.class, "CDC directory").value(ft.walCdc().getAbsolutePath());
428420

429421
curSegmentIdx = mreg.longMetric(CUR_SEG_IDX, "Current segment index");
430422
committedSegmentIdx = mreg.longMetric(COMMITTED_SEG_IDX, "Committed segment index");
@@ -489,7 +481,7 @@ public void consumeWalSegmentsUntilStopped() {
489481
return;
490482
}
491483

492-
try (Stream<Path> cdcFiles = Files.list(cdcDir)) {
484+
try (Stream<Path> cdcFiles = Files.list(ft.walCdc().toPath())) {
493485
Set<Path> exists = new HashSet<>();
494486

495487
Iterator<Path> segments = cdcFiles
@@ -757,14 +749,14 @@ private void updateMappings() {
757749
/** Search for new or changed {@link CdcCacheEvent} and notifies the consumer. */
758750
private void updateCaches() {
759751
try {
760-
if (!dbDir.exists())
752+
if (!ft.nodeStorage().exists())
761753
return;
762754

763755
Set<Integer> destroyed = new HashSet<>(cachesState.keySet());
764756

765757
Iterator<CdcCacheEvent> cacheEvts = GridLocalConfigManager
766758
.readCachesData(
767-
dbDir,
759+
ft.nodeStorage(),
768760
kctx.marshallerContext().jdkMarshaller(),
769761
igniteCfg)
770762
.entrySet().stream()
@@ -887,35 +879,18 @@ private CdcFileLockHolder tryLock(File dbStoreDirWithSubdirectory) {
887879
return null;
888880
}
889881

890-
File cdcRoot = new File(igniteCfg.getDataStorageConfiguration().getCdcWalPath());
891-
892-
if (!cdcRoot.isAbsolute()) {
893-
cdcRoot = new File(
894-
igniteCfg.getWorkDirectory(),
895-
igniteCfg.getDataStorageConfiguration().getCdcWalPath()
896-
);
897-
}
898-
899-
if (!cdcRoot.exists()) {
900-
log.warning("CDC root directory not exists. Should be created by Ignite Node. " +
901-
"Is Change Data Capture enabled in IgniteConfiguration? [dir=" + cdcRoot + ']');
882+
String folderName = dbStoreDirWithSubdirectory.getName();
902883

903-
return null;
904-
}
884+
ft = new NodeFileTree(igniteCfg, folderName);
905885

906-
Path cdcDir = Paths.get(cdcRoot.getAbsolutePath(), dbStoreDirWithSubdirectory.getName());
907-
908-
if (!Files.exists(cdcDir)) {
886+
if (!ft.walCdc().exists()) {
909887
log.warning("CDC directory not exists. Should be created by Ignite Node. " +
910-
"Is Change Data Capture enabled in IgniteConfiguration? [dir=" + cdcDir + ']');
888+
"Is Change Data Capture enabled in IgniteConfiguration? [dir=" + ft.walCdc() + ']');
911889

912890
return null;
913891
}
914892

915-
this.cdcDir = cdcDir;
916-
this.dbDir = dbStoreDirWithSubdirectory;
917-
918-
CdcFileLockHolder lock = new CdcFileLockHolder(cdcDir.toString(), "cdc.lock", log);
893+
CdcFileLockHolder lock = new CdcFileLockHolder(ft.walCdc().toString(), "cdc.lock", log);
919894

920895
try {
921896
lock.tryLock(cdcCfg.getLockTimeout());
@@ -926,7 +901,7 @@ private CdcFileLockHolder tryLock(File dbStoreDirWithSubdirectory) {
926901
U.closeQuiet(lock);
927902

928903
if (log.isInfoEnabled()) {
929-
log.info("Unable to acquire lock to lock CDC folder [dir=" + cdcRoot + "]" + NL +
904+
log.info("Unable to acquire lock to lock CDC folder [dir=" + ft.walCdc() + "]" + NL +
930905
"Reason: " + e.getMessage());
931906
}
932907

‎modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcUtilityActiveCdcManager.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
2626
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
2727
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.PartitionsExchangeAware;
28-
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
2928
import org.apache.ignite.internal.util.typedef.internal.U;
3029

3130
import static org.apache.ignite.internal.cdc.CdcConsumerState.CDC_MODE_FILE_NAME;
@@ -45,7 +44,7 @@ public class CdcUtilityActiveCdcManager extends GridCacheSharedManagerAdapter im
4544
if (fut.localJoinExchange() || fut.activateCluster()) {
4645
try {
4746
File cdcModeFile = Paths.get(
48-
((FileWriteAheadLogManager)cctx.wal(true)).walCdcDirectory().getAbsolutePath(),
47+
cctx.kernalContext().pdsFolderResolver().fileTree().walCdc().getAbsolutePath(),
4948
STATE_DIR,
5049
CDC_MODE_FILE_NAME).toFile();
5150

‎modules/core/src/main/java/org/apache/ignite/internal/management/cdc/CdcDeleteLostSegmentsTask.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.ignite.internal.cdc.CdcFileLockHolder;
3535
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
3636
import org.apache.ignite.internal.processors.task.GridInternal;
37+
import org.apache.ignite.internal.util.typedef.internal.CU;
3738
import org.apache.ignite.internal.util.typedef.internal.U;
3839
import org.apache.ignite.internal.visor.VisorJob;
3940
import org.apache.ignite.internal.visor.VisorMultiNodeTask;
@@ -90,13 +91,11 @@ protected CdcDeleteLostSegmentsJob(CdcDeleteLostSegmentLinksCommandArg arg, bool
9091

9192
/** {@inheritDoc} */
9293
@Override protected Void run(CdcDeleteLostSegmentLinksCommandArg arg) throws IgniteException {
93-
FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ignite.context().cache().context().wal(true);
94-
95-
File walCdcDir = wal.walCdcDirectory();
96-
97-
if (walCdcDir == null)
94+
if (!CU.isCdcEnabled(ignite.configuration()))
9895
throw new IgniteException("CDC is not configured.");
9996

97+
File walCdcDir = ignite.context().pdsFolderResolver().fileTree().walCdc();
98+
10099
CdcFileLockHolder lock = new CdcFileLockHolder(walCdcDir.getAbsolutePath(), "Delete lost segments job", log);
101100

102101
try {

‎modules/core/src/main/java/org/apache/ignite/internal/management/wal/WalTask.java‎

Lines changed: 5 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,10 @@
3434
import org.apache.ignite.IgniteException;
3535
import org.apache.ignite.IgniteLogger;
3636
import org.apache.ignite.compute.ComputeJobResult;
37-
import org.apache.ignite.configuration.DataStorageConfiguration;
38-
import org.apache.ignite.configuration.IgniteConfiguration;
3937
import org.apache.ignite.internal.GridKernalContext;
4038
import org.apache.ignite.internal.management.wal.WalPrintCommand.WalPrintCommandArg;
4139
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
42-
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
40+
import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree;
4341
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
4442
import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
4543
import org.apache.ignite.internal.processors.task.GridInternal;
@@ -274,32 +272,12 @@ private int resolveMaxReservedIndex(FileWriteAheadLogManager wal, WALPointer low
274272
* @throws IgniteCheckedException if failed.
275273
*/
276274
private File getWalArchiveDir() throws IgniteCheckedException {
277-
IgniteConfiguration igCfg = ignite.context().config();
275+
NodeFileTree ft = ignite.context().pdsFolderResolver().fileTree();
278276

279-
DataStorageConfiguration dsCfg = igCfg.getDataStorageConfiguration();
277+
if (!ft.walArchive().exists())
278+
throw new IgniteCheckedException("WAL archive directory does not exists" + ft.walArchive().getAbsolutePath());
280279

281-
PdsFolderSettings resFldrs = ignite.context().pdsFolderResolver().resolveFolders();
282-
283-
String consId = resFldrs.folderName();
284-
285-
File dir;
286-
287-
if (dsCfg.getWalArchivePath() != null) {
288-
File workDir0 = new File(dsCfg.getWalArchivePath());
289-
290-
dir = workDir0.isAbsolute() ?
291-
new File(workDir0, consId) :
292-
new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), dsCfg.getWalArchivePath(), false),
293-
consId);
294-
}
295-
else
296-
dir = new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(),
297-
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), consId);
298-
299-
if (!dir.exists())
300-
throw new IgniteCheckedException("WAL archive directory does not exists" + dir.getAbsolutePath());
301-
302-
return dir;
280+
return ft.walArchive();
303281
}
304282

305283
/**

‎modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -235,13 +235,6 @@ public WALIterator replay(
235235
*/
236236
void startAutoReleaseSegments();
237237

238-
/**
239-
* Archive directory if any.
240-
*
241-
* @return Archive directory.
242-
*/
243-
@Nullable File archiveDir();
244-
245238
/**
246239
* @param idx Segment index.
247240
* @return Compressed archive segment.

0 commit comments

Comments
 (0)