Skip to content

Commit 607b587

Browse files
nizhikovnizhikov
authored
IGNITE-24130 Move binaryMetadata, marshaller directories logic to NodeFileTree (#11840)
1 parent 29d21b2 commit 607b587

File tree

48 files changed

+794
-410
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+794
-410
lines changed

‎modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,14 @@
3131
import org.apache.ignite.configuration.DataStorageConfiguration;
3232
import org.apache.ignite.configuration.IgniteConfiguration;
3333
import org.apache.ignite.configuration.WALMode;
34+
import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree;
3435
import org.apache.ignite.internal.util.typedef.F;
3536
import org.apache.ignite.internal.util.typedef.G;
3637
import org.apache.ignite.internal.util.typedef.internal.U;
3738
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
3839
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
3940
import org.openjdk.jol.info.GraphLayout;
4041

41-
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
42-
4342
/**
4443
*
4544
*/
@@ -68,10 +67,12 @@ private void cleanPersistenceDir() throws Exception {
6867
if (!F.isEmpty(G.allGrids()))
6968
throw new IgniteException("Grids are not stopped");
7069

70+
SharedFileTree sft = new SharedFileTree(U.defaultWorkDirectory());
71+
7172
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false));
72-
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
73-
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false));
74-
U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", false));
73+
U.delete(sft.db());
74+
U.delete(sft.marshaller());
75+
U.delete(sft.binaryMetaRoot());
7576
}
7677

7778
/** */

‎modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.apache.ignite.cache.QueryEntity;
3030
import org.apache.ignite.cache.query.annotations.QuerySqlField;
3131
import org.apache.ignite.configuration.CacheConfiguration;
32-
import org.apache.ignite.configuration.DataStorageConfiguration;
3332
import org.apache.ignite.internal.IgniteEx;
3433
import org.apache.ignite.internal.util.typedef.F;
3534
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -144,16 +143,7 @@ protected String getCfgUrl() {
144143
assertTrue(conn.isClosed());
145144
}
146145

147-
cleanUpWorkingDir();
148-
}
149-
150-
/**
151-
* Clean up working directory.
152-
*/
153-
private void cleanUpWorkingDir() throws Exception {
154-
String workDir = U.defaultWorkDirectory();
155-
156-
U.delete(U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH, false));
146+
U.delete(sharedFileTree().marshaller());
157147
}
158148

159149
/**

‎modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/FoldersReuseCompatibilityTest.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@
2626
import org.apache.ignite.IgniteCache;
2727
import org.apache.ignite.IgniteCheckedException;
2828
import org.apache.ignite.compatibility.testframework.junits.SkipTestIfIsJdkNewer;
29-
import org.apache.ignite.configuration.DataStorageConfiguration;
3029
import org.apache.ignite.configuration.IgniteConfiguration;
3130
import org.apache.ignite.configuration.MemoryConfiguration;
3231
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
3332
import org.apache.ignite.configuration.PersistentStoreConfiguration;
3433
import org.apache.ignite.internal.IgniteEx;
3534
import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
35+
import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree;
3636
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver;
3737
import org.apache.ignite.internal.util.typedef.internal.U;
3838
import org.apache.ignite.lang.IgniteInClosure;
@@ -222,7 +222,10 @@ private void assertNodeIndexesInFolder(Integer... indexes) throws IgniteCheckedE
222222
* @throws IgniteCheckedException if IO error occur
223223
*/
224224
private void assertPdsDirsDefaultExist(String subDirName) throws IgniteCheckedException {
225-
assertDirectoryExist(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, subDirName);
225+
NodeFileTree ft = nodeFileTree(subDirName);
226+
227+
assertTrue(ft.binaryMeta().exists() && ft.binaryMeta().isDirectory());
228+
226229
assertDirectoryExist(PersistentStoreConfiguration.DFLT_WAL_STORE_PATH, subDirName);
227230
assertDirectoryExist(PersistentStoreConfiguration.DFLT_WAL_ARCHIVE_PATH, subDirName);
228231
assertDirectoryExist(PdsFolderResolver.DB_DEFAULT_FOLDER, subDirName);

‎modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePersistenceCompatibilityAbstractTest.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,16 @@
2525
import org.apache.ignite.internal.util.typedef.internal.U;
2626

2727
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
28+
import static org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree.BINARY_METADATA_DIR;
29+
import static org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree.MARSHALLER_DIR;
2830

2931
/**
3032
* Super class for all persistence compatibility tests.
3133
*/
3234
public abstract class IgnitePersistenceCompatibilityAbstractTest extends IgniteCompatibilityAbstractTest {
3335
/** Persistence directories. */
34-
private static final List<String> PERSISTENCE_DIRS = Arrays.asList(DFLT_STORE_DIR, "binary_meta", "cp", "marshaller");
36+
private static final List<String> PERSISTENCE_DIRS
37+
= Arrays.asList(DFLT_STORE_DIR, BINARY_METADATA_DIR, "cp", MARSHALLER_DIR);
3538

3639
/** {@inheritDoc} */
3740
@Override protected void beforeTest() throws Exception {

‎modules/compress/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCompressionBasicTest.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
import org.apache.ignite.cache.QueryIndex;
4545
import org.apache.ignite.cluster.ClusterState;
4646
import org.apache.ignite.configuration.CacheConfiguration;
47-
import org.apache.ignite.configuration.DataStorageConfiguration;
4847
import org.apache.ignite.configuration.DiskPageCompression;
4948
import org.apache.ignite.configuration.IgniteConfiguration;
5049
import org.apache.ignite.internal.GridKernalContextImpl;
@@ -292,8 +291,8 @@ public void testRestoreNotCompressed_OnGridWithoutCompression() throws Exception
292291

293292
U.delete(U.resolveWorkDirectory(dir.toString(), "cp", false));
294293
U.delete(U.resolveWorkDirectory(dir.toString(), DFLT_STORE_DIR, false));
295-
U.delete(U.resolveWorkDirectory(dir.toString(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false));
296-
U.delete(U.resolveWorkDirectory(dir.toString(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false));
294+
U.delete(nodeFileTree(dir.toString()).marshaller());
295+
U.delete(nodeFileTree(dir.toString()).binaryMetaRoot());
297296
}
298297
}
299298
catch (IOException e) {

‎modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -167,12 +167,6 @@ public class DataStorageConfiguration implements Serializable {
167167
/** Default change data capture directory maximum size. */
168168
public static final long DFLT_CDC_WAL_DIRECTORY_MAX_SIZE = 0;
169169

170-
/** Default path (relative to working directory) of binary metadata folder */
171-
public static final String DFLT_BINARY_METADATA_PATH = "db/binary_meta";
172-
173-
/** Default path (relative to working directory) of marshaller mappings folder */
174-
public static final String DFLT_MARSHALLER_PATH = "db/marshaller";
175-
176170
/** Default write throttling enabled. */
177171
public static final boolean DFLT_WRITE_THROTTLING_ENABLED = false;
178172

‎modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@
4646
import org.apache.ignite.spi.encryption.EncryptionSpi;
4747

4848
import static java.util.concurrent.TimeUnit.MILLISECONDS;
49-
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH;
5049
import static org.apache.ignite.internal.IgniteKernal.NL;
5150
import static org.apache.ignite.internal.IgniteKernal.SITE;
5251
import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
@@ -87,7 +86,7 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) {
8786
cnsmr.start();
8887

8988
try {
90-
File[] files = new File(cfg.dumpRoot(), DFLT_MARSHALLER_PATH).listFiles(BinaryUtils::notTmpFile);
89+
File[] files = F.first(dump.fileTrees()).marshaller().listFiles(BinaryUtils::notTmpFile);
9190

9291
if (files != null)
9392
cnsmr.onMappings(CdcMain.typeMappingIterator(files, tm -> true));

‎modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java‎

Lines changed: 10 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,9 @@
3737
import org.apache.ignite.IgniteCheckedException;
3838
import org.apache.ignite.IgniteException;
3939
import org.apache.ignite.IgniteLogger;
40-
import org.apache.ignite.configuration.DataStorageConfiguration;
41-
import org.apache.ignite.configuration.IgniteConfiguration;
4240
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
4341
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
42+
import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree;
4443
import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
4544
import org.apache.ignite.internal.processors.marshaller.MappedName;
4645
import org.apache.ignite.internal.processors.marshaller.MappingExchangeResult;
@@ -192,8 +191,10 @@ public void onMappingDataReceived(IgniteLogger log, List<Map<Integer, MappedName
192191
*/
193192
public static void saveMappings(GridKernalContext ctx, List<Map<Integer, MappedName>> mappings, File dir)
194193
throws IgniteCheckedException {
195-
MarshallerMappingFileStore writer = new MarshallerMappingFileStore(ctx,
196-
resolveMappingFileStoreWorkDir(dir.getAbsolutePath()));
194+
MarshallerMappingFileStore writer = new MarshallerMappingFileStore(
195+
ctx,
196+
new SharedFileTree(dir).mkdirMarshaller()
197+
);
197198

198199
addPlatformMappings(ctx.log(MarshallerContextImpl.class),
199200
mappings,
@@ -583,17 +584,14 @@ private static void putAtIndex(
583584
* @param transport Transport.
584585
*/
585586
public void onMarshallerProcessorStarted(
586-
GridKernalContext ctx,
587-
MarshallerMappingTransport transport
587+
GridKernalContext ctx,
588+
MarshallerMappingTransport transport
588589
) throws IgniteCheckedException {
589590
assert ctx != null;
590591

591-
IgniteConfiguration cfg = ctx.config();
592-
String workDir = U.workDirectory(cfg.getWorkDirectory(), cfg.getIgniteHome());
593-
594-
fileStore = marshallerMappingFileStoreDir == null ?
595-
new MarshallerMappingFileStore(ctx, resolveMappingFileStoreWorkDir(workDir)) :
596-
new MarshallerMappingFileStore(ctx, marshallerMappingFileStoreDir);
592+
fileStore = marshallerMappingFileStoreDir == null
593+
? new MarshallerMappingFileStore(ctx, ctx.pdsFolderResolver().fileTree().mkdirMarshaller())
594+
: new MarshallerMappingFileStore(ctx, marshallerMappingFileStoreDir);
597595
this.transport = transport;
598596
closProc = ctx.closure();
599597
clientNode = ctx.clientNode();
@@ -604,30 +602,6 @@ public void onMarshallerProcessorStarted(
604602
MarshallerUtils.setNodeName(jdkMarsh, ctx.igniteInstanceName());
605603
}
606604

607-
/**
608-
* @param igniteWorkDir Base ignite working directory.
609-
* @return Resolved directory.
610-
*/
611-
public static File resolveMappingFileStoreWorkDir(String igniteWorkDir) {
612-
File dir = mappingFileStoreWorkDir(igniteWorkDir);
613-
614-
if (!U.mkdirs(dir))
615-
throw new IgniteException("Could not create directory for marshaller mappings: " + dir);
616-
617-
return dir;
618-
}
619-
620-
/**
621-
* @param igniteWorkDir Base ignite working directory.
622-
* @return Work directory for marshaller mappings.
623-
*/
624-
public static File mappingFileStoreWorkDir(String igniteWorkDir) {
625-
if (F.isEmpty(igniteWorkDir))
626-
throw new IgniteException("Work directory has not been set: " + igniteWorkDir);
627-
628-
return new File(igniteWorkDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH);
629-
}
630-
631605
/**
632606
*
633607
*/

‎modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.ignite.IgniteCheckedException;
3030
import org.apache.ignite.IgniteLogger;
3131
import org.apache.ignite.internal.binary.BinaryUtils;
32+
import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree;
3233
import org.apache.ignite.internal.util.GridStripedLock;
3334
import org.apache.ignite.internal.util.IgniteUtils;
3435
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -223,7 +224,7 @@ private void fixLegacyFolder() throws IgniteCheckedException {
223224

224225
File legacyDir = new File(
225226
ctx.config().getWorkDirectory(),
226-
"marshaller"
227+
SharedFileTree.MARSHALLER_DIR
227228
);
228229

229230
File legacyTmpDir = new File(legacyDir + TMP_SUFFIX);

‎modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java‎

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,14 @@
4747
import org.apache.ignite.configuration.IgniteConfiguration;
4848
import org.apache.ignite.internal.GridLoggerProxy;
4949
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
50-
import org.apache.ignite.internal.MarshallerContextImpl;
5150
import org.apache.ignite.internal.binary.BinaryUtils;
5251
import org.apache.ignite.internal.cdc.WalRecordsConsumer.DataEntryIterator;
5352
import org.apache.ignite.internal.pagemem.wal.WALIterator;
5453
import org.apache.ignite.internal.pagemem.wal.record.CdcManagerRecord;
5554
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
5655
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
5756
import org.apache.ignite.internal.processors.cache.GridLocalConfigManager;
58-
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
57+
import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree;
5958
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver;
6059
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
6160
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
@@ -233,11 +232,8 @@ public class CdcMain implements Runnable {
233232
/** Database directory. */
234233
private File dbDir;
235234

236-
/** Binary meta directory. */
237-
private File binaryMeta;
238-
239-
/** Marshaller directory. */
240-
private File marshaller;
235+
/** Ignite folders. */
236+
private NodeFileTree ft;
241237

242238
/** Standalone kernal context. */
243239
private StandaloneGridKernalContext kctx;
@@ -329,21 +325,15 @@ public void runX() throws Exception {
329325
}
330326

331327
try (CdcFileLockHolder lock = lockPds()) {
332-
String consIdDir = cdcDir.getName(cdcDir.getNameCount() - 1).toString();
333-
334328
Files.createDirectories(cdcDir.resolve(STATE_DIR));
335329

336-
binaryMeta = CacheObjectBinaryProcessorImpl.binaryWorkDir(igniteCfg.getWorkDirectory(), consIdDir);
337-
338-
marshaller = MarshallerContextImpl.mappingFileStoreWorkDir(igniteCfg.getWorkDirectory());
339-
340330
if (log.isInfoEnabled()) {
341331
log.info("Change Data Capture [dir=" + cdcDir + ']');
342-
log.info("Ignite node Binary meta [dir=" + binaryMeta + ']');
343-
log.info("Ignite node Marshaller [dir=" + marshaller + ']');
332+
log.info("Ignite node Binary meta [dir=" + ft.binaryMeta() + ']');
333+
log.info("Ignite node Marshaller [dir=" + ft.marshaller() + ']');
344334
}
345335

346-
kctx = startStandaloneKernal();
336+
startStandaloneKernal();
347337

348338
initMetrics();
349339

@@ -392,8 +382,8 @@ protected CdcConsumerState createState(Path stateDir) {
392382
* @return Kernal instance.
393383
* @throws IgniteCheckedException If failed.
394384
*/
395-
private StandaloneGridKernalContext startStandaloneKernal() throws IgniteCheckedException {
396-
StandaloneGridKernalContext kctx = new StandaloneGridKernalContext(log, binaryMeta, marshaller) {
385+
private void startStandaloneKernal() throws IgniteCheckedException {
386+
kctx = new StandaloneGridKernalContext(log, ft.binaryMeta(), ft.marshaller()) {
397387
@Override protected IgniteConfiguration prepareIgniteConfiguration() {
398388
IgniteConfiguration cfg = super.prepareIgniteConfiguration();
399389

@@ -428,14 +418,12 @@ private StandaloneGridKernalContext startStandaloneKernal() throws IgniteChecked
428418
}
429419

430420
mreg = kctx.metric().registry("cdc");
431-
432-
return kctx;
433421
}
434422

435423
/** Initialize metrics. */
436424
private void initMetrics() {
437-
mreg.objectMetric(BINARY_META_DIR, String.class, "Binary meta directory").value(binaryMeta.getAbsolutePath());
438-
mreg.objectMetric(MARSHALLER_DIR, String.class, "Marshaller directory").value(marshaller.getAbsolutePath());
425+
mreg.objectMetric(BINARY_META_DIR, String.class, "Binary meta directory").value(ft.binaryMeta().getAbsolutePath());
426+
mreg.objectMetric(MARSHALLER_DIR, String.class, "Marshaller directory").value(ft.marshaller().getAbsolutePath());
439427
mreg.objectMetric(CDC_DIR, String.class, "CDC directory").value(cdcDir.toFile().getAbsolutePath());
440428

441429
curSegmentIdx = mreg.longMetric(CUR_SEG_IDX, "Current segment index");
@@ -468,6 +456,8 @@ private CdcFileLockHolder lockPds() throws IgniteCheckedException {
468456
"[workDir=" + igniteCfg.getWorkDirectory() + ", consistentId=" + igniteCfg.getConsistentId() + ']');
469457
}
470458

459+
ft = new NodeFileTree(igniteCfg, settings.folderName());
460+
471461
CdcFileLockHolder lock = settings.getLockedFileLockHolder();
472462

473463
if (lock == null) {
@@ -567,8 +557,8 @@ private boolean consumeSegment(Path segment) {
567557
IgniteWalIteratorFactory.IteratorParametersBuilder builder =
568558
new IgniteWalIteratorFactory.IteratorParametersBuilder()
569559
.log(log)
570-
.binaryMetadataFileStoreDir(binaryMeta)
571-
.marshallerMappingFileStoreDir(marshaller)
560+
.binaryMetadataFileStoreDir(ft.binaryMeta())
561+
.marshallerMappingFileStoreDir(ft.marshaller())
572562
.igniteConfigurationModifier((cfg) -> cfg.setPluginProviders(igniteCfg.getPluginProviders()))
573563
.keepBinary(cdcCfg.isKeepBinary())
574564
.filesOrDirs(segment.toFile());
@@ -692,7 +682,7 @@ private void updateMetadata() {
692682
/** Search for new or changed {@link BinaryType} and notifies the consumer. */
693683
private void updateTypes() {
694684
try {
695-
File[] files = binaryMeta.listFiles();
685+
File[] files = ft.binaryMeta().listFiles();
696686

697687
if (files == null)
698688
return;
@@ -710,7 +700,7 @@ private void updateTypes() {
710700
typesState.put(typeId, lastModified);
711701

712702
try {
713-
kctx.cacheObjects().cacheMetadataLocally(binaryMeta, typeId);
703+
kctx.cacheObjects().cacheMetadataLocally(ft.binaryMeta(), typeId);
714704
}
715705
catch (IgniteCheckedException e) {
716706
throw new IgniteException(e);
@@ -739,7 +729,7 @@ private void updateTypes() {
739729
/** Search for new or changed {@link TypeMapping} and notifies the consumer. */
740730
private void updateMappings() {
741731
try {
742-
File[] files = marshaller.listFiles(BinaryUtils::notTmpFile);
732+
File[] files = ft.marshaller().listFiles(BinaryUtils::notTmpFile);
743733

744734
if (files == null)
745735
return;

0 commit comments

Comments
 (0)