Skip to content

Commit 8a3c923

Browse files
adamretteradamretter
committed
[feature] Introduce version number to the Journal file, to indicate format of log data
1 parent 439c5b8 commit 8a3c923

File tree

3 files changed

+99
-10
lines changed

3 files changed

+99
-10
lines changed

‎src/org/exist/storage/journal/Journal.java‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.exist.storage.BrokerPool;
4040
import org.exist.storage.lock.FileLock;
4141
import org.exist.storage.txn.Checkpoint;
42+
import org.exist.util.ByteConversion;
4243
import org.exist.util.FileUtils;
4344
import org.exist.util.ReadOnlyException;
4445
import org.exist.util.sanity.SanityCheck;
@@ -80,6 +81,15 @@ public final class Journal {
8081
*/
8182
private static final Logger LOG = LogManager.getLogger(Journal.class);
8283

84+
/**
85+
* The length in bytes of the Header in the Journal file
86+
*
87+
* 4 bytes for the magic number, and then 2 bytes for the journal version
88+
*/
89+
public static final int JOURNAL_HEADER_LEN = 6;
90+
public static final byte[] JOURNAL_MAGIC_NUMBER = {0x0E, 0x0D, 0x0B, 0x01};
91+
public static final short JOURNAL_VERSION = 2;
92+
8393
public static final String RECOVERY_SYNC_ON_COMMIT_ATTRIBUTE = "sync-on-commit";
8494
public static final String RECOVERY_JOURNAL_DIR_ATTRIBUTE = "journal-dir";
8595
public static final String RECOVERY_SIZE_LIMIT_ATTRIBUTE = "size";
@@ -205,6 +215,8 @@ public final class Journal {
205215

206216
private final Path fsJournalDir;
207217

218+
private volatile boolean initialised = false;
219+
208220
public Journal(final BrokerPool pool, final Path directory) throws EXistException {
209221
this.pool = pool;
210222
this.fsJournalDir = directory.resolve("fs.journal");
@@ -456,13 +468,13 @@ public void switchFiles() throws LogException {
456468
final Path file = dir.resolve(fname);
457469
if (Files.exists(file)) {
458470
if (LOG.isDebugEnabled()) {
459-
LOG.debug("Journal file " + file.toAbsolutePath() + " already exists. Copying it.");
471+
LOG.debug("Journal file " + file.toAbsolutePath() + " already exists. Moving it to a backup file.");
460472
}
461473

462474
try {
463475
final Path renamed = Files.move(file, file.resolveSibling(FileUtils.fileName(file) + BAK_FILE_SUFFIX), StandardCopyOption.ATOMIC_MOVE);
464476
if (LOG.isDebugEnabled()) {
465-
LOG.debug("Old file renamed from '" + file.toAbsolutePath().toString() + "' to '" + renamed.toAbsolutePath().toString() + "'");
477+
LOG.debug("Old Journal file renamed from '" + file.toAbsolutePath().toString() + "' to '" + renamed.toAbsolutePath().toString() + "'");
466478
}
467479
} catch (final IOException ioe) {
468480
LOG.warn(ioe); //TODO(AR) should probably be an LogException but wasn't previously!
@@ -477,13 +489,30 @@ public void switchFiles() throws LogException {
477489
close();
478490
try {
479491
channel = Files.newByteChannel(file, CREATE_NEW, WRITE);
492+
writeJournalHeader(channel);
480493
fileSyncRunnable.setChannel((FileChannel) channel);
494+
initialised = true;
481495
} catch (final IOException e) {
482496
throw new LogException("Failed to open new journal: " + file.toAbsolutePath().toString(), e);
483497
}
484498
}
485499
}
486500

501+
private void writeJournalHeader(final SeekableByteChannel channel) throws IOException {
502+
final ByteBuffer buf = ByteBuffer.allocate(JOURNAL_HEADER_LEN);
503+
504+
// write the magic number
505+
buf.put(JOURNAL_MAGIC_NUMBER);
506+
507+
// write the version of the journal format
508+
final byte[] journalVersion = new byte[2];
509+
ByteConversion.shortToByteH(JOURNAL_VERSION, journalVersion, 0);
510+
buf.put(journalVersion);
511+
512+
buf.flip();
513+
channel.write(buf);
514+
}
515+
487516
/**
488517
* Close the journal.
489518
*/
@@ -552,6 +581,11 @@ public Path getFile(final int fileNum) {
552581
* @param checkpoint true if a checkpoint should be written before shitdown
553582
*/
554583
public void shutdown(final long txnId, final boolean checkpoint) {
584+
if (!initialised) {
585+
// no journal is initialized
586+
return;
587+
}
588+
555589
if (currentBuffer == null) {
556590
return; // the db has probably shut down already
557591
}

‎src/org/exist/storage/journal/JournalReader.java‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.logging.log4j.LogManager;
2323
import org.apache.logging.log4j.Logger;
2424
import org.exist.storage.DBBroker;
25+
import org.exist.util.ByteConversion;
2526

2627
import javax.annotation.Nullable;
2728
import java.io.IOException;
@@ -31,9 +32,7 @@
3132
import java.nio.file.Path;
3233

3334
import static java.nio.file.StandardOpenOption.READ;
34-
import static org.exist.storage.journal.Journal.LOG_ENTRY_BACK_LINK_LEN;
35-
import static org.exist.storage.journal.Journal.LOG_ENTRY_BASE_LEN;
36-
import static org.exist.storage.journal.Journal.LOG_ENTRY_HEADER_LEN;
35+
import static org.exist.storage.journal.Journal.*;
3736

3837
/**
3938
* Read log entries from the journal file. This class is used during recovery to scan the
@@ -66,12 +65,40 @@ public JournalReader(final DBBroker broker, final Path file, final int fileNumbe
6665
this.fileNumber = fileNumber;
6766
try {
6867
this.fc = Files.newByteChannel(file, READ);
68+
validateJournalHeader(file, fc);
6969
} catch (final IOException e) {
7070
close();
7171
throw new LogException("Failed to read journal file " + file.toAbsolutePath().toString(), e);
7272
}
7373
}
7474

75+
private void validateJournalHeader(final Path file, final SeekableByteChannel fc) throws IOException, LogException {
76+
// read the magic number
77+
final ByteBuffer buf = ByteBuffer.allocate(JOURNAL_HEADER_LEN);
78+
fc.read(buf);
79+
buf.flip();
80+
81+
// check the magic number
82+
final boolean validMagic =
83+
buf.get() == JOURNAL_MAGIC_NUMBER[0]
84+
&& buf.get() == JOURNAL_MAGIC_NUMBER[1]
85+
&& buf.get() == JOURNAL_MAGIC_NUMBER[2]
86+
&& buf.get() == JOURNAL_MAGIC_NUMBER[3];
87+
88+
if (!validMagic) {
89+
throw new LogException("File was not recognised as a valid eXist-db journal file: " + file.toAbsolutePath().toString());
90+
}
91+
92+
// check the version of the journal format
93+
final short storedVersion = ByteConversion.byteToShortH(new byte[] {buf.get(), buf.get()}, 0);
94+
final boolean validVersion =
95+
storedVersion == JOURNAL_VERSION;
96+
97+
if (!validVersion) {
98+
throw new LogException("Journal file was version " + storedVersion + ", but required version " + JOURNAL_VERSION + ": " + file.toAbsolutePath().toString());
99+
}
100+
}
101+
75102
/**
76103
* Returns the next entry found from the current position.
77104
*
@@ -105,8 +132,8 @@ Loggable previousEntry() throws LogException {
105132
try {
106133
checkOpen();
107134

108-
// are we at the start of the journal?
109-
if (fc.position() == 0) {
135+
// is there a previous entry to read?
136+
if (fc.position() < JOURNAL_HEADER_LEN + LOG_ENTRY_BASE_LEN) {
110137
return null;
111138
}
112139

@@ -143,7 +170,7 @@ Loggable previousEntry() throws LogException {
143170
Loggable lastEntry() throws LogException {
144171
try {
145172
checkOpen();
146-
fc.position(fc.size());
173+
positionLast();
147174
return previousEntry();
148175
} catch (final IOException e) {
149176
throw new LogException("Fatal error while reading last journal entry: " + e.getMessage(), e);
@@ -230,6 +257,34 @@ public void position(final long lsn) throws LogException {
230257
}
231258
}
232259

260+
/**
261+
* Re-position the file position so it points to the first entry.
262+
*
263+
* @throws LogException if the journal file cannot be re-positioned
264+
*/
265+
public void positionFirst() throws LogException {
266+
try {
267+
checkOpen();
268+
fc.position(JOURNAL_HEADER_LEN);
269+
} catch (final IOException e) {
270+
throw new LogException("Fatal error while seeking first journal entry: " + e.getMessage(), e);
271+
}
272+
}
273+
274+
/**
275+
* Re-position the file position so it points to the last entry.
276+
*
277+
* @throws LogException if the journal file cannot be re-positioned
278+
*/
279+
public void positionLast() throws LogException {
280+
try {
281+
checkOpen();
282+
fc.position(fc.size());
283+
} catch (final IOException e) {
284+
throw new LogException("Fatal error while seeking last journal entry: " + e.getMessage(), e);
285+
}
286+
}
287+
233288
private void checkOpen() throws IOException {
234289
if (fc == null) {
235290
throw new IOException("Journal file is closed");

‎src/org/exist/storage/recovery/RecoveryManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public boolean recover() throws LogException {
113113
if (!checkpointFound) {
114114
LOG.info("Unclean shutdown detected. Scanning journal...");
115115
broker.getBrokerPool().reportStatus("Unclean shutdown detected. Scanning log...");
116-
reader.position(1);
116+
reader.positionFirst();
117117
final Long2ObjectHashMap<Loggable> txnsStarted = new Long2ObjectHashMap<>();
118118
Checkpoint lastCheckpoint = null;
119119
long lastLsn = Lsn.LSN_INVALID;
@@ -149,7 +149,7 @@ public boolean recover() throws LogException {
149149
LOG.info("Dirty transactions: " + txnsStarted.size());
150150
// starting recovery: reposition the log reader to the last checkpoint
151151
if (lastCheckpoint == null)
152-
{reader.position(1);}
152+
{reader.positionFirst();}
153153
else {
154154
reader.position(lastCheckpoint.getLsn());
155155
next = reader.nextEntry();

0 commit comments

Comments
 (0)