[test] Further tests for large XML nodes which cause overflow pages to be journalled

Author: adamretter

src/org/exist/storage/dom/WriteOverflowPageLoggable.java

@@ -87,4 +87,12 @@ public void undo() throws LogException {
     public String dump() {
         return super.dump() + " - writing overflow page " + pageNum + "; next: " + nextPage;
     }
+
+    public long getPageNum() {
+        return pageNum;
+    }
+
+    public Value getValue() {
+        return value;
+    }
 }

src/org/exist/storage/journal/Journal.java

@@ -122,12 +122,18 @@ public final class Journal {
     /**
      * default maximum journal size
      */
-    public static final int DEFAULT_MAX_SIZE = 100; //MB
+    public static final int DEFAULT_MAX_SIZE = 100;  //MB
 
     /**
      * minimal size the journal needs to have to be replaced by a new file during a checkpoint
      */
-    private static final int DEFAULT_MIN_SIZE = 1; // MB
+    private static final int DEFAULT_MIN_SIZE = 1;  // MB
+
+    /**
+     * We use a 1 megabyte buffer.
+     */
+    public static final int BUFFER_SIZE = 1024 * 1024;  // bytes
+
 
     /**
      * Minimum size limit for the journal file before it is replaced by a new file.
@@ -220,8 +226,7 @@ public final class Journal {
     public Journal(final BrokerPool pool, final Path directory) throws EXistException {
         this.pool = pool;
         this.fsJournalDir = directory.resolve("fs.journal");
-        // we use a 1 megabyte buffer:
-        this.currentBuffer = ByteBuffer.allocateDirect(1024 * 1024);
+        this.currentBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE);
 
         this.fileSyncRunnable = new FileSyncRunnable(latch);
         this.fileSyncThread = newInstanceThread(pool, "file-sync-thread", fileSyncRunnable);
@@ -296,6 +301,11 @@ public synchronized void writeToLog(final Loggable entry) throws JournalExceptio
 
         SanityCheck.ASSERT(!inRecovery, "Write to log during recovery. Should not happen!");
         final int size = entry.getLogSize();
+
+        if (size > Short.MAX_VALUE) {
+            throw new JournalException("Journal can only write log entries of less that 32KB");
+        }
+
         final int required = size + LOG_ENTRY_BASE_LEN;
         if (required > currentBuffer.remaining()) {
             flushToLog(false);
@@ -315,7 +325,7 @@ public synchronized void writeToLog(final Loggable entry) throws JournalExceptio
         try {
             currentBuffer.put(entry.getLogType());
             currentBuffer.putLong(entry.getTransactionId());
-            currentBuffer.putShort((short) entry.getLogSize());
+            currentBuffer.putShort((short) size);
             entry.write(currentBuffer);
             currentBuffer.putShort((short) (size + LOG_ENTRY_HEADER_LEN));
         } catch (final BufferOverflowException e) {

src/org/exist/util/StringInputSource.java

@@ -0,0 +1,89 @@
+/*
+ * eXist Open Source Native XML Database
+ * Copyright (C) 2001-2018 The eXist Project
+ * http://exist-db.org
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+package org.exist.util;
+
+import com.evolvedbinary.j8fu.Either;
+import org.exist.util.io.FastByteArrayInputStream;
+import org.xml.sax.InputSource;
+
+import java.io.*;
+
+import static com.evolvedbinary.j8fu.Either.Left;
+import static com.evolvedbinary.j8fu.Either.Right;
+
+public class StringInputSource extends InputSource {
+
+    private final Either<byte[], String> source;
+
+    /**
+     * Creates a String Source from a string
+     * the InputSource will be read using
+     * {@link #getCharacterStream()}.
+     */
+    public StringInputSource(final String string) {
+        super();
+        this.source = Right(string);
+    }
+
+    /**
+     * Creates a String Source from bytes
+     * the InputSource will be read using
+     * {@link #getByteStream()}.
+     */
+    public StringInputSource(final byte[] string) {
+        super();
+        this.source = Left(string);
+    }
+
+    @Override
+    public Reader getCharacterStream() {
+        if (source.isLeft()) {
+            return null;
+        } else {
+            return new StringReader(source.right().get());
+        }
+    }
+
+    /**
+     * @throws IllegalStateException this class is immutable!
+     */
+    @Override
+    public void setCharacterStream(final Reader r) {
+        throw new IllegalStateException("StringInputSource is immutable");
+    }
+
+    @Override
+    public InputStream getByteStream() {
+        if (source.isLeft()) {
+            return new FastByteArrayInputStream(source.left().get());
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * @throws IllegalStateException this class is immutable!
+     */
+    @Override
+    public void setByteStream(final InputStream is) {
+        throw new IllegalStateException("StringInputSource is immutable");
+    }
+}

test/src/org/exist/storage/AbstractRecoverTest.java

@@ -34,19 +34,21 @@
 import org.exist.test.ExistEmbeddedServer;
 import org.exist.test.TestConstants;
 import org.exist.util.DatabaseConfigurationException;
+import org.exist.util.FileInputSource;
 import org.exist.util.FileUtils;
 import org.exist.util.LockException;
 import org.exist.xmldb.XmldbURI;
+import org.junit.After;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
+import org.xml.sax.InputSource;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Optional;
-import java.util.concurrent.Executors;
 
 import static org.junit.Assert.*;
 
@@ -55,11 +57,11 @@
  */
 public abstract class AbstractRecoverTest {
 
-    private static final boolean COMMIT = true;
-    private static final boolean NO_COMMIT = false;
+    protected static final boolean COMMIT = true;
+    protected static final boolean NO_COMMIT = false;
 
-    private static final boolean MUST_EXIST = true;
-    private static final boolean MUST_NOT_EXIST = false;
+    protected static final boolean MUST_EXIST = true;
+    protected static final boolean MUST_NOT_EXIST = false;
 
     /**
      * We set useTemporaryStorage=true for ExistEmbeddedServer
@@ -69,6 +71,11 @@ public abstract class AbstractRecoverTest {
     public final ExistEmbeddedServer existEmbeddedServer =
             new ExistEmbeddedServer(true, true);
 
+    @After
+    public void tearDown() {
+        BrokerPool.FORCE_CORRUPTION = false;
+    }
+
     @Test
     public void storeAndLoad() throws LockException, TriggerException, PermissionDeniedException, EXistException,
             IOException, DatabaseConfigurationException, InterruptedException {
@@ -603,7 +610,7 @@ public void replaceWithoutCommitThenDeleteWithoutCommitAndLoad_isRepeatable() th
      *      unfinished (i.e. neither committed, aborted, or closed)
      * @param file The file that to store
      */
-    private void store(final boolean commitAndClose, final Path file) throws EXistException, PermissionDeniedException,
+    protected void store(final boolean commitAndClose, final Path file) throws EXistException, PermissionDeniedException,
             IOException, TriggerException, LockException, InterruptedException {
         store(commitAndClose, file, FileUtils.fileName(file));
     }
@@ -618,13 +625,27 @@ private void store(final boolean commitAndClose, final Path file) throws EXistEx
      */
     private void store(final boolean commitAndClose, final Path file, final String dbFilename) throws EXistException,
             PermissionDeniedException, IOException, TriggerException, LockException, InterruptedException {
+        store(commitAndClose, new FileInputSource(file), dbFilename);
+    }
+
+    /**
+     * Store a document into the database.
+     *
+     * @param commitAndClose true if the transaction should be committed. false will leave the transaction
+     *      unfinished (i.e. neither committed, aborted, or closed)
+     * @param data The data to store in the document
+     * @param dbFilename the name to use when storing the file in the database
+     */
+    protected void store(final boolean commitAndClose, final InputSource data, final String dbFilename) throws EXistException,
+            PermissionDeniedException, IOException, TriggerException, LockException, InterruptedException {
+
 
         runSync(new BrokerTask(existEmbeddedServer.getBrokerPool(), (broker, transaction) -> {
             final Collection root = broker.getOrCreateCollection(transaction, TestConstants.TEST_COLLECTION_URI);
             assertNotNull(root);
             broker.saveCollection(transaction, root);
 
-            storeAndVerify(broker, transaction, root, file, dbFilename);
+            storeAndVerify(broker, transaction, root, data, dbFilename);
 
             if(commitAndClose) {
                 transaction.commit();
@@ -641,11 +662,11 @@ private void store(final boolean commitAndClose, final Path file, final String d
      * @param broker The database broker
      * @param transaction The database transaction
      * @param collection The Collection into which the document should be stored
-     * @param file The file which holds the content for the document to store in the database
+     * @param data The content for the document to store in the database
      * @param dbFilename The name to store the document as in the database
      */
     protected abstract void storeAndVerify(final DBBroker broker, final Txn transaction, final Collection collection,
-            final Path file, final String dbFilename) throws EXistException, PermissionDeniedException,
+            final InputSource data, final String dbFilename) throws EXistException, PermissionDeniedException,
             IOException, TriggerException, LockException;
 
     /**
@@ -668,6 +689,18 @@ private void read(final boolean shouldExist, final Path file)
      */
     private void read(final boolean shouldExist, final Path file, final String dbFilename)
             throws EXistException, PermissionDeniedException, IOException {
+        read(shouldExist, new FileInputSource(file), dbFilename);
+    }
+
+    /**
+     * Read a document from the database.
+     *
+     * @param shouldExist true if the document should exist in the database, false if the document should not exist
+     * @param data The data that was previously stored
+     * @param dbFilename The name of the file to read from the database
+     */
+    protected void read(final boolean shouldExist, final InputSource data, final String dbFilename)
+            throws EXistException, PermissionDeniedException, IOException {
         final BrokerPool pool = existEmbeddedServer.getBrokerPool();
         try(final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject()))) {
             final XmldbURI uri = TestConstants.TEST_COLLECTION_URI.append(dbFilename);
@@ -679,7 +712,7 @@ private void read(final boolean shouldExist, final Path file, final String dbFil
                 } else {
                     assertNotNull("Document does not exist in the database: " + uri, doc);
 
-                    readAndVerify(broker, doc.getDocument(), file, dbFilename);
+                    readAndVerify(broker, doc.getDocument(), data, dbFilename);
                 }
             }
         }
@@ -690,11 +723,11 @@ private void read(final boolean shouldExist, final Path file, final String dbFil
      *
      * @param broker The database broker.
      * @param doc The document from the database.
-     * @param file The file that was previously stored
+     * @param data The data that was previously stored
      * @param dbFilename The name of the file read from the database
      */
     protected abstract void readAndVerify(final DBBroker broker, final DocumentImpl doc,
-            final Path file, final String dbFilename) throws EXistException, PermissionDeniedException, IOException;
+            final InputSource data, final String dbFilename) throws EXistException, PermissionDeniedException, IOException;
 
     /**
      * Delete a document from the database.
@@ -737,7 +770,7 @@ private void delete(final boolean commitAndClose, final String dbFilename)
         }));
     }
 
-    private void flushJournal() {
+    protected void flushJournal() {
         final BrokerPool pool = existEmbeddedServer.getBrokerPool();
         pool.getJournalManager().get().flush(true, false);
     }

test/src/org/exist/storage/RecoverBinaryTest.java

@@ -31,8 +31,10 @@
 import org.exist.dom.persistent.DocumentImpl;
 import org.exist.security.PermissionDeniedException;
 import org.exist.storage.txn.Txn;
+import org.exist.util.FileInputSource;
 import org.exist.util.LockException;
 import org.exist.xmldb.XmldbURI;
+import org.xml.sax.InputSource;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -54,20 +56,24 @@ protected Path getTestFile2() throws IOException {
 
     @Override
     protected void storeAndVerify(final DBBroker broker, final Txn transaction, final Collection collection,
-            final Path file, final String dbFilename) throws EXistException, PermissionDeniedException, IOException,
+            final InputSource data, final String dbFilename) throws EXistException, PermissionDeniedException, IOException,
             TriggerException, LockException {
 
-        final byte[] data = Files.readAllBytes(file);
-        final BinaryDocument doc = collection.addBinaryResource(transaction, broker, XmldbURI.create(dbFilename), data, "application/octet-stream");
+        final Path file = ((FileInputSource)data).getFile();
+
+        final byte[] content = Files.readAllBytes(file);
+        final BinaryDocument doc = collection.addBinaryResource(transaction, broker, XmldbURI.create(dbFilename), content, "application/octet-stream");
 
         assertNotNull(doc);
         assertEquals(Files.size(file), doc.getContentLength());
     }
 
     @Override
-    protected void readAndVerify(final DBBroker broker, final DocumentImpl doc, final Path file,
+    protected void readAndVerify(final DBBroker broker, final DocumentImpl doc, final InputSource data,
             final String dbFilename) throws IOException {
 
+        final Path file = ((FileInputSource)data).getFile();
+
         final BinaryDocument binDoc = (BinaryDocument)doc;
 
         // verify the size, to ensure it is the correct content
@@ -80,8 +86,8 @@ protected void readAndVerify(final DBBroker broker, final DocumentImpl doc, fina
             final int read = cis.read(bdata);
             assertEquals(bdata.length, read);
 
-            final String data = new String(bdata);
-            assertNotNull(data);
+            final String content = new String(bdata);
+            assertNotNull(content);
 
             assertEquals(expectedSize, cis.getByteCount());
         }