[feature] Adds ContentFilePool and companion tests

Backport from develop branch.

Signed-off-by: Patrick Reinhart <[email protected]>

Author: reinhapa

exist-core/pom.xml

@@ -509,6 +509,11 @@
             <artifactId>junit-jupiter-params</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.easymock</groupId>
             <artifactId>easymock</artifactId>

exist-core/src/main/java/org/exist/util/Configuration.java

@@ -33,6 +33,7 @@
 import org.exist.start.Main;
 import org.exist.storage.lock.LockManager;
 import org.exist.storage.lock.LockTable;
+import org.exist.util.io.ContentFilePool;
 import org.exist.xquery.*;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -96,6 +97,8 @@
 
 import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple;
 import static javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING;
+import static org.exist.util.io.ContentFilePool.PROPERTY_IN_MEMORY_SIZE;
+import static org.exist.util.io.VirtualTempPath.DEFAULT_IN_MEMORY_SIZE;
 
 
 public class Configuration implements ErrorHandler
@@ -107,6 +110,7 @@ public class Configuration implements ErrorHandler
     protected DocumentBuilder         builder        = null;
     protected HashMap<String, Object> config         = new HashMap<>(); //Configuration
 
+    private static final String PRP_DETAILS = "{}: {}";
     private static final String XQUERY_CONFIGURATION_ELEMENT_NAME = "xquery";
     private static final String XQUERY_BUILTIN_MODULES_CONFIGURATION_MODULES_ELEMENT_NAME = "builtin-modules";
     private static final String XQUERY_BUILTIN_MODULES_CONFIGURATION_MODULE_ELEMENT_NAME = "module";
@@ -291,6 +295,11 @@ public Configuration(String configFilename, Optional<Path> existHomeDirname) thr
                 configureValidation(existHomeDirname, (Element)validations.item(0));
             }
 
+            //RPC server
+            final NodeList rpcServer = doc.getElementsByTagName("rpc-server");
+            if (rpcServer.getLength() > 0) {
+                configureRpcServer((Element) rpcServer.item(0));
+            }
         }
         catch(final SAXException | IOException | ParserConfigurationException e) {
             LOG.error("error while reading config file: {}", configFilename, e);
@@ -1499,8 +1508,28 @@ private void configureValidation(final Optional<Path> dbHome, final Element vali
         final GrammarPool gp = new GrammarPool();
         config.put( XMLReaderObjectFactory.GRAMMAR_POOL, gp);
     }
-    
-     /**
+
+    private void configureRpcServer(final Element validation) {
+        final NodeList contentFile = validation.getElementsByTagName("content-file");
+        if (contentFile.getLength() > 0) {
+            final String inMemorySize = ((Element) contentFile.item(0)).getAttribute("in-memory-size");
+            if (inMemorySize != null) {
+                config.put(PROPERTY_IN_MEMORY_SIZE, parseInt(inMemorySize, DEFAULT_IN_MEMORY_SIZE));
+                LOG.debug(PRP_DETAILS, PROPERTY_IN_MEMORY_SIZE, config.get(PROPERTY_IN_MEMORY_SIZE));
+            }
+        }
+        final NodeList contentFilePool = validation.getElementsByTagName("content-file-pool");
+        if (contentFilePool.getLength() > 0) {
+            final String size = ((Element) contentFilePool.item(0)).getAttribute("size");
+            config.put(ContentFilePool.PROPERTY_POOL_SIZE, parseInt(size, -1));
+            LOG.debug(PRP_DETAILS, ContentFilePool.PROPERTY_POOL_SIZE, config.get(ContentFilePool.PROPERTY_POOL_SIZE));
+            final String maxIdle = ((Element) contentFilePool.item(0)).getAttribute("max-idle");
+            config.put(ContentFilePool.PROPERTY_POOL_MAX_IDLE, parseInt(maxIdle, 5));
+            LOG.debug(PRP_DETAILS, ContentFilePool.PROPERTY_POOL_MAX_IDLE, config.get(ContentFilePool.PROPERTY_POOL_MAX_IDLE));
+        }
+    }
+
+    /**
      * Gets the value of a configuration attribute
      *
      * The value typically is specified in the conf.xml file, but can be overridden with using a System Property
@@ -1596,6 +1625,17 @@ public void removeProperty(final String name) {
         config.remove(name);
     }
 
+    public int getInteger(final String name) {
+        return getInteger(name, -1);
+    }
+
+    public int getInteger(final String name, final int defaultValue) {
+        return Optional.ofNullable(getProperty(name))
+                .filter(v -> v instanceof Integer)
+                .map(v -> (int) v)
+                .orElse(defaultValue);
+    }
+
     /**
      * Takes the passed string and converts it to a non-null <code>Boolean</code> object. If value is null, the specified default value is used.
      * Otherwise, Boolean.TRUE is returned if and only if the passed string equals &quot;yes&quot; or &quot;true&quot;, ignoring case.
@@ -1633,14 +1673,6 @@ public static int parseInt(@Nullable final String value, final int defaultValue)
         }
     }
 
-    public int getInteger(final String name) {
-        return Optional.ofNullable(getProperty(name))
-                .filter(v -> v instanceof Integer)
-                .map(v -> (int)v)
-                .orElse(-1);
-    }
-
-
     /**
      * (non-Javadoc).
      *

exist-core/src/main/java/org/exist/util/io/ContentFile.java

@@ -25,21 +25,50 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 
 /**
  * @author <a href="mailto:[email protected]">Patrick Reinhart</a>
  */
 public interface ContentFile extends AutoCloseable {
 
     @Override
+    /**
+     * Closes all resource held by the implementation as open files or off HEAP memory as example.
+     */
     void close();
 
+    /**
+     * Returns the complete content as an byte array.
+     *
+     * @return the content as byte array
+     */
     byte[] getBytes();
 
+    /**
+     * Returns the size of the conent in bytes
+     *
+     * @return the content size
+     */
     long size();
 
+    /**
+     * Returns a new {@link InputStream} instance based on the content data.
+     *
+     * @return a new input stream based on the content.
+     * @throws IOException if an error occurs accessing the content
+     */
     default InputStream newInputStream() throws IOException {
-       return new ByteArrayInputStream(getBytes());
+        return new ByteArrayInputStream(getBytes());
     }
 
+    /**
+     * Returns a new {@link OutputStream} instance to write content data.
+     * @return a new output stream for writing content
+     * @throws IOException if an error occurs accessing the content
+     * @throws UnsupportedOperationException if the operation is not available
+     */
+    default OutputStream newOutputStream() throws IOException {
+        throw new UnsupportedOperationException("not supported");
+    }
 }

exist-core/src/main/java/org/exist/util/io/ContentFilePool.java

@@ -0,0 +1,94 @@
+/*
+ * eXist-db Open Source Native XML Database
+ * Copyright (C) 2001 The eXist-db Authors
+ *
+ * [email protected]
+ * http://www.exist-db.org
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+package org.exist.util.io;
+
+import org.apache.commons.pool2.impl.GenericObjectPool;
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.exist.util.Configuration;
+
+/**
+ * Generic pool for {@link ContentFile} instances used to represent RPC server data up to a defined size in memory first
+ * before storing the data in the file system using the {@link TemporaryFileManager}.
+ *
+ * @author <a href="mailto:[email protected]">Patrick Reinhart</a>
+ */
+public final class ContentFilePool extends GenericObjectPool<ContentFile> {
+    /**
+     * Variable defining the {@link ContentFile} maximum in memory stored size in bytes.
+     */
+    public static final String PROPERTY_IN_MEMORY_SIZE = "rpc-server.content-file.in-memory-size";
+    /**
+     * Variable defining the maximum mool size holding {@link ContentFile values.}
+     */
+    public static final String PROPERTY_POOL_SIZE = "rpc-server.content-file-pool.size";
+    /**
+     * Variable defining the maximum mool size holding {@link ContentFile values.}
+     */
+    public static final String PROPERTY_POOL_MAX_IDLE = "rpc-server.content-file-pool.max-idle";
+
+    /**
+     * Creates a new pool using the givem temporary file manager, configuration and maximum idle time.
+     *
+     * @param tempFileManager the temporary file manager used when need to swap out data bigger than max in memory size
+     * @param config the configuration used to configure the main pool properties
+     */
+    public ContentFilePool(final TemporaryFileManager tempFileManager, final Configuration config) {
+        super(new ContentFilePoolObjectFactory(tempFileManager, toInMemorySize(config)), toPoolConfig(config));
+    }
+
+    private static int toInMemorySize(final Configuration config) {
+        return config.getInteger(PROPERTY_IN_MEMORY_SIZE, VirtualTempPath.DEFAULT_IN_MEMORY_SIZE);
+    }
+
+    private static GenericObjectPoolConfig<ContentFile> toPoolConfig(final Configuration config) {
+        final GenericObjectPoolConfig<ContentFile> poolConfig = new GenericObjectPoolConfig<>();
+        poolConfig.setBlockWhenExhausted(false);
+        poolConfig.setLifo(true);
+        poolConfig.setMaxIdle(config.getInteger(PROPERTY_POOL_MAX_IDLE));
+        poolConfig.setMaxTotal(config.getInteger(PROPERTY_POOL_SIZE));
+        poolConfig.setJmxNameBase("org.exist.management.exist:type=ContentFilePool");
+        return poolConfig;
+    }
+
+    @Override
+    public ContentFile borrowObject() {
+        try {
+            return super.borrowObject();
+        } catch (final Exception e) {
+            throw new IllegalStateException("Error while borrowing ContentFile", e);
+        }
+    }
+
+    @Override
+    public void returnObject(final ContentFile obj) {
+        if (obj == null) {
+            return;
+        }
+        try {
+            obj.close();
+            super.returnObject(obj);
+        } catch (final Exception e) {
+            throw new IllegalStateException("Error while returning ContentFile", e);
+        }
+    }
+}

exist-core/src/main/java/org/exist/util/io/ContentFilePoolObjectFactory.java

@@ -0,0 +1,58 @@
+/*
+ * eXist-db Open Source Native XML Database
+ * Copyright (C) 2001 The eXist-db Authors
+ *
+ * [email protected]
+ * http://www.exist-db.org
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+package org.exist.util.io;
+
+import org.apache.commons.pool2.BasePooledObjectFactory;
+import org.apache.commons.pool2.PooledObject;
+import org.apache.commons.pool2.impl.DefaultPooledObject;
+
+/**
+ * Object factory responsible for creating {@link ContentFile} instances and wrap thos into pool objects.
+ *
+ * @author <a href="mailto:[email protected]">Patrick Reinhart</a>
+ */
+public final class ContentFilePoolObjectFactory extends BasePooledObjectFactory<ContentFile> {
+    private final TemporaryFileManager tempFileManager;
+    private final int inMemorySize;
+
+    /**
+     * Creates a object factory instance using the given temporary file manager and maximum in memotx size.
+     *
+     * @param tempFileManager the temporary file manager used when need to swap out data bigger than max in memory size
+     * @param inMemorySize the maximum in memory size before swappiing the data to a temporay file
+     */
+    public ContentFilePoolObjectFactory(final TemporaryFileManager tempFileManager, int inMemorySize) {
+        this.tempFileManager = tempFileManager;
+        this.inMemorySize = inMemorySize;
+    }
+
+    @Override
+    public ContentFile create() throws Exception {
+        return new VirtualTempPath(inMemorySize, tempFileManager);
+    }
+
+    @Override
+    public PooledObject<ContentFile> wrap(ContentFile obj) {
+        return new DefaultPooledObject<>(obj);
+    }
+}