Merge pull request #112 from evolvedbinary/7.x.x/hotfix/array-set-sizing

[7.x.x] Reduce memory use and improve NodeSet sizing

Author: adamretter

exist-core/pom.xml

@@ -868,7 +868,7 @@
                                 <include>src/test/java/org/exist/deadlocks/GetReleaseBrokerDeadlocksTest.java</include>
                                 <include>src/main/java/org/exist/dom/NodeListImpl.java</include>
                                 <include>src/main/java/org/exist/dom/QName.java</include>
-                                <exclude>src/main/java/org/exist/dom/memtree/AbstractCharacterData.java</exclude>
+                                <include>src/main/java/org/exist/dom/memtree/AbstractCharacterData.java</include>
                                 <include>src/main/java/org/exist/dom/memtree/AttrImpl.java</include>
                                 <include>src/main/java/org/exist/dom/memtree/DocumentBuilderReceiver.java</include>
                                 <include>src/main/java/org/exist/dom/memtree/DocumentImpl.java</include>
@@ -879,9 +879,10 @@
                                 <include>src/main/java/org/exist/dom/memtree/ElementImpl.java</include>
                                 <include>src/test/java/org/exist/dom/memtree/MemtreeInXQueryTest.java</include>
                                 <include>src/main/java/org/exist/dom/memtree/MemTreeBuilder.java</include>
-                                <exclude>src/main/java/org/exist/dom/memtree/NamespaceNode.java</exclude>
+                                <include>src/main/java/org/exist/dom/memtree/NamespaceNode.java</include>
                                 <include>src/main/java/org/exist/dom/memtree/NodeImpl.java</include>
-                                <exclude>src/main/java/org/exist/dom/memtree/ProcessingInstructionImpl.java</exclude>
+                                <include>src/main/java/org/exist/dom/memtree/ProcessingInstructionImpl.java</include>
+                                <include>src/main/java/org/exist/dom/persistent/AbstractArrayNodeSet.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/AbstractCharacterData.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/AttrImpl.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/BinaryDocument.java</include>
@@ -892,7 +893,9 @@
                                 <include>src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/DocumentSet.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/ElementImpl.java</include>
+                                <include>src/main/java/org/exist/dom/persistent/NewArrayNodeSet.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/NodeProxy.java</include>
+                                <include>src/main/java/org/exist/dom/persistent/NodeSet.java</include>
                                 <include>src/test/java/org/exist/dom/persistent/NodeTest.java</include>
                                 <include>src/main/java/org/exist/dom/persistent/LockToken.java</include>
                                 <include>src/test/java/org/exist/dom/persistent/PersistentDomTest.java</include>
@@ -1466,6 +1469,7 @@
                                 <exclude>src/main/java/org/exist/dom/memtree/reference/ElementReferenceImpl.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/memtree/reference/ProcessingInstructionReferenceImpl.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/memtree/reference/TextReferenceImpl.java</exclude>
+                                <exclude>src/main/java/org/exist/dom/persistent/AbstractArrayNodeSet.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/AbstractCharacterData.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/AttrImpl.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/BinaryDocument.java</exclude>
@@ -1476,7 +1480,9 @@
                                 <exclude>src/main/java/org/exist/dom/persistent/DocumentSet.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/DocumentTypeImpl.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/ElementImpl.java</exclude>
+                                <exclude>src/main/java/org/exist/dom/persistent/NewArrayNodeSet.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/NodeProxy.java</exclude>
+                                <exclude>src/main/java/org/exist/dom/persistent/NodeSet.java</exclude>
                                 <exclude>src/test/java/org/exist/dom/persistent/NodeTest.java</exclude>
                                 <exclude>src/main/java/org/exist/dom/persistent/LockToken.java</exclude>
                                 <exclude>src/test/java/org/exist/dom/persistent/PersistentDomTest.java</exclude>

exist-core/src/main/java/org/exist/dom/persistent/AbstractArrayNodeSet.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *

exist-core/src/main/java/org/exist/dom/persistent/AbstractNodeSet.java

@@ -547,7 +547,7 @@ public NodeSet getContextNodes(final int contextId) {
                     if (Expression.NO_CONTEXT_ID != contextId) {
                         context.addContextNode(contextId, context);
                     }
-                    if (lastDoc != null && lastDoc.getDocId() != context.getOwnerDocument().getDocId()) {
+                    if (lastDoc == null || lastDoc.getDocId() != context.getOwnerDocument().getDocId()) {
                         lastDoc = context.getOwnerDocument();
                         result.add(context, getSizeHint(lastDoc));
                     } else {

exist-core/src/main/java/org/exist/dom/persistent/NewArrayNodeSet.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -41,7 +65,12 @@
 import org.w3c.dom.Node;
 
 import javax.annotation.Nullable;
-import java.util.*;
+import java.lang.ref.WeakReference;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Set;
 
 /**
  * A fast node set implementation, based on arrays to store nodes and documents.
@@ -67,7 +96,7 @@
  */
 public class NewArrayNodeSet extends AbstractArrayNodeSet implements ExtNodeSet, DocumentSet {
 
-    private Set<Collection> cachedCollections = null;
+    @Nullable private WeakReference<Set<Collection>> cachedCollectionsRef = null;
 
     private int documentCount = 0;
 
@@ -935,10 +964,10 @@ public NodeSet getContextNodes(final int contextId) {
                 if(contextNode.getContextId() == contextId) {
                     final NodeProxy context = contextNode.getNode();
                     context.addMatches(current);
-                    if(Expression.NO_CONTEXT_ID != contextId) {
+                    if (Expression.NO_CONTEXT_ID != contextId) {
                         context.addContextNode(contextId, context);
                     }
-                    if(lastDoc != null && lastDoc.getDocId() != context.getOwnerDocument().getDocId()) {
+                    if (lastDoc == null || lastDoc.getDocId() != context.getOwnerDocument().getDocId()) {
                         lastDoc = context.getOwnerDocument();
                         result.add(context, getSizeHint(lastDoc));
                     } else {
@@ -1014,16 +1043,27 @@ public boolean equalDocs(final DocumentSet other) {
 
     @Override
     public Iterator<Collection> getCollectionIterator() {
-        sort();
-        if(cachedCollections == null) {
-            cachedCollections = new HashSet<>();
-            for(int i = 0; i < documentCount; i++) {
-                final DocumentImpl doc = nodes[documentNodesOffset[i]].getOwnerDocument();
-                if(!cachedCollections.contains(doc.getCollection())) {
-                    cachedCollections.add(doc.getCollection());
-                }
+        // First, try and retrieve from Cache
+        Set<Collection> cachedCollections;
+        if (this.cachedCollectionsRef != null) {
+            cachedCollections = this.cachedCollectionsRef.get();
+            if (cachedCollections != null) {
+                return cachedCollections.iterator();
             }
         }
+
+        sort();
+
+        // Second, Cache is empty, so create a Cache and return
+        cachedCollections = new HashSet<>();
+        for (int i = 0; i < documentCount; i++) {
+            final DocumentImpl doc = nodes[documentNodesOffset[i]].getOwnerDocument();
+            final Collection collection = doc.getCollection();
+            cachedCollections.add(collection);
+        }
+
+        this.cachedCollectionsRef = new WeakReference<>(cachedCollections);
+
         return cachedCollections.iterator();
     }
 

exist-core/src/main/java/org/exist/dom/persistent/NodeSet.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -377,7 +401,7 @@ boolean matchAncestorDescendant(NodeSet al, int mode, boolean includeSelf,
      *
      * @param contextId used to track context nodes when evaluating predicate
      *                  expressions. If contextId != {@link org.exist.xquery.Expression#NO_CONTEXT_ID}, the current context
-     *                  will be added to each result of the of the selection.
+     *                  will be added to each result of the selection.
      * @return all context nodes associated with the nodes in this node set.
      */
     NodeSet getContextNodes(int contextId);