Merge pull request #11374 from GlobalDataverseCommunityConsortium/solr-index-improvements

Solr Index Improvements

Author: ofahimIQSS

conf/solr/solrconfig.xml

@@ -238,7 +238,7 @@
          have some sort of hard autoCommit to limit the log size.
       -->
     <autoCommit>
-      <maxTime>${solr.autoCommit.maxTime:30000}</maxTime>
+      <maxTime>${solr.autoCommit.maxTime:300000}</maxTime>
       <openSearcher>false</openSearcher>
     </autoCommit>
 

doc/release-notes/11374-indexing-improvement.md

@@ -0,0 +1,5 @@
+### Solr Indexing speed improved
+
+The performance of Solr indexing has been significantly improved, particularly for datasets with many files.
+
+A new dataverse.solr.min-files-to-use-proxy microprofile setting can be used to further improve performance/lower memory requirements for datasets with many files (e.g. 500+) (defaults to Integer.MAX, disabling use of the new functionality)

doc/sphinx-guides/source/installation/config.rst

@@ -2689,6 +2689,17 @@ when using it to configure your core name!
 
 Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_SOLR_PATH``.
 
+dataverse.solr.min-files-to-use-proxy
++++++++++++++++++++++++++++++++++++++
+
+Specifies when to use a smaller datafile proxy object for the purposes of dataset indexing. This can lower memory requirements
+and improve performance when reindexing large datasets (e.g. those with hundreds or thousands of files). (Creating the proxy may slightly slow indexing datasets with only a few files.)
+
+This setting represents a number of files for which the datafile procy should be used. By default, this is set to Interger.MAX which disables using the proxy.
+A recommended value would be ~1000 but the optimal value may vary depending on details of your installation.  
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_SOLR_MIN_FILES_TO_USE_PROXY``.
+
 dataverse.solr.concurrency.max-async-indexes
 ++++++++++++++++++++++++++++++++++++++++++++
 

src/main/java/edu/harvard/iq/dataverse/DataFile.java

@@ -13,6 +13,7 @@
 import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
 import edu.harvard.iq.dataverse.ingest.IngestReport;
 import edu.harvard.iq.dataverse.ingest.IngestRequest;
+import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.ShapefileHandler;
@@ -23,6 +24,7 @@
 import java.util.Objects;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -50,6 +52,26 @@
         @NamedQuery(name="DataFile.findDataFileThatReplacedId",
                 query="SELECT s.id FROM DataFile s WHERE s.previousDataFileId=:identifier")
 })
+@NamedNativeQuery(
+        name = "DataFile.getDataFileInfoForPermissionIndexing",
+        query = "SELECT fm.label, df.id, dvo.publicationDate " +
+                "FROM filemetadata fm " +
+                "JOIN datafile df ON fm.datafile_id = df.id " +
+                "JOIN dvobject dvo ON df.id = dvo.id " +
+                "WHERE fm.datasetversion_id = ?",
+        resultSetMapping = "DataFileInfoMapping"
+    )
+    @SqlResultSetMapping(
+        name = "DataFileInfoMapping",
+        classes = @ConstructorResult(
+            targetClass = SolrIndexServiceBean.DataFileProxy.class,
+            columns = {
+                @ColumnResult(name = "label", type = String.class),
+                @ColumnResult(name = "id", type = Long.class),
+                @ColumnResult(name = "publicationDate", type = Date.class)
+            }
+        )
+    )
 @Entity
 @Table(indexes = {@Index(columnList="ingeststatus")
         , @Index(columnList="checksumvalue")

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -20,17 +20,20 @@
 import java.util.Objects;
 import java.util.Set;
 import jakarta.persistence.CascadeType;
+import jakarta.persistence.ColumnResult;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Index;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.ManyToOne;
+import jakarta.persistence.NamedNativeQuery;
 import jakarta.persistence.NamedQueries;
 import jakarta.persistence.NamedQuery;
 import jakarta.persistence.NamedStoredProcedureQuery;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.OneToOne;
 import jakarta.persistence.OrderBy;
 import jakarta.persistence.ParameterMode;
+import jakarta.persistence.SqlResultSetMapping;
 import jakarta.persistence.StoredProcedureParameter;
 import jakarta.persistence.Table;
 import jakarta.persistence.Temporal;
@@ -71,6 +74,23 @@
     @NamedQuery(name = "Dataset.countAll",
                 query = "SELECT COUNT(ds) FROM Dataset ds")
 })
+@NamedNativeQuery(
+        name = "Dataset.findAllOrSubsetOrderByFilesOwned",
+        query = "SELECT DISTINCT CAST(o.id AS BIGINT) as id, COUNT(f.id) as numFiles " +
+                "FROM dvobject o " +
+                "LEFT JOIN dvobject f ON f.owner_id = o.id " +
+                "WHERE o.dtype = 'Dataset' " +
+                "AND (? = false OR o.indexTime IS NULL) " +
+                "GROUP BY o.id " +
+                "ORDER BY numfiles ASC, id",
+        resultSetMapping = "DatasetIdMapping"
+    )
+@SqlResultSetMapping(
+    name = "DatasetIdMapping",
+    columns = {
+        @ColumnResult(name = "id", type = Long.class)
+    }
+)
 
 /*
     Below is the database stored procedure for getting a string dataset id.

src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java

@@ -279,32 +279,9 @@ public List<Long> findAllOrSubsetOrderByFilesOwned(boolean skipIndexed) {
         SEK - 11/09/2021
         */
 
-        String skipClause = skipIndexed ? "AND o.indexTime is null " : "";
-        Query query = em.createNativeQuery(" Select distinct(o.id), count(f.id) as numFiles FROM dvobject o " +
-            "left join dvobject f on f.owner_id = o.id  where o.dtype = 'Dataset' "
-                + skipClause
-                + " group by o.id "
-                + "ORDER BY count(f.id) asc, o.id");
-
-        List<Object[]> queryResults;
-        queryResults = query.getResultList();
-
-        List<Long> retVal = new ArrayList();
-        for (Object[] result : queryResults) {
-            Long dsId;
-            if (result[0] != null) {
-                try {
-                    dsId = Long.parseLong(result[0].toString()) ;
-                } catch (Exception ex) {
-                    dsId = null;
-                }
-                if (dsId == null) {
-                    continue;
-                }
-                retVal.add(dsId);
-            }
-        }
-        return retVal;
+        return em.createNamedQuery("Dataset.findAllOrSubsetOrderByFilesOwned", Long.class)
+                .setParameter(1, skipIndexed)
+                .getResultList();
     }
 
     /**

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -25,6 +25,7 @@
 import jakarta.json.Json;
 import jakarta.json.JsonArrayBuilder;
 import jakarta.persistence.Column;
+import jakarta.persistence.ColumnResult;
 import jakarta.persistence.Entity;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
@@ -35,8 +36,10 @@
 import jakarta.persistence.JoinTable;
 import jakarta.persistence.ManyToMany;
 import jakarta.persistence.ManyToOne;
+import jakarta.persistence.NamedNativeQuery;
 import jakarta.persistence.OneToMany;
 import jakarta.persistence.OrderBy;
+import jakarta.persistence.SqlResultSetMapping;
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
 import jakarta.persistence.Version;
@@ -62,6 +65,39 @@
  * @author skraffmiller
  */
 @Table(indexes = {@Index(columnList="datafile_id"), @Index(columnList="datasetversion_id")} )
+@NamedNativeQuery(
+        name = "FileMetadata.compareFileMetadata",
+        query = "WITH fm_categories AS (" +
+                "    SELECT fmd.filemetadatas_id, " +
+                "           STRING_AGG(dfc.name, ',' ORDER BY dfc.name) AS categories " +
+                "    FROM FileMetadata_DataFileCategory fmd " +
+                "    JOIN DataFileCategory dfc ON fmd.filecategories_id = dfc.id " +
+                "    GROUP BY fmd.filemetadatas_id " +
+                ") " +
+                "SELECT fm1.id " +
+                "FROM FileMetadata fm1 " +
+                "LEFT JOIN FileMetadata fm2 ON fm1.datafile_id = fm2.datafile_id " +
+                "    AND fm2.datasetversion_id = ?1 " +
+                "LEFT JOIN fm_categories fc1 ON fc1.filemetadatas_id = fm1.id " +
+                "LEFT JOIN fm_categories fc2 ON fc2.filemetadatas_id = fm2.id " +
+                "WHERE fm1.datasetversion_id = ?2 " +
+                "    AND (fm2.id IS NULL " +
+                "         OR (fm1.datafile_id = fm2.datafile_id " +
+                "             AND (fm2.description IS DISTINCT FROM fm1.description " +
+                "                  OR fm2.directoryLabel IS DISTINCT FROM fm1.directoryLabel " +
+                "                  OR fm2.label != fm1.label " +
+                "                  OR fm2.restricted IS DISTINCT FROM fm1.restricted " +
+                "                  OR fm2.prov_freeform IS DISTINCT FROM fm1.prov_freeform " +
+                "                  OR fc1.categories IS DISTINCT FROM fc2.categories " +
+                "                 ) " +
+                "            ) " +
+                "        )",
+                resultSetMapping = "IdToLongMapping"
+    )
+@SqlResultSetMapping(
+        name = "IdToLongMapping",
+        columns = @ColumnResult(name = "id", type = Long.class)
+    )
 @Entity
 public class FileMetadata implements Serializable {
     private static final long serialVersionUID = 1L;

src/main/java/edu/harvard/iq/dataverse/FileVersionDifference.java

@@ -64,26 +64,41 @@ When there are changes (after v4.19)to the file metadata data model this method
 
         if (newFileMetadata.getDataFile() == null && originalFileMetadata != null){
             //File Deleted
-            updateDifferenceSummary("", BundleUtil.getStringFromBundle("file.versionDifferences.fileGroupTitle"),  0, 0, 1, 0);
+            if (details) {
+                updateDifferenceSummary("", BundleUtil.getStringFromBundle("file.versionDifferences.fileGroupTitle"), 0, 0, 1, 0);
+            }
             return false;
         }
-        
-        if (this.originalFileMetadata == null && this.newFileMetadata.getDataFile() != null ){
+
+        if (this.originalFileMetadata == null && this.newFileMetadata.getDataFile() != null){
             //File Added
-            if (!details) return false;
-            retVal = false;
-            updateDifferenceSummary( "", BundleUtil.getStringFromBundle("file.versionDifferences.fileGroupTitle"), 1, 0, 0, 0);
-        }
-        
-        //Check to see if File replaced
-        if (originalFileMetadata != null &&
-                 newFileMetadata.getDataFile() != null && originalFileMetadata.getDataFile() != null &&!this.originalFileMetadata.getDataFile().equals(this.newFileMetadata.getDataFile())){
-            if (!details) return false;
-            updateDifferenceSummary( "", BundleUtil.getStringFromBundle("file.versionDifferences.fileGroupTitle"), 0, 0, 0, 1);
+            if (!details) {
+                return false;
+            }
             retVal = false;
+            updateDifferenceSummary("", BundleUtil.getStringFromBundle("file.versionDifferences.fileGroupTitle"), 1, 0, 0, 0);
         }
 
-        if ( originalFileMetadata != null) {
+        if (originalFileMetadata != null) {
+            // Check to see if File replaced
+            if (newFileMetadata.getDataFile() != null && originalFileMetadata.getDataFile() != null && !this.originalFileMetadata.getDataFile().equals(this.newFileMetadata.getDataFile())) {
+                if (!details)
+                    return false;
+                updateDifferenceSummary("", BundleUtil.getStringFromBundle("file.versionDifferences.fileGroupTitle"), 0, 0, 0, 1);
+                retVal = false;
+            }
+
+            /*
+             * Get Restriction Differences
+             */
+            if (originalFileMetadata.isRestricted() != newFileMetadata.isRestricted()) {
+                if (details) {
+                    String value2 = newFileMetadata.isRestricted() ? BundleUtil.getStringFromBundle("file.versionDifferences.fileRestricted") : BundleUtil.getStringFromBundle("file.versionDifferences.fileUnrestricted");
+                    updateDifferenceSummary(BundleUtil.getStringFromBundle("file.versionDifferences.fileAccessTitle"), value2, 0, 0, 0, 0);
+                }
+                retVal = false;
+            }
+
             if (!newFileMetadata.getLabel().equals(originalFileMetadata.getLabel())) {
                 if (details) {
                     differenceDetailItems.add(new FileDifferenceDetailItem(BundleUtil.getStringFromBundle("file.versionDifferences.fileNameDetailTitle"), originalFileMetadata.getLabel(), newFileMetadata.getLabel()));
@@ -94,10 +109,8 @@ When there are changes (after v4.19)to the file metadata data model this method
                         BundleUtil.getStringFromBundle("file.versionDifferences.fileNameDetailTitle"), 0, 1, 0, 0);
                 retVal = false;
             }
-        }
 
-        //Description differences
-        if ( originalFileMetadata != null) {
+            //Description differences
             if (newFileMetadata.getDescription() != null
                     && originalFileMetadata.getDescription() != null
                     && !newFileMetadata.getDescription().equals(originalFileMetadata.getDescription())) {
@@ -134,9 +147,7 @@ When there are changes (after v4.19)to the file metadata data model this method
                         BundleUtil.getStringFromBundle("file.versionDifferences.descriptionDetailTitle"), 0, 0, 1, 0);
                 retVal = false;
             }
-        } 
-        //Provenance Description differences
-        if ( originalFileMetadata != null) {
+            //Provenance Description differences
             if ((newFileMetadata.getProvFreeForm() != null && !newFileMetadata.getProvFreeForm().isEmpty())
                     && (originalFileMetadata.getProvFreeForm() != null && !originalFileMetadata.getProvFreeForm().isEmpty())
                     && !newFileMetadata.getProvFreeForm().equals(originalFileMetadata.getProvFreeForm())) {
@@ -173,8 +184,6 @@ When there are changes (after v4.19)to the file metadata data model this method
                         BundleUtil.getStringFromBundle("file.versionDifferences.provenanceDetailTitle"), 0, 0, 1, 0);
                 retVal = false;
             }
-        }
-        if (originalFileMetadata != null) {
             /*
             get Tags differences
             */
@@ -188,7 +197,9 @@ When there are changes (after v4.19)to the file metadata data model this method
             }
 
             if (!value1.equals(value2)) {
-                if (!details) return false;
+                if (!details) {
+                    return false;
+                }
                 int added = 0;
                 int deleted = 0;
 
@@ -223,16 +234,7 @@ When there are changes (after v4.19)to the file metadata data model this method
                 }
                 retVal = false;
             }
-            
-            /*
-            Get Restriction Differences
-            */
-            value1 = originalFileMetadata.isRestricted() ? BundleUtil.getStringFromBundle("file.versionDifferences.fileRestricted") : BundleUtil.getStringFromBundle("file.versionDifferences.fileUnrestricted");
-            value2 = newFileMetadata.isRestricted() ? BundleUtil.getStringFromBundle("file.versionDifferences.fileRestricted") : BundleUtil.getStringFromBundle("file.versionDifferences.fileUnrestricted");
-            if (!value1.equals(value2)) {
-                    updateDifferenceSummary(BundleUtil.getStringFromBundle("file.versionDifferences.fileAccessTitle"), value2, 0, 0, 0, 0);
-                    retVal = false;
-            }
+
         }
         return retVal;
     }

src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java

@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.RoleAssignee;
 import edu.harvard.iq.dataverse.authorization.groups.Group;
 import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean;
@@ -27,6 +28,7 @@
 import jakarta.ejb.Stateless;
 import jakarta.inject.Named;
 import jakarta.persistence.EntityManager;
+import jakarta.persistence.NamedNativeQuery;
 import jakarta.persistence.PersistenceContext;
 import org.apache.commons.lang3.StringUtils;
 
@@ -395,6 +397,15 @@ public List<RoleAssignee> filterRoleAssignees(String query, DvObject dvObject, L
 
         return roleAssigneeList;
     }
+    
+
+    public List<String> findAssigneesWithPermissionOnDvObject(Long objectId, Permission permission) {
+        int bitpos = 63 - permission.ordinal();
+        return em.createNamedQuery("RoleAssignment.findAssigneesWithPermissionOnDvObject", String.class)
+                 .setParameter(1, bitpos)
+                 .setParameter(2, objectId)
+                 .getResultList();
+    }
 
     private void msg(String s) {
         //System.out.println(s);