Merge branch 'develop' into 11528-display-collection-size-on-add-files-page

Author: stevenwinship

doc/release-notes/11976-fix-file-replace.md

@@ -0,0 +1,3 @@
+A bug introduced in Dataverse 6.8 that makes attempts to replace non-tabular files via the current Dataverse UI fail has been fixed. (The bug would also cause the replace API to fail if an empty dataFileTags array is sent.)
+
+See #11976

doc/release-notes/12008-dataset-api-locks.md

@@ -0,0 +1 @@
+The API returning information about datasets (`/api/datasets/{id}`) now includes a `locks` field containing a list of the types of all existing locks, e.g. `"locks": ["InReview"]`.

doc/release-notes/12094permission-indexing-improvements3.md

@@ -0,0 +1,2 @@
+(assuming the earlier PRs have been merged, tehre will be a section on indexing improvements already)
+This release also avoids creating unused Solr entries for files in drafts of new versions of published datasets (decreasing the Solr db size and thereby improving performance).

doc/release-notes/dataverse-uploader-github-action-fixed.md

@@ -0,0 +1 @@
+It came to our attention that the [Dataverse Uploader GitHub Action](https://guides.dataverse.org/en/6.10/admin/integrations.html#github) was [failing](https://github.com/IQSS/dataverse-uploader/issues/28) with an "unhashable type" error. This has been fixed in a new release, [v1.7](https://github.com/IQSS/dataverse-uploader/releases/tag/v1.7).

doc/sphinx-guides/source/api/search.rst

@@ -17,7 +17,6 @@ The parameters and JSON response are partly inspired by the `GitHub Search API <
 
 Please note that in Dataverse Software 4.3 and older the "citation" field wrapped the persistent ID URL in an ``<a>`` tag but this has been changed to plaintext. If you want the old value with HTML in it, a new field called "citationHtml" can be used.
 
-
 Parameters
 ----------
 
@@ -27,7 +26,7 @@ Name             Type     Description
 q                string   The search term or terms. Using "title:data" will search only the "title" field. "*" can be used as a wildcard either alone or adjacent to a term (i.e. "bird*"). For example, https://demo.dataverse.org/api/search?q=title:data . For a list of fields to search, please see https://github.com/IQSS/dataverse/issues/2558 (for now).
 type             string   Can be either "dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned.  For example, https://demo.dataverse.org/api/search?q=*&type=dataset
 subtree          string   The identifier of the Dataverse collection to which the search should be narrowed. The subtree of this Dataverse collection and all its children will be searched.  Multiple "subtree" parameters can be used to include multiple Dataverse collections. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats .
-sort             string   The sort field. Supported values include "name", "date" and "relevance". See example under "order".
+sort             string   The sort field. Supported values include "name", "date", and "score". Sorting by "score" orders by **relevance** and is the default if this parameter is omitted.
 order            string   The order in which to sort. Can either be "asc" or "desc".  For example, https://demo.dataverse.org/api/search?q=data&sort=name&order=asc
 per_page         int      The number of results to return per request. The default is 10. The max is 1000. See :ref:`iteration example <iteration-example>`.
 start            int      A cursor for paging through search results. See :ref:`iteration example <iteration-example>`.

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -67,15 +67,15 @@
  */
 @Table(indexes = {@Index(columnList="datafile_id"), @Index(columnList="datasetversion_id")} )
 @NamedNativeQuery(
-        name = "FileMetadata.compareFileMetadata",
+        name = "FileMetadata.getDatafilesWithChangedMetadata",
         query = "WITH fm_categories AS (" +
                 "    SELECT fmd.filemetadatas_id, " +
                 "           STRING_AGG(dfc.name, ',' ORDER BY dfc.name) AS categories " +
                 "    FROM FileMetadata_DataFileCategory fmd " +
                 "    JOIN DataFileCategory dfc ON fmd.filecategories_id = dfc.id " +
                 "    GROUP BY fmd.filemetadatas_id " +
                 ") " +
-                "SELECT fm1.id " +
+                "SELECT fm1.datafile_id AS id " +
                 "FROM FileMetadata fm1 " +
                 "LEFT JOIN FileMetadata fm2 ON fm1.datafile_id = fm2.datafile_id " +
                 "    AND fm2.datasetversion_id = ?1 " +
@@ -93,11 +93,11 @@
                 "                 ) " +
                 "            ) " +
                 "        )",
-                resultSetMapping = "IdToLongMapping"
+                resultSetMapping = "IdToIntegerMapping"
     )
 /* When this mapping was to Long.class, Postgres was still returning an Integer, causing indexing failures - see #11776 */ 
 @SqlResultSetMapping(
-        name = "IdToLongMapping",
+        name = "IdToIntegerMapping",
         columns = @ColumnResult(name = "id", type = Integer.class)
     )
 @Entity

src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java

@@ -618,7 +618,7 @@ private void replaceFileDataTagsInFile(DataFile df) throws DataFileTagException{
         // --------------------------------------------------
         // Is this a tabular file?
         // --------------------------------------------------
-        if (!df.isTabularData()){
+        if (!df.isTabularData() && !getDataFileTags().isEmpty()){
             String errMsg = BundleUtil.getStringFromBundle("file.metadata.datafiletag.not_tabular");
 
             throw new DataFileTagException(errMsg);

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -602,10 +602,24 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr
                 writeDebugInfo(debug, dataset);
             }
             if (doNormalSolrDocCleanUp) {
+                List<String> solrIdsOfPermissionDocsToDelete = new ArrayList<>();
                 try {
                     solrIdsOfDocsToDelete = findFilesOfParentDataset(dataset.getId());
                     logger.fine("Existing file docs: " + String.join(", ", solrIdsOfDocsToDelete));
                     if (!solrIdsOfDocsToDelete.isEmpty()) {
+                        if (!latestVersion.isDraft()) {
+                            // After publication, we need to delete old draft perm docs
+                            // For the first draft, a perm doc will exist for each file
+                            // For subsequent drafts, perm docs should only exist for new files/those with changed metadata
+                            // This code adds the ids of draft perm docs for all files - if the docs don't exist, Solr will just ignore them
+                            for (String fileDocId : solrIdsOfDocsToDelete) {
+                                if (!fileDocId.endsWith(draftSuffix)) {
+                                    solrIdsOfPermissionDocsToDelete.add(fileDocId + draftSuffix + discoverabilityPermissionSuffix);
+                                }
+                            }
+
+                            logger.fine("Existing permission docs: " + String.join(", ", solrIdsOfPermissionDocsToDelete));
+                        }
                         // We keep the latest version's docs unless it is deaccessioned and there is no
                         // published/released version
                         // So skip the loop removing those docs from the delete list except in that case
@@ -649,7 +663,7 @@ private void doIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) thr
                 logger.fine("Solr docs to delete: " + String.join(", ", solrIdsOfDocsToDelete));
 
                 if (!solrIdsOfDocsToDelete.isEmpty()) {
-                    List<String> solrIdsOfPermissionDocsToDelete = new ArrayList<>();
+
                     for (String file : solrIdsOfDocsToDelete) {
                         // Also remove associated permission docs
                         solrIdsOfPermissionDocsToDelete.add(file + discoverabilityPermissionSuffix);
@@ -1416,7 +1430,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
         long maxSize = maxFTIndexingSize != null ? maxFTIndexingSize.longValue() : Long.MAX_VALUE;
 
         List<String> filesIndexed = new ArrayList<>();
-        final List<Long> changedFileMetadataIds = new ArrayList<>();
+        final List<Long> changedFileIds = new ArrayList<>();
         if (datasetVersion != null) {
             List<FileMetadata> fileMetadatas = datasetVersion.getFileMetadatas();
             List<FileMetadata> rfm = new ArrayList<>();
@@ -1427,42 +1441,17 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                     fileMap.put(released.getDataFile().getId(), released);
                 }
 
-                Query query = em.createNamedQuery("FileMetadata.compareFileMetadata", Long.class);
-                query.setParameter(1, dataset.getReleasedVersion().getId());
-                query.setParameter(2, datasetVersion.getId());
-
-                /*
-                 * When the query was configured to return Long, it was returning Integer. The query has been changed to return Integer now. The code here is robust if that changes in the future.
-                 */
-                List<Object> queryResults = query.getResultList();
-                for (Object result : queryResults) {
-                    if (result != null) {
-                        // Ensure we're adding Long objects to the list
-                        if (result instanceof Integer intResult) {
-                            logger.finest("Converted Integer result to Long: " + result);
-                            changedFileMetadataIds.add(Long.valueOf(intResult));
-                        } else if (result instanceof Long longResult) {
-                            // Already a Long, add directly
-                            logger.finest("Added existing Long to list: " + result);
-                            changedFileMetadataIds.add(longResult);
-                        } else {
-                            // If it's not a Long, convert it to one via String
-                            try {
-                                changedFileMetadataIds.add(Long.valueOf(result.toString()));
-                                logger.finest("Converted non-Long result to Long: " + result + " of type " + result.getClass().getName());
-                            } catch (NumberFormatException e) {
-                                logger.warning("Could not convert query result to Long: " + result);
-                            }
-                        }
-                    }
-                }
+                solrIndexService.populateChangedFileIds(
+                        dataset.getReleasedVersion().getId(),
+                        datasetVersion.getId(),
+                        changedFileIds);
                 logger.fine(
                         "We are indexing a draft version of a dataset that has a released version. We'll be checking file metadatas if they are exact clones of the released versions.");
             } else if (datasetVersion.isDraft()) {
                 // Add all file metadata ids to changedFileMetadataIds
-                changedFileMetadataIds.addAll(
+                changedFileIds.addAll(
                     fileMetadatas.stream()
-                        .map(FileMetadata::getId)
+                        .map(fm -> fm.getDataFile().getId())
                         .collect(Collectors.toList())
                 );
             }
@@ -1526,7 +1515,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                 }
                 boolean indexThisFile = false;
 
-                if (indexThisMetadata && (isReleasedVersion || changedFileMetadataIds.contains(fileMetadata.getId()))) {
+                if (indexThisMetadata && (isReleasedVersion || changedFileIds.contains(datafile.getId()))) {
                     indexThisFile = true;
                 } else if (indexThisMetadata) {
                     // Draft version, file is not new or all file metadata matches the released version

src/main/java/edu/harvard/iq/dataverse/search/SearchPermissionsServiceBean.java

@@ -92,52 +92,6 @@ public List<String> findDvObjectPerms(DvObject dvObject) {
         return permStrings;
     }
 
-    public Map<DatasetVersion.VersionState, Boolean> getDesiredCards(Dataset dataset) {
-        Map<DatasetVersion.VersionState, Boolean> desiredCards = new LinkedHashMap<>();
-        DatasetVersion latestVersion = dataset.getLatestVersion();
-        DatasetVersion.VersionState latestVersionState = latestVersion.getVersionState();
-        DatasetVersion releasedVersion = dataset.getReleasedVersion();
-        boolean atLeastOnePublishedVersion = false;
-        if (releasedVersion != null) {
-            atLeastOnePublishedVersion = true;
-        } else {
-            atLeastOnePublishedVersion = false;
-        }
-
-        if (atLeastOnePublishedVersion == false) {
-            if (latestVersionState.equals(DatasetVersion.VersionState.DRAFT)) {
-                desiredCards.put(DatasetVersion.VersionState.DRAFT, true);
-                desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false);
-                desiredCards.put(DatasetVersion.VersionState.RELEASED, false);
-            } else if (latestVersionState.equals(DatasetVersion.VersionState.DEACCESSIONED)) {
-                desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, true);
-                desiredCards.put(DatasetVersion.VersionState.RELEASED, false);
-                desiredCards.put(DatasetVersion.VersionState.DRAFT, false);
-            } else {
-                String msg = "No-op. Unexpected condition reached: There is no published version and the latest published version is neither " + DatasetVersion.VersionState.DRAFT + " nor " + DatasetVersion.VersionState.DEACCESSIONED + ". Its state is " + latestVersionState + ".";
-                logger.info(msg);
-            }
-        } else if (atLeastOnePublishedVersion == true) {
-            if (latestVersionState.equals(DatasetVersion.VersionState.RELEASED)
-                    || latestVersionState.equals(DatasetVersion.VersionState.DEACCESSIONED)) {
-                desiredCards.put(DatasetVersion.VersionState.RELEASED, true);
-                desiredCards.put(DatasetVersion.VersionState.DRAFT, false);
-                desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false);
-            } else if (latestVersionState.equals(DatasetVersion.VersionState.DRAFT)) {
-                desiredCards.put(DatasetVersion.VersionState.DRAFT, true);
-                desiredCards.put(DatasetVersion.VersionState.RELEASED, true);
-                desiredCards.put(DatasetVersion.VersionState.DEACCESSIONED, false);
-            } else {
-                String msg = "No-op. Unexpected condition reached: There is at least one published version but the latest version is neither published nor draft";
-                logger.info(msg);
-            }
-        } else {
-            String msg = "No-op. Unexpected condition reached: Has a version been published or not?";
-            logger.info(msg);
-        }
-        return desiredCards;
-    }
-
     private boolean hasBeenPublished(Dataverse dataverse) {
         return dataverse.isReleased();
     }