Merge remote-tracking branch 'IQSS/develop' into AWSv2

Author: qqmyers

conf/solr/schema.xml

@@ -157,7 +157,8 @@
     <field name="dvSubject" type="string" stored="true" indexed="true" multiValued="true"/>
 
     <field name="publicationStatus" type="string" stored="true" indexed="true" multiValued="true"/>
-    <field name="externalStatus" type="string" stored="true" indexed="true" multiValued="false"/>
+    <field name="curationStatus" type="string" stored="true" indexed="true" multiValued="false"/>
+    <field name="curationStatusCreateTime" type="pdate" indexed="true" stored="true"/>
     <field name="embargoEndDate" type="plong" stored="true" indexed="true" multiValued="false"/>
     <field name="retentionEndDate" type="plong" stored="true" indexed="true" multiValued="false"/>
 

doc/release-notes/9247-CurationStatusUpdates.md

@@ -0,0 +1,14 @@
+The External/Curation Status Label mechanism has been enhanced:
+
+- adding tracking of who creates the status label and when,
+- keeping a history of past statuses
+- updating the CSV report to include the creation time and assigner of a status
+- updating the getCurationStatus api call to return a JSON object for the status with label, assigner, and create time
+- adding an includeHistory query param for these API calls to allow seeing prior statuses
+- adding a facet to allow filtering by curation status (for users able to set them)
+- adding the creation time to solr as a pdate to support search by time period, e.g. current status set prior to a give date
+- standardizing the language around 'curation status' vs 'external status'
+- adding a 'curation-status' class to displayed labels to allow styling
+- adding a dataverse.ui.show-curation-status-to-all feature flag that allows users who can see a draft but not publish it to also view the curation status
+
+Due to changes in the solr schema, updating the solr schema and reindexing is required. Background reindexing should be OK.

doc/sphinx-guides/source/api/changelog.rst

@@ -11,6 +11,8 @@ v6.7
 ----
 
 - An undocumented :doc:`search` parameter called "show_my_data" has been removed. It was never exercised by tests and is believed to be unused. API users should use the :ref:`api-mydata` API instead.
+- /api/datasets/{id}/curationStatus API now includes a JSON object with curation label, createtime, and assigner rather than a string 'label' and it supports a new boolean includeHistory parameter (default false) that returns a JSON array of statuses
+- /api/datasets/{id}/listCurationStates includes new columns "Status Set Time" and "Status Set By" columns listing the time the current status was applied and by whom. It also supports the boolean includeHistory parameter. 
 - Due to updates in libraries used by Dataverse, XML serialization may have changed slightly with respect to whether self-closing tags are used for empty elements. This primiarily affects XML-based metadata exports. The XML structure of the export itself has not changed, so this is only an incompatibility if you are not using an XML parser.
 
 v6.6

doc/sphinx-guides/source/api/curation-labels.rst

@@ -1,16 +1,19 @@
-Dataset Curation Label API
-==========================
+Dataset Curation Status API
+===========================
 
 When the :ref:`:AllowedCurationLabels <:AllowedCurationLabels>` setting has been used to define Curation Labels, this API can be used to set these labels on draft datasets. 
 Superusers can define which set of labels are allowed for a given datasets in a collection/an individual dataset using the api described in the :doc:`/admin/dataverses-datasets` section.
 The API here can be used by curators/those who have permission to publish the dataset to get/set/change/delete the label currently assigned to a draft dataset.
+If the :ref:`dataverse.ui.show-curation-status-to-all` flag is enabled, users who can see the draft dataset version can use the get API call.
 
 This functionality is intended as a mechanism to integrate the Dataverse software with an external curation process/application: it is a way to make the state of a draft dataset, 
 as defined in the external process, visible within Dataverse. These labels have no other effect in Dataverse and are only visible to curators/those with permission to publish the dataset. 
 Any curation label assigned to a draft dataset will be removed upon publication.
+
+Dataverse tracks the Curation Label as well as when it was assigned and by whom. It also keeps track of the history of prior assignments.
 
-Get a Draft Dataset's Curation Label
-------------------------------------
+Get a Draft Dataset's Curation Status
+-------------------------------------
 
 .. code-block:: bash
 
@@ -27,8 +30,13 @@ Get a Draft Dataset's Curation Label
 
     curl -H X-Dataverse-key:$API_TOKEN "$SERVER_URL/api/datasets/:persistentId/curationStatus?persistentId=$DATASET_PID"
 
-You should expect a 200 ("OK") response and the draft dataset's curation status label contained in a JSON 'data' object.
+You should expect a 200 ("OK") response and the draft dataset's curation status as a JSON object contained in a JSON 'data' object. The status will include a 'label','createTime', and the 'assigner'.
+
+If the optional includeHistory query parameter is set to true, the responses 'data' entry will be a JSON array of curation status objects 
+
+    curl -H X-Dataverse-key:$API_TOKEN "$SERVER_URL/api/datasets/:persistentId/curationStatus?persistentId=$DATASET_PID&includeHistory=true"
 
+For draft datasets that were created prior to v6.7, it is possible that curation status objects will have no createTime or assigner.
 
 Set a Draft Dataset's Curation Label
 ------------------------------------
@@ -53,6 +61,8 @@ To add a curation label for a draft Dataset, specify the Dataset ID (DATASET_ID)
 
 You should expect a 200 ("OK") response indicating that the label has been set. 403/Forbidden and 400/Bad Request responses are also possible, i.e. if you don't have permission to make this change or are trying to add a label that isn't in the allowed set or to add a label to a dataset with no draft version.
 
+Note that Dataverse will add the current time as the createTime and the user as the 'assigner' of the label.
+
 
 Delete a Draft Dataset's Curation Label
 ---------------------------------------
@@ -98,7 +108,7 @@ You should expect a 200 ("OK") response with a comma-separated list of allowed l
 Get a Report on the Curation Status of All Datasets
 ---------------------------------------------------
 
-To get a CSV file listing the curation label assigned to each Dataset with a draft version, along with the creation and last modification dates, and list of those with permissions to publish the version.
+To get a CSV file listing the curation statuses assigned to each Dataset with a draft version, along with the creation and last modification dates, and list of those with permissions to publish the version.
 
 This API call is restricted to superusers.
 
@@ -112,3 +122,14 @@ This API call is restricted to superusers.
     curl -H X-Dataverse-key:$API_TOKEN "$SERVER_URL/api/datasets/listCurationStates"
 
 You should expect a 200 ("OK") response with a CSV formatted response.
+
+The CSV response includes the following columns in order:
+#. Dataset Title (as a hyperlink to the dataset page)
+#. Creation Date of the draft dataset version
+#. Latest Modification Date of the draft dataset version
+#. Assigned curation status or '<none>' if no curation status is assigned but was previously, null if no curation state has every been set.
+#. Time when the curation status was applied to the draft dataset version
+#. The user who assigned this curation status
+#. (and beyond): Users (comma separated list) with the Roles (column headings) that can publish datasets and therefore see/set curation status
+When includeHistory is true, multiple rows may be present for each dataset, showing the full history of curation statuses.
+

doc/sphinx-guides/source/installation/config.rst

@@ -3269,6 +3269,21 @@ Defaults to ``true``.
 Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable
 ``DATAVERSE_API_SHOW_LABEL_FOR_INCOMPLETE_WHEN_PUBLISHED``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions.
 
+.. _dataverse.ui.show-curation-status-to-all:
+
+dataverse.ui.show-curation-status-to-all
+++++++++++++++++++++++++++++++++++++++++
+
+By default the curation status assigned to a draft dataset versioncan only be seen by those who can publish it. When this flag is true, anyone who can see the draft dataset can see the assigned status.
+These users will also get notifications/emails about changes to the status.
+See :ref:`:AllowedCurationLabels <:AllowedCurationLabels>` and the :doc:`/admin/dataverses-datasets` section for more information about curation status.
+
+Defaults to ``false``.
+
+Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable
+``DATAVERSE_API_SHOW_CURATION_STATUS_TO_ALL``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions.
+
+
 .. _dataverse.signposting.level1-author-limit:
 
 dataverse.signposting.level1-author-limit

src/main/java/edu/harvard/iq/dataverse/CurationStatus.java

@@ -0,0 +1,91 @@
+package edu.harvard.iq.dataverse;
+
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import jakarta.persistence.*;
+
+import java.io.Serializable;
+import java.util.Date;
+
+@Entity
+@Table(name = "curationstatus", indexes = {
+        @Index(name = "index_curationstatus_datasetversion", columnList = "datasetversion_id")
+    })
+public class CurationStatus implements Serializable {
+    
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(nullable = true)
+    private String label;
+
+    @ManyToOne
+    @JoinColumn(name = "datasetversion_id", nullable = false)
+    private DatasetVersion datasetVersion;
+
+    @ManyToOne
+    @JoinColumn(name = "authenticateduser_id", nullable = true)
+    private AuthenticatedUser authenticatedUser;
+
+    @Temporal(TemporalType.TIMESTAMP)
+    @Column(nullable = true)
+    private Date createTime;
+
+    // Constructors, getters, and setters
+
+    public CurationStatus() {
+    }
+
+    public CurationStatus(String label, DatasetVersion datasetVersion, AuthenticatedUser authenticatedUser) {
+        this.label = label;
+        this.datasetVersion = datasetVersion;
+        this.authenticatedUser = authenticatedUser;
+        this.createTime = new Date();
+    }
+
+    // Getters and setters for all fields
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getLabel() {
+        return label;
+    }
+
+    public void setLabel(String label) {
+        this.label = label;
+    }
+
+    public DatasetVersion getDatasetVersion() {
+        return datasetVersion;
+    }
+
+    public void setDatasetVersion(DatasetVersion datasetVersion) {
+        this.datasetVersion = datasetVersion;
+    }
+
+    public AuthenticatedUser getAuthenticatedUser() {
+        return authenticatedUser;
+    }
+
+    public void setAuthenticatedUser(AuthenticatedUser authenticatedUser) {
+        this.authenticatedUser = authenticatedUser;
+    }
+
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(Date createTime) {
+        this.createTime = createTime;
+    }
+    
+    public boolean isNoStatus() {
+        return label == null || label.trim().isEmpty();
+    }
+}

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -143,6 +143,7 @@
 import jakarta.servlet.http.HttpServletResponse;
 
 import org.apache.commons.text.StringEscapeUtils;
+import org.apache.logging.log4j.util.Strings;
 import org.apache.commons.lang3.mutable.MutableBoolean;
 import org.apache.commons.io.IOUtils;
 import org.primefaces.component.selectonemenu.SelectOneMenu;
@@ -1477,6 +1478,15 @@ public boolean canViewUnpublishedDataset() {
         return permissionsWrapper.canViewUnpublishedDataset( dvRequestService.getDataverseRequest(), dataset);
     }
 
+    public boolean canSeeCurationStatus() {
+        boolean creatorsCanSeeStatus = JvmSettings.UI_SHOW_CURATION_STATUS_TO_ALL.lookupOptional(Boolean.class).orElse(false);
+        if (creatorsCanSeeStatus) {
+            return canViewUnpublishedDataset();
+        } else {
+            return canPublishDataset();
+        }
+    }
+
     /*
      * 4.2.1 optimization.
      * HOWEVER, this doesn't appear to be saving us anything!
@@ -6278,28 +6288,28 @@ public String getFieldLanguage(String languages) {
         return fieldService.getFieldLanguage(languages,session.getLocaleCode());
     }
 
-    public void setExternalStatus(String status) {
+    public void setCurationStatus(String status) {
         try {
             dataset = commandEngine.submit(new SetCurationStatusCommand(dvRequestService.getDataverseRequest(), dataset, status));
             workingVersion=dataset.getLatestVersion();
-            if (status == null || status.isEmpty()) {
-                JsfHelper.addInfoMessage(BundleUtil.getStringFromBundle("dataset.externalstatus.removed"));
+            if (Strings.isBlank(status)) {
+                JsfHelper.addInfoMessage(BundleUtil.getStringFromBundle("dataset.curationstatus.removed"));
             } else {
-                JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.externalstatus.header"),
-                        BundleUtil.getStringFromBundle("dataset.externalstatus.info",
-                                Arrays.asList(DatasetUtil.getLocaleExternalStatus(status))
+                JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.curationstatus.header"),
+                        BundleUtil.getStringFromBundle("dataset.curationstatus.info",
+                                Arrays.asList(DatasetUtil.getLocaleCurationStatusLabelFromString(status))
                         ));
             }
 
         } catch (CommandException ex) {
-            String msg = BundleUtil.getStringFromBundle("dataset.externalstatus.cantchange");
+            String msg = BundleUtil.getStringFromBundle("dataset.curationstatus.cantchange");
             logger.warning("Unable to change external status to " + status + " for dataset id " + dataset.getId() + ". Message to user: " + msg + " Exception: " + ex);
             JsfHelper.addErrorMessage(msg);
         }
     }
 
-    public List<String> getAllowedExternalStatuses() {
-        return settingsWrapper.getAllowedExternalStatuses(dataset);
+    public List<String> getAllowedCurationStatuses() {
+        return settingsWrapper.getAllowedCurationStatuses(dataset);
     }
 
     public Embargo getSelectionEmbargo() {

src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java

@@ -216,9 +216,10 @@ public enum VersionState {
     @OneToMany(mappedBy = "datasetVersion", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
     private List<WorkflowComment> workflowComments;
 
-    @Column(nullable=true)
-    private String externalStatusLabel;
-    
+    @OneToMany(mappedBy = "datasetVersion", cascade = CascadeType.ALL, orphanRemoval = true)
+    @OrderBy("createTime DESC NULLS LAST")
+    private List<CurationStatus> curationStatuses = new ArrayList<>();
+
     @Transient
     private DatasetVersionDifference dvd;
 
@@ -2156,12 +2157,44 @@ public String getLocaleLastUpdateTime() {
         return DateUtil.formatDate(new Timestamp(lastUpdateTime.getTime()));
     }
 
-    public String getExternalStatusLabel() {
-        return externalStatusLabel;
+    // Add methods to manage curationLabels
+    public List<CurationStatus> getCurationStatuses() {
+        return curationStatuses;
+    }
+
+    protected void setCurationStatuses(List<CurationStatus> curationStatuses) {
+        this.curationStatuses = curationStatuses;
+    }
+
+    public CurationStatus getCurrentCurationStatus() {
+        return !getCurationStatuses().isEmpty() ? getCurationStatuses().get(0) : null;
+    }
+
+    
+    public void addCurationStatus(CurationStatus status) {
+        status.setDatasetVersion(this);
+        curationStatuses.add(0, status); // Add the new status at the beginning of the list
     }
 
-    public void setExternalStatusLabel(String externalStatusLabel) {
-        this.externalStatusLabel = externalStatusLabel;
+    public void removeCurationStatus(CurationStatus curationStatus) {
+        curationStatuses.remove(curationStatus);
+        curationStatus.setDatasetVersion(null);
+    }
+
+    public CurationStatus getCurationStatusAsOfDate(Date date) {
+        if (curationStatuses == null || curationStatuses.isEmpty()) {
+            return null;
+        }
+
+        // Find the first status whose createTime is before or equal to the given date
+        for (CurationStatus status : curationStatuses) {
+            if (status.getCreateTime().compareTo(date) <= 0) {
+                return status;
+            }
+        }
+
+        // If no status is found before the given date, return null
+        return null;
     }
 
     public String getVersionNote() {

src/main/java/edu/harvard/iq/dataverse/DataversePage.java

@@ -1241,7 +1241,7 @@ public Set<Entry<String, String>> getCurationLabelSetOptions() {
                 setNames.put(BundleUtil.getStringFromBundle("dataverse.curationLabels.disabled"), SystemConfig.CURATIONLABELSDISABLED);
 
                 allowedSetNames.forEach(name -> {
-                    String localizedName = DatasetUtil.getLocaleExternalStatus(name) ;
+                    String localizedName = DatasetUtil.getLocaleCurationStatusLabelFromString(name) ;
                     setNames.put(localizedName,name);
                 });
             }

src/main/java/edu/harvard/iq/dataverse/FilePage.java

@@ -345,7 +345,15 @@ private boolean canViewUnpublishedDataset() {
     public boolean canPublishDataset(){
         return permissionsWrapper.canIssuePublishDatasetCommand(fileMetadata.getDatasetVersion().getDataset());
     }
-   
+    
+    public boolean canSeeCurationStatus() {
+        boolean creatorsCanSeeStatus = JvmSettings.UI_SHOW_CURATION_STATUS_TO_ALL.lookupOptional(Boolean.class).orElse(false);
+        if (creatorsCanSeeStatus) {
+            return canViewUnpublishedDataset();
+        } else {
+            return canPublishDataset();
+        }
+    }
 
     public FileMetadata getFileMetadata() {
         return fileMetadata;