Merge pull request #11247 from IQSS/11188-version-difference-summary

11188 API to get list of dataset versions with summary of differences

Author: ofahimIQSS

doc/release-notes/11188-dataset-version-api.md

@@ -0,0 +1,4 @@
+### Retreiving a list of Dataset Versions
+
+An API has been added to get dataset versions including a summary of diiferences between consecutive versions where available. See [the docs](https://guides.dataverse.org/en/6.6/api/native-api.html#get-versions-of-a-dataset-with-summary-of-changes ),  and #10888.
+

doc/sphinx-guides/source/api/native-api.rst

@@ -2002,6 +2002,27 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/:latest-published/compare/:draft"
 
+Get Versions of a Dataset with Summary of Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Returns a list of versions for a given dataset including a summary of differences between consecutive versions where available. Draft versions will only
+be available to users who have permission to view unpublished drafts. The api token is optional.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
+
+  curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/versions/compareSummary?persistentId=$PERSISTENT_IDENTIFIER" 
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/versions/compareSummary?persistentId=doi:10.5072/FK2/BCCP9Z" 
+
+
 Update Metadata For a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java

@@ -46,8 +46,8 @@ public final class DatasetVersionDifference {
     private List<FileMetadata> changedVariableMetadata = new ArrayList<>();
     private List<FileMetadata[]> replacedFiles = new ArrayList<>();
     private List<String[]> changedTermsAccess = new ArrayList<>();
-    private List<Object[]> summaryDataForNote = new ArrayList<>();
-    private List<Object[]> blockDataForNote = new ArrayList<>();
+    private List<SummaryNote> summaryDataForNote = new ArrayList<>();
+    private List<SummaryNote> blockDataForNote = new ArrayList<>();
 
     private List<DifferenceSummaryGroup> differenceSummaryGroups = new ArrayList<>();
 
@@ -300,32 +300,33 @@ private void addToSummary(DatasetField dsfo, DatasetField dsfn) {
 
     private void updateBlockSummary(DatasetField dsf, int added, int deleted, int changed) {
         boolean addedToAll = false;
-        for (Object[] blockList : blockDataForNote) {
-            DatasetField dsft = (DatasetField) blockList[0];
+        for (SummaryNote blockList : blockDataForNote) {
+            
+            DatasetField dsft = blockList.dsfo;
             if (dsft.getDatasetFieldType().getMetadataBlock().equals(dsf.getDatasetFieldType().getMetadataBlock())) {
-                blockList[1] = (Integer) blockList[1] + added;
-                blockList[2] = (Integer) blockList[2] + deleted;
-                blockList[3] = (Integer) blockList[3] + changed;
+                blockList.added = blockList.added + added;
+                blockList.deleted = blockList.deleted + deleted;
+                blockList.changed = blockList.changed + changed;
                 addedToAll = true;
             }
         }
         if (!addedToAll) {
-            Object[] newArray = new Object[4];
-            newArray[0] = dsf;
-            newArray[1] = added;
-            newArray[2] = deleted;
-            newArray[3] = changed;
-            blockDataForNote.add(newArray);
+            SummaryNote newNote = new SummaryNote();
+            newNote.dsfo = dsf;
+            newNote.added = added;
+            newNote.deleted = deleted;
+            newNote.changed = changed;
+            blockDataForNote.add(newNote);
         }
     }
 
-    private void addToNoteSummary(DatasetField dsfo, int added, int deleted, int changed) {
-        Object[] noteArray = new Object[4];
-        noteArray[0] = dsfo;
-        noteArray[1] = added;
-        noteArray[2] = deleted;
-        noteArray[3] = changed;
-        summaryDataForNote.add(noteArray);
+    private void addToNoteSummary(DatasetField dsfo, Integer added, Integer deleted, Integer changed) {
+        SummaryNote summaryNote = new SummaryNote();
+        summaryNote.dsfo = dsfo;
+        summaryNote.added = added;
+        summaryNote.deleted = deleted;
+        summaryNote.changed = changed;
+        summaryDataForNote.add(summaryNote);
     }
 
     static boolean compareVarGroup(FileMetadata fmdo, FileMetadata fmdn) {
@@ -363,14 +364,19 @@ public static Map<String,List<String>> compareFileMetadatas(FileMetadata fmdo, F
                     List.of(StringUtil.nullToEmpty(fmdo.getDescription()), StringUtil.nullToEmpty(fmdn.getDescription())));
         }
 
-        if (!StringUtils.equals(fmdo.getCategoriesByName().toString(), fmdn.getCategoriesByName().toString())) {
+        if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getCategoriesByName().toString()), StringUtil.nullToEmpty(fmdn.getCategoriesByName().toString()))) {
             fileMetadataChanged.put("Categories",
                     List.of(fmdo.getCategoriesByName().toString(), fmdn.getCategoriesByName().toString()));
         }
 
-        if (!StringUtils.equals(fmdo.getLabel(), fmdn.getLabel())) {
+        if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getLabel()), StringUtil.nullToEmpty(fmdn.getLabel()))) {
             fileMetadataChanged.put("Label",
-                    List.of(fmdo.getLabel(), fmdn.getLabel()));
+                    List.of(StringUtil.nullToEmpty(fmdo.getLabel()), StringUtil.nullToEmpty(fmdn.getLabel())));
+        }
+        
+        if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getDirectoryLabel()), StringUtil.nullToEmpty(fmdn.getDirectoryLabel()))) {
+            fileMetadataChanged.put("File Path",
+                    List.of(StringUtil.nullToEmpty(fmdo.getDirectoryLabel()), StringUtil.nullToEmpty(fmdn.getDirectoryLabel())));
         }
 
         if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getProvFreeForm()), StringUtil.nullToEmpty(fmdn.getProvFreeForm()))) {
@@ -464,6 +470,8 @@ private void compareValues(DatasetField originalField, DatasetField newField, bo
             }
         }
     }
+    
+    
 
     public String getFileNote() {
         String retString = "";
@@ -568,19 +576,19 @@ public void setChangedFileMetadata(List<FileMetadata> changedFileMetadata) {
         this.changedFileMetadata = changedFileMetadata;
     }
 
-    public List<Object[]> getSummaryDataForNote() {
+    public List<SummaryNote> getSummaryDataForNote() {
         return summaryDataForNote;
     }
 
-    public List<Object[]> getBlockDataForNote() {
+    public List<SummaryNote> getBlockDataForNote() {
         return blockDataForNote;
     }
 
-    public void setSummaryDataForNote(List<Object[]> summaryDataForNote) {
+    public void setSummaryDataForNote(List<SummaryNote> summaryDataForNote) {
         this.summaryDataForNote = summaryDataForNote;
     }
 
-    public void setBlockDataForNote(List<Object[]> blockDataForNote) {
+    public void setBlockDataForNote(List<SummaryNote> blockDataForNote) {
         this.blockDataForNote = blockDataForNote;
     }
 
@@ -1205,6 +1213,47 @@ public List<DifferenceSummaryItem> getDifferenceSummaryItems() {
         public void setDifferenceSummaryItems(List<DifferenceSummaryItem> differenceSummaryItems) {
             this.differenceSummaryItems = differenceSummaryItems;
         }
+    }
+    
+        public class SummaryNote  {
+         DatasetField dsfo;
+         Integer added;
+         Integer deleted; 
+         Integer changed; 
+         
+         public void setDatasetField(DatasetField dsfIn){
+             dsfo = dsfIn;
+         }
+         
+         public DatasetField getDatasetField (){
+             return dsfo;
+         }
+         
+         public void setAdded(Integer addin){
+             added = addin;
+         }
+         
+         public Integer getAdded(){
+             return added;
+         }
+         
+        public void setDeleted(Integer delin){
+             deleted = delin;
+         }
+         
+         public Integer getDeleted(){
+             return deleted;
+        }
+        
+        public void setChanged(Integer changedin){
+             changed = changedin;
+         }
+         
+         public Integer getChanged(){
+             return changed;
+        } 
+         
+         
     }
 
     public class DifferenceSummaryItem {
@@ -1641,6 +1690,93 @@ List<FileMetadata> getChangedVariableMetadata() {
     List<FileMetadata[]> getReplacedFiles() {
         return replacedFiles;
     }
+    
+    public JsonObjectBuilder getSummaryDifferenceAsJson(){
+
+        JsonObjectBuilder jobVersion = new NullSafeJsonBuilder();
+
+        JsonObjectBuilder jobDsfOnCreate = new NullSafeJsonBuilder();
+        
+        
+        for (SummaryNote sn : this.summaryDataForNote) {
+            jobDsfOnCreate.add( sn.getDatasetField().getDatasetFieldType().getDisplayName(), getSummaryNoteAsJson(sn));
+        }
+        
+        if (!this.summaryDataForNote.isEmpty()){
+            
+            jobVersion.add("Citation Metadata", jobDsfOnCreate);
+            
+        }
+        
+        for (SummaryNote sn : this.getBlockDataForNote()){
+             String mdbDisplayName = sn.getDatasetField().getDatasetFieldType().getMetadataBlock().getDisplayName();
+             if (mdbDisplayName.equals("Citation Metadata")){
+                 mdbDisplayName = "Additional Citation Metadata";
+             }
+             jobVersion.add( mdbDisplayName, getSummaryNoteAsJson(sn));    
+        }   
+        
+        jobVersion.add("files", getFileSummaryAsJson());
+        
+        if (!this.changedTermsAccess.isEmpty()) {
+            jobVersion.add("termsAccessChanged", true);
+        } else{
+            jobVersion.add("termsAccessChanged", false);
+        }      
+                
+        return jobVersion;
+    }
+    
+    private JsonObjectBuilder getSummaryNoteAsJson(SummaryNote sn){
+        JsonObjectBuilder job = new NullSafeJsonBuilder();
+        //job.add("datasetFieldType", sn.getDatasetField().getDatasetFieldType().getDisplayName());
+        job.add("added", sn.added);
+        job.add("deleted", sn.deleted);
+        job.add("changed", sn.changed);
+        return job;
+    }
+    
+    private JsonObjectBuilder getFileSummaryAsJson(){
+        JsonObjectBuilder job = new NullSafeJsonBuilder();
+        
+        if (!addedFiles.isEmpty()) {
+            job.add("added", addedFiles.size());
+           
+        } else{
+            job.add("added", 0);
+        }
+        
+        if (!removedFiles.isEmpty()) {
+            job.add("removed", removedFiles.size());
+           
+        } else{
+            job.add("removed", 0);
+        }
+        
+        if (!replacedFiles.isEmpty()) {
+            job.add("replaced", replacedFiles.size());
+           
+        } else{
+            job.add("replaced", 0);
+        }
+        
+        if (!changedFileMetadata.isEmpty()) {
+            job.add("changedFileMetaData", changedFileMetadata.size());
+           
+        } else{
+            job.add("changedFileMetaData", 0);
+        }
+        
+        if (!changedVariableMetadata.isEmpty()) {
+            job.add("changedVariableMetadata", changedVariableMetadata.size());
+           
+        } else{
+            job.add("changedVariableMetadata", 0);
+        }
+
+        return job;
+    }
+       
     public JsonObjectBuilder compareVersionsAsJson() {
         JsonObjectBuilder job = new NullSafeJsonBuilder();
         JsonObjectBuilder jobVersion = new NullSafeJsonBuilder();

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -3,6 +3,7 @@
 import com.amazonaws.services.s3.model.PartETag;
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.DatasetLock.Reason;
+import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
 import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
 import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO;
@@ -3047,6 +3048,62 @@ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathPa
             return wr.getResponse();
         }
     }
+    
+    @GET
+    @AuthRequired
+    @Path("{id}/versions/compareSummary")
+    public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, @PathParam("id") String id,
+                                      @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        try {
+            Dataset dataset = findDatasetOrDie(id);
+            User user = getRequestUser(crc);
+            JsonArrayBuilder differenceSummaries = Json.createArrayBuilder();
+
+            for (DatasetVersion dv : dataset.getVersions()) {
+                //only get summaries of draft is user may view unpublished
+
+                if (dv.isPublished() || permissionService.hasPermissionsFor(user, dv.getDataset(),
+                        EnumSet.of(Permission.ViewUnpublishedDataset))) {
+
+                    JsonObjectBuilder versionBuilder = new NullSafeJsonBuilder();
+                    versionBuilder.add("id", dv.getId());
+                    versionBuilder.add("versionNumber", dv.getFriendlyVersionNumber());
+                    DatasetVersionDifference dvdiff = dv.getDefaultVersionDifference();
+                    if (dvdiff == null) {
+                        if (dv.isReleased()) {
+                            if (dv.getPriorVersionState() == null) {
+                                versionBuilder.add("summary", "firstPublished");
+                            }
+                            if (dv.getPriorVersionState() != null && dv.getPriorVersionState().equals(VersionState.DEACCESSIONED)) {
+                                versionBuilder.add("summary", "previousVersionDeaccessioned");
+                            }
+                        }
+                        if (dv.isDraft()) {
+                            if (dv.getPriorVersionState() == null) {
+                                versionBuilder.add("summary", "firstDraft");
+                            }
+                            if (dv.getPriorVersionState() != null && dv.getPriorVersionState().equals(VersionState.DEACCESSIONED)) {
+                                versionBuilder.add("summary", "previousVersionDeaccessioned");
+                            }
+                        }
+                        if (dv.isDeaccessioned()) {
+                            versionBuilder.add("summary", "versionDeaccessioned");
+                        }
+
+                    } else {
+                        versionBuilder.add("summary", dvdiff.getSummaryDifferenceAsJson());
+                    }
+
+                    versionBuilder.add("contributors", datasetversionService.getContributorsNames(dv));
+                    versionBuilder.add("publishedOn", !dv.isDraft() ? dv.getPublicationDateAsString() : "");
+                    differenceSummaries.add(versionBuilder);
+                }
+            }
+            return ok(differenceSummaries);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+    }
 
     private static Set<String> getDatasetFilenames(Dataset dataset) {
         Set<String> files = new HashSet<>();