Merge pull request #11983 from GlobalDataverseCommunityConsortium/GDCC/8914-COAR-compliant_messaging2

GDCC/8914 COAR Messaging Improvement

Author: sekmiller

doc/release-notes/11983-COAR-Notify2.md

@@ -0,0 +1,3 @@
+### Improved COAR Notify Relationship Announcement Support
+
+Dataverse no longer sends duplicate [COAR Notify Relationship Announcement Workflow](https://coar-notify.net/catalogue/workflows/repository-relationship-repository/) messages when new dataset versions are published (and the relationship metadata has not been changed).

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -495,6 +495,26 @@ public DatasetVersion getReleasedVersion() {
         }
         return null;
     }
+
+    /**
+     * Returns the second-most-recent released version of this dataset.
+     * Assumes versions are ordered from most recent to oldest.
+     * 
+     * @return The prior released version, or null if there is only one or no released versions
+     */
+    public DatasetVersion getPriorReleasedVersion() {
+        boolean foundReleasedVersion = false;
+        for (DatasetVersion version : this.getVersions()) {
+            if (version.isReleased()) {
+                if(foundReleasedVersion) {
+                    return version;
+                } else {
+                    foundReleasedVersion = true;
+                }
+            }
+        }
+        return null;
+    }
 
     public DatasetVersion getVersionFromId(Long datasetVersionId) {
         for (DatasetVersion version : this.getVersions()) {

src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java

@@ -27,6 +27,7 @@
 import jakarta.persistence.Transient;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Strings;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 
@@ -254,4 +255,83 @@ private Map<DatasetField, String> removeLastComma(Map<DatasetField, String> mapI
 
         return mapIn;
     }
+    
+
+    /**
+     * Compares this DatasetFieldCompoundValue with another for equality based on
+     * their child fields. Two compound values are considered equal if they have the
+     * same child fields with the same values in the same order.
+     * 
+     * @param other The DatasetFieldCompoundValue to compare with
+     * @return true if both compound values have equal child fields, false otherwise
+     */
+    public boolean valuesEqual(DatasetFieldCompoundValue other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null) {
+            return false;
+        }
+
+        List<DatasetField> children1 = this.getChildDatasetFields();
+        List<DatasetField> children2 = other.getChildDatasetFields();
+
+        if (children1.size() != children2.size()) {
+            return false;
+        }
+        
+        // Compare each child field
+        for (DatasetField child1 : children1) {
+
+            DatasetField child2 = children2.stream()
+                    .filter(c -> c.getDatasetFieldType().equals(child1.getDatasetFieldType())).findFirst().orElse(null);
+            
+            if (child2 == null) {
+                return false;
+            }
+
+            // Compare values based on field type
+            if (child1.getDatasetFieldType().isControlledVocabulary()) {
+
+                List<ControlledVocabularyValue> cvs1 = child1.getControlledVocabularyValues();
+                List<ControlledVocabularyValue> cvs2 = child2.getControlledVocabularyValues();
+                
+                if (cvs1.size() != cvs2.size()) {
+                    
+                    return false;
+                }
+
+                for (ControlledVocabularyValue cv1Val : cvs1) {
+                    boolean found = cvs2.stream().anyMatch(cv2Val -> cv1Val.getStrValue().equals(cv2Val.getStrValue()));
+                    if (!found) {
+                        return false;
+                    }
+                }
+            } else {
+                // Handle regular field values (including multiple values)
+                List<DatasetFieldValue> dfvs1 = child1.getDatasetFieldValues();
+                List<DatasetFieldValue> dfvs2 = child2.getDatasetFieldValues();
+
+                if (dfvs1.size() != dfvs2.size()) {
+                    return false;
+                }
+
+                for (DatasetFieldValue dfv1 : dfvs1) {
+                    String value1 = dfv1.getValue();
+                    boolean found = dfvs2.stream()
+                            .anyMatch(dfv2 -> {
+                                String value2 = dfv2.getValue();
+                                if (value1 == null && value2 == null) {
+                                    return true;
+                                }
+                                return value1 != null && value1.equals(value2);
+                            });
+                    if (!found) {
+                        return false;
+                    }
+                }
+            }
+        }
+        return true;
+    }
 }

src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java

@@ -22,6 +22,7 @@
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Strings;
 
 /**
  *
@@ -193,6 +194,22 @@ public DatasetFieldValue copy(DatasetField dsf) {
         dsfv.setValue(value);
 
         return dsfv;
-    }    
+    }
 
+    /**
+     * Compares this DatasetFieldValue with another for equality based on their values.
+     * 
+     * @param other The DatasetFieldValue to compare with
+     * @return true if both values are equal (case-sensitive), false otherwise
+     */
+    public boolean valuesEqual(DatasetFieldValue other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null) {
+            return false;
+        }
+        return Strings.CS.equals(this.getValue(), other.getValue());
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java

@@ -20,6 +20,7 @@
 import jakarta.json.Json;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
+import jakarta.json.JsonValue;
 import jakarta.ws.rs.BadRequestException;
 
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -73,12 +74,20 @@ public COARNotifyRelationshipAnnouncement(
      */
     public void processMessage(JsonObject msgObject) {
         // Extract subject, object, and relationship from the message
-        String subjectId = extractField(msgObject, subjectKey);
-        String objectId = extractField(msgObject, objectKey);
-        String relationshipId = extractField(msgObject, relationshipKey);
+        String subjectId;
+        String objectId;
+        String relationshipId;
+        try {
+            // Extract subject, object, and relationship from the message
+            subjectId = extractField(msgObject, subjectKey);
+            objectId = extractField(msgObject, objectKey);
+            relationshipId = extractField(msgObject, relationshipKey);
 
-        if (subjectId == null || objectId == null || relationshipId == null) {
-            throw new BadRequestException("Can't find the subject, relationship or object in the message - ignoring");
+            if (subjectId == null || objectId == null || relationshipId == null) {
+                throw new BadRequestException("Can't find the subject, relationship or object in the message - ignoring");
+            }
+        } catch (Exception e) {
+            throw new BadRequestException("Failed to parse subject, relationship or object from the message - ignoring", e);
         }
 
         // Get metadata about the citing resource
@@ -112,7 +121,15 @@ public void processMessage(JsonObject msgObject) {
      * Extract a field value from the message object.
      */
     private String extractField(JsonObject msgObject, String key) {
-        return msgObject.containsKey(key) ? msgObject.getString(key) : null;
+        if (msgObject.containsKey(key)) {
+            JsonValue value = msgObject.get(key);
+            if (value.getValueType() == JsonValue.ValueType.OBJECT) {
+                return ((JsonObject) value).getString("@id", null);
+            } else if (value.getValueType() == JsonValue.ValueType.STRING) {
+                return msgObject.getString(key);
+            }
+        }
+        return null;
     }
 
     /**

src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java

@@ -1,8 +1,11 @@
 package edu.harvard.iq.dataverse.workflow.internalspi;
 
+import edu.harvard.iq.dataverse.ControlledVocabularyValue;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetField;
+import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
 import edu.harvard.iq.dataverse.DatasetFieldType;
+import edu.harvard.iq.dataverse.DatasetFieldValue;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
@@ -26,7 +29,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -88,12 +91,34 @@ public WorkflowStepResult run(WorkflowContext context) {
                 // First check that we have what is required
                 Dataset d = context.getDataset();
                 DatasetVersion dv = d.getReleasedVersion();
+                DatasetVersion priorVersion = d.getPriorReleasedVersion();
                 List<DatasetField> dvf = dv.getDatasetFields();
                 Map<String, DatasetField> fields = new HashMap<String, DatasetField>();
                 List<String> reqFields = ListSplitUtil.split((String) context.getSettings().getOrDefault(COARNotifyRelationshipAnnouncementTriggerFields.toString(), ""));
+                
+                Map<String, DatasetField> priorFields = new HashMap<String, DatasetField>();
+                if (priorVersion != null) {
+                    for (DatasetField pdf : priorVersion.getDatasetFields()) {
+                        if (!pdf.isEmpty() && reqFields.contains(pdf.getDatasetFieldType().getName())) {
+                            priorFields.put(pdf.getDatasetFieldType().getName(), pdf);
+                        }
+                    }
+                }
+                
                 for (DatasetField df : dvf) {
                     if (!df.isEmpty() && reqFields.contains(df.getDatasetFieldType().getName())) {
-                        fields.put(df.getDatasetFieldType().getName(), df);
+                        DatasetField priorField = priorFields.get(df.getDatasetFieldType().getName());
+                        
+                        if (priorVersion == null || priorField == null) {
+                            // No prior version, include all values
+                            fields.put(df.getDatasetFieldType().getName(), df);
+                        } else {
+                            // Create a filtered field with only new values
+                            DatasetField filteredField = filterNewValues(df, priorField);
+                            if (!filteredField.isEmpty()) {
+                                fields.put(df.getDatasetFieldType().getName(), filteredField);
+                            }
+                        }
                     }
                 }
 
@@ -214,6 +239,14 @@ JsonArray getObjects(WorkflowContext ctxt, Map<String, DatasetField> fields) {
 
     private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, Dataset d,
             Map<String, String> localContext) {
+        if (logger.isLoggable(Level.FINE)) {
+            if (jval.getValueType().equals(jakarta.json.JsonValue.ValueType.OBJECT)) {
+                logger.fine("Parsing : " + JsonUtil.prettyPrint(jval.asJsonObject()));
+            }
+            else if (jval.getValueType().equals(jakarta.json.JsonValue.ValueType.STRING)) {
+                logger.fine("Parsing : " + jval.toString());
+            }
+        }
         String[] answers = getBestIdAndType(dft, jval);
         String id = answers[0];
         String type = answers[1];
@@ -335,7 +368,8 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) {
                         break;
                     }
                 }
-            } else if (jo.containsKey(publicationURL.getLabel())) {
+            }
+            if (id == null && jo.containsKey(publicationURL.getLabel())) {
 
                 String value = jo.getString(publicationURL.getLabel());
                 if (isURI(value)) {
@@ -412,4 +446,90 @@ private boolean isURI(String number) {
         return false;
     }
 
+    /**
+     * Create a new DatasetField containing only values that are new compared to the
+     * prior field. This creates a detached copy to avoid modifying the managed
+     * entity.
+     * 
+     * @param currentField The field from the current version
+     * @param priorField   The field from the prior version
+     * @return A new DatasetField with only new values
+     */
+    private DatasetField filterNewValues(DatasetField currentField, DatasetField priorField) {
+        DatasetField filtered = new DatasetField();
+        DatasetFieldType fieldType = currentField.getDatasetFieldType();
+        filtered.setDatasetFieldType(fieldType);
+
+        // Handle primitive fields
+        if (fieldType.isPrimitive()) {
+            if (fieldType.isControlledVocabulary()) {
+                // Handle controlled vocabulary fields
+                List<ControlledVocabularyValue> currentCVs = currentField.getControlledVocabularyValues();
+                List<ControlledVocabularyValue> priorCVs = priorField != null ? priorField.getControlledVocabularyValues() : new ArrayList<>();
+                
+                List<ControlledVocabularyValue> newCVs = new ArrayList<>();
+                for (ControlledVocabularyValue currentCV : currentCVs) {
+                    boolean isNew = true;
+                    for (ControlledVocabularyValue priorCV : priorCVs) {
+                        if (currentCV.getStrValue().equals(priorCV.getStrValue())) {
+                            isNew = false;
+                            break;
+                        }
+                    }
+                    if (isNew) {
+                        newCVs.add(currentCV);
+                    }
+                }
+                filtered.setControlledVocabularyValues(newCVs);
+            } else {
+             // Handle regular fields
+                List<DatasetFieldValue> currentDFVs = currentField.getDatasetFieldValues();
+                List<DatasetFieldValue> priorDFVs = priorField != null ? priorField.getDatasetFieldValues() : new ArrayList<>();
+                
+                List<DatasetFieldValue> newDFVs = new ArrayList<>();
+                for (DatasetFieldValue currentDFV : currentDFVs) {
+                    boolean isNew = true;
+                    for (DatasetFieldValue priorDFV : priorDFVs) {
+                        if (currentDFV.valuesEqual(priorDFV)) {
+                            isNew = false;
+                            break;
+                        }
+                    }
+                    if (isNew) {
+                        newDFVs.add(currentDFV);
+                    }
+                }
+                filtered.setDatasetFieldValues(newDFVs);
+            }
+        } else {
+            // Handle compound fields
+            List<DatasetFieldCompoundValue> currentCompounds = currentField.getDatasetFieldCompoundValues();
+            List<DatasetFieldCompoundValue> priorCompounds = priorField != null ? priorField.getDatasetFieldCompoundValues() : new ArrayList<>();
+            
+            List<DatasetFieldCompoundValue> newCompounds = new ArrayList<>();
+            
+            for (DatasetFieldCompoundValue currentCompound : currentCompounds) {
+                boolean isNew = true;
+                
+                for (DatasetFieldCompoundValue priorCompound : priorCompounds) {
+
+                    if (currentCompound.valuesEqual(priorCompound)) {
+                        isNew = false;
+                        break;
+                    }
+                }
+                
+                if (isNew) {
+                    // Create a copy of the compound value with all its children
+                    DatasetFieldCompoundValue newCompound = currentCompound.copy(filtered);
+                    newCompound.setParentDatasetField(filtered);
+                    newCompounds.add(newCompound);
+                }
+            }
+            
+            filtered.setDatasetFieldCompoundValues(newCompounds);
+        }
+        
+        return filtered;
+    }
 }

src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java

@@ -1,4 +1,3 @@
-
 package edu.harvard.iq.dataverse.api;
 
 import org.junit.jupiter.api.Test;