Merge remote-tracking branch 'origin/dev-v3' into 109-multipleFAIR_DOs

# Conflicts:
#	build.gradle

Author: Pfeil

build.gradle

@@ -81,9 +81,8 @@ dependencies {
     implementation "org.springframework.data:spring-data-elasticsearch"
 
     // More flexibility when (de-)serializing json:
-    //implementation("com.monitorjbl:spring-json-view:1.1.0")
-    implementation("com.github.everit-org.json-schema:org.everit.json.schema:1.14.4")
-
+    implementation(group: 'com.networknt', name: 'json-schema-validator', version: '1.5.7');
+    
     implementation('org.apache.httpcomponents:httpclient:4.5.14')
     implementation('org.apache.httpcomponents:httpclient-cache:4.5.14')
 
@@ -188,15 +187,15 @@ jacocoTestReport {
     afterEvaluate {
         //exclude some classes/package from code coverage report
         classDirectories.setFrom(files(classDirectories.files.collect {
-            fileTree(dir: it, exclude: [\
-            'edu/kit/datamanager/pit/configuration/**',          \
-                     'edu/kit/datamanager/pit/web/converter/**',          \
-                     'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**',          \
-                     'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**',          \
-                     'edu/kit/datamanager/pit/common/**',          \
-                     'edu/kit/datamanager/pit/Application*'
-            ])
-        }))
+                    fileTree(dir: it, exclude: [\
+            'edu/kit/datamanager/pit/configuration/**', \
+            'edu/kit/datamanager/pit/web/converter/**', \
+            'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**', \
+            'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**', \
+            'edu/kit/datamanager/pit/common/**', \
+            'edu/kit/datamanager/pit/Application*'
+                        ])
+                }))
     }
 }
 

src/main/java/edu/kit/datamanager/pit/common/ExternalServiceException.java

@@ -6,12 +6,12 @@
 
 public class ExternalServiceException extends ResponseStatusException {
 
-    private static final String MESSAGE_TERM_SERVICE = "Service";
+    private static final String MESSAGE_TERM_SERVICE = "Service ";
     private static final long serialVersionUID = 1L;
     private static final HttpStatus HTTP_STATUS = HttpStatus.SERVICE_UNAVAILABLE;
 
     public ExternalServiceException(String serviceName) {
-        super(HTTP_STATUS, "Service " + serviceName + " not available.");
+        super(HTTP_STATUS, MESSAGE_TERM_SERVICE + serviceName + " not available.");
     }
 
     public ExternalServiceException(String serviceName, Throwable e) {

src/main/java/edu/kit/datamanager/pit/domain/Operations.java

@@ -1,6 +1,7 @@
 package edu.kit.datamanager.pit.domain;
 
 import java.io.IOException;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -15,13 +16,11 @@
 import edu.kit.datamanager.pit.typeregistry.AttributeInfo;
 import edu.kit.datamanager.pit.typeregistry.ITypeRegistry;
 import org.apache.commons.lang3.stream.Streams;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
+import java.time.ZonedDateTime;
 
 /**
  * Simple operations on PID records.
- * 
+ * <p>
  * Caches results e.g. for type queries
  */
 public class Operations {
@@ -35,8 +34,8 @@ public class Operations {
         "21.T11148/397d831aa3a9d18eb52c"
     };
 
-    private ITypeRegistry typeRegistry;
-    private IIdentifierSystem identifierSystem;
+    private final ITypeRegistry typeRegistry;
+    private final IIdentifierSystem identifierSystem;
 
     public Operations(ITypeRegistry typeRegistry, IIdentifierSystem identifierSystem) {
         this.typeRegistry = typeRegistry;
@@ -45,133 +44,125 @@ public Operations(ITypeRegistry typeRegistry, IIdentifierSystem identifierSystem
 
     /**
      * Tries to get the date when a FAIR DO was created from a PID record.
-     * 
+     * <p>
      * Strategy:
      * - try to get it from known "dateCreated" types
-     * - as a fallback, try to get it by its human readable name
-     * 
+     * - as a fallback, try to get it by its human-readable name
+     * <p>
      * Semantic reasoning in some sense is planned but not yet supported.
      * 
      * @param pidRecord the record to extract the information from.
-     * @return the date, if it could been extracted.
+     * @return the date, if it could have been extracted.
      * @throws IOException on IO errors regarding resolving types.
      */
     public Optional<Date> findDateCreated(PIDRecord pidRecord) throws IOException {
         /* try known types */
         List<String> knownDateTypes = Arrays.asList(Operations.KNOWN_DATE_CREATED);
         Optional<Date> date = knownDateTypes
-            .stream()
-            .map(pidRecord::getPropertyValues)
-            .map(Arrays::asList)
-            .flatMap(List<String>::stream)
-            .map(this::extractDate)
-            .filter(Optional<Date>::isPresent)
-            .map(Optional<Date>::get)
-            .sorted(Comparator.comparingLong(Date::getTime))
-            .findFirst();
+                .stream()
+                .map(pidRecord::getPropertyValues)
+                .map(Arrays::asList)
+                .flatMap(List<String>::stream)
+                .map(this::extractDate)
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .min(Comparator.comparingLong(Date::getTime));
         if (date.isPresent()) {
             return date;
         }
 
         Collection<AttributeInfo> types = new ArrayList<>();
         List<CompletableFuture<?>> futures = Streams.failableStream(
                 pidRecord.getPropertyIdentifiers().stream())
-                .filter(attributePid -> this.identifierSystem.isPidRegistered(attributePid))
-                .map(attributePid -> {
-                    return this.typeRegistry
+                .filter(this.identifierSystem::isPidRegistered)
+                .map(attributePid -> this.typeRegistry
                             .queryAttributeInfo(attributePid)
-                            .thenAcceptAsync(types::add);
-                })
+                            .thenAcceptAsync(types::add))
                 .collect(Collectors.toList());
         CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
 
         /*
-         * as a last fallback, try find types with human readable names containing
+         * as a last fallback, try to find types with human-readable names containing
          * "dateCreated" or "createdAt" or "creationDate".
          * 
          * This can be removed as soon as we have some default FAIR DO types new type
          * definitions can refer to (e.g. "extend" them or declare the same meaning as
          * our known types, see above)
          */
         return types
-            .stream()
-            .filter(type -> 
-                type.name().equalsIgnoreCase("dateCreated")
-                || type.name().equalsIgnoreCase("createdAt")
-                || type.name().equalsIgnoreCase("creationDate"))
-            .map(type -> pidRecord.getPropertyValues(type.pid()))
-            .map(Arrays::asList)
-            .flatMap(List<String>::stream)
-            .map(this::extractDate)
-            .filter(Optional<Date>::isPresent)
-            .map(Optional<Date>::get)
-            .sorted(Comparator.comparingLong(Date::getTime))
-            .findFirst();
+                .stream()
+                .filter(type ->
+                        type.name().equalsIgnoreCase("dateCreated")
+                                || type.name().equalsIgnoreCase("createdAt")
+                                || type.name().equalsIgnoreCase("creationDate"))
+                .map(type -> pidRecord.getPropertyValues(type.pid()))
+                .map(Arrays::asList)
+                .flatMap(List<String>::stream)
+                .map(this::extractDate)
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .min(Comparator.comparingLong(Date::getTime));
     }
 
     /**
      * Tries to get the date when a FAIR DO was modified from a PID record.
-     * 
+     * <p>
      * Strategy:
      * - try to get it from known "dateModified" types
-     * - as a fallback, try to get it by its human readable name
-     * 
+     * - as a fallback, try to get it by its human-readable name
+     * <p>
      * Semantic reasoning in some sense is planned but not yet supported.
      * 
      * @param pidRecord the record to extract the information from.
-     * @return the date, if it could been extracted.
+     * @return the date, if it could have been extracted.
      * @throws IOException on IO errors regarding resolving types.
      */
     public Optional<Date> findDateModified(PIDRecord pidRecord) throws IOException {
         /* try known types */
         List<String> knownDateTypes = Arrays.asList(Operations.KNOWN_DATE_MODIFIED);
         Optional<Date> date = knownDateTypes
-            .stream()
-            .map(pidRecord::getPropertyValues)
-            .map(Arrays::asList)
-            .flatMap(List<String>::stream)
-            .map(this::extractDate)
-            .filter(Optional<Date>::isPresent)
-            .map(Optional<Date>::get)
-            .sorted(Comparator.comparingLong(Date::getTime))
-            .findFirst();
+                .stream()
+                .map(pidRecord::getPropertyValues)
+                .map(Arrays::asList)
+                .flatMap(List<String>::stream)
+                .map(this::extractDate)
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .min(Comparator.comparingLong(Date::getTime));
         if (date.isPresent()) {
             return date;
         }
 
         Collection<AttributeInfo> types = new ArrayList<>();
         List<CompletableFuture<?>> futures = Streams.failableStream(pidRecord.getPropertyIdentifiers().stream())
-                .filter(attributePid -> this.identifierSystem.isPidRegistered(attributePid))
-                .map(attributePid -> {
-                    return this.typeRegistry
+                .filter(this.identifierSystem::isPidRegistered)
+                .map(attributePid -> this.typeRegistry
                             .queryAttributeInfo(attributePid)
-                            .thenAcceptAsync(types::add);
-                })
+                            .thenAcceptAsync(types::add))
                 .collect(Collectors.toList());
         CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
 
         /*
-         * as a last fallback, try find types with human readable names containing
+         * as a last fallback, try to find types with human-readable names containing
          * "dateModified" or "lastModified" or "modificationDate".
          * 
          * This can be removed as soon as we have some default FAIR DO types new type
          * definitions can refer to (e.g. "extend" them or declare the same meaning as
          * our known types, see above)
          */
         return types
-            .stream()
-            .filter(type -> 
-                type.name().equalsIgnoreCase("dateModified")
-                || type.name().equalsIgnoreCase("lastModified")
-                || type.name().equalsIgnoreCase("modificationDate"))
-            .map(type -> pidRecord.getPropertyValues(type.pid()))
-            .map(Arrays::asList)
-            .flatMap(List<String>::stream)
-            .map(this::extractDate)
-            .filter(Optional<Date>::isPresent)
-            .map(Optional<Date>::get)
-            .sorted(Comparator.comparingLong(Date::getTime))
-            .findFirst();
+                .stream()
+                .filter(type ->
+                        type.name().equalsIgnoreCase("dateModified")
+                                || type.name().equalsIgnoreCase("lastModified")
+                                || type.name().equalsIgnoreCase("modificationDate"))
+                .map(type -> pidRecord.getPropertyValues(type.pid()))
+                .map(Arrays::asList)
+                .flatMap(List<String>::stream)
+                .map(this::extractDate)
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .min(Comparator.comparingLong(Date::getTime));
     }
 
     /**
@@ -181,10 +172,10 @@ public Optional<Date> findDateModified(PIDRecord pidRecord) throws IOException {
      * @return the extracted Date object.
      */
     protected Optional<Date> extractDate(String dateString) {
-        DateTimeFormatter dateFormatter = ISODateTimeFormat.dateTime();
+        DateTimeFormatter dateFormatter = DateTimeFormatter.ISO_DATE_TIME;
         try {
-            DateTime dateTime = dateFormatter.parseDateTime(dateString);
-            return Optional.of(dateTime.toDate());
+            ZonedDateTime dateTime = ZonedDateTime.parse(dateString, dateFormatter);
+            return Optional.of(Date.from(dateTime.toInstant()));
         } catch (Exception e) {
             return Optional.empty();
         }

src/main/java/edu/kit/datamanager/pit/typeregistry/AttributeInfo.java

@@ -1,12 +1,18 @@
 package edu.kit.datamanager.pit.typeregistry;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.networknt.schema.JsonSchema;
+import com.networknt.schema.ValidationMessage;
+import edu.kit.datamanager.pit.Application;
 import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo;
-import org.everit.json.schema.Schema;
-import org.everit.json.schema.ValidationException;
-import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * @param pid the pid of this attribute
@@ -22,23 +28,50 @@ public record AttributeInfo(
         String typeName,
         Collection<SchemaInfo> jsonSchema
 ) {
+    private static final Logger log = LoggerFactory.getLogger(AttributeInfo.class);
+
     public boolean validate(String value) {
         return this.jsonSchema().stream()
-                .map(SchemaInfo::schema)
-                .filter(Objects::nonNull)
-                .anyMatch(schema -> validate(schema, value));
+                .filter(schemaInfo -> schemaInfo.error() == null)
+                .filter(schemaInfo -> schemaInfo.schema() != null)
+                .peek(schemaInfo -> log.warn("Found valid schema from {} to validate {} / {}.", schemaInfo.origin(), pid, value))
+                .anyMatch(schemaInfo -> this.validate(schemaInfo.schema(), value));
     }
 
-    private boolean validate(Schema schema, String value) {
-        Object toValidate = value;
-        if (value.startsWith("{")) {
-            toValidate = new JSONObject(value);
-        }
+    private boolean validate(JsonSchema schema, String value) {
         try {
-            schema.validate(toValidate);
-        } catch (ValidationException e) {
+            JsonNode toValidate = valueToJsonNode(value);
+            Set<ValidationMessage> errors = schema.validate(toValidate, executionContext -> {
+                // By default, since Draft 2019-09, the format keyword only generates annotations and not assertions
+                executionContext.getExecutionConfig().setFormatAssertionsEnabled(true);
+            });
+            if (!errors.isEmpty()) {
+                log.warn("Validation errors for value '{}': {}", value, errors);
+            }
+            return errors.isEmpty();
+        } catch (Exception e) {
+            log.error("Exception during validation for value '{}': {}", value, e.getMessage(), e);
             return false;
         }
-        return true;
     }
-}
+
+    /**
+     * Converts the given value to a JsonNode.
+     *
+     * @param value the value to convert
+     * @return a JsonNode representation of the value
+     */
+    public static JsonNode valueToJsonNode(String value) {
+        JsonNode toValidate;
+        if (value.isBlank()) {
+            return new TextNode(value);
+        }
+        try {
+            toValidate = Application.jsonObjectMapper().readTree(value);
+        } catch (JsonProcessingException e) {
+            log.warn("Failed to parse value '{}' as JSON, treating it as a plain text node.", value);
+            toValidate = new TextNode(value);
+        }
+        return toValidate;
+    }
+}