Merge branch 'PIDFailTracking' of https://github.com/QualitativeDataRepository/dataverse.git into PIDFailTracking

Author: qqmyers

doc/release-notes/11703-notifications-and-templates-api-fixes.md

@@ -0,0 +1,13 @@
+# Release Notes
+
+## Templates API fixes
+- Added the missing `isDefault` field to the **Get Dataset Templates** API response.
+- Added the missing `fileAccessRequest` field of terms of use and access to the **Get Dataset Templates** API response.
+- Added the missing `contactForAccess` field of terms of use and access to the **Get Dataset Templates** API response.
+- Resolved an API **500 error** that occurred when working with templates containing custom license terms.
+- Updated API behavior so templates are **no longer returned from parent dataverses** when the "Include Templates from Root" option is unchecked in the UI.
+
+## Notifications API fixes
+- Fixed API errors caused by NullPointerException when retrieving notifications without a requestor.
+
+See #11704

doc/sphinx-guides/source/developers/deployment.rst

@@ -110,14 +110,16 @@ Migrating Datafiles from Local Storage to S3
 
 A number of pilot Dataverse installations start on local storage, then administrators are tasked with migrating datafiles into S3 or similar object stores. The files may be copied with a command-line utility such as `s3cmd <https://s3tools.org/s3cmd>`_. You will want to retain the local file hierarchy, keeping the authority (for example: 10.5072) at the bucket "root."
 
-The below example queries may assist with updating dataset and datafile locations in the Dataverse installation's PostgresQL database. Depending on the initial version of the Dataverse Software and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases.
+The below example queries may assist with updating dataset and datafile locations in the Dataverse installation's PostgresQL database. Depending on the initial version of the Dataverse Software and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. 
+
+In the following queries, ``<id>://`` refers to the ``id`` you have set for the datastore in the configuration (see the :ref:`file-storage` section of the Installation Guide).
 
 To Update Dataset Location to S3, Assuming a ``file://`` Prefix
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 ::
 
-  UPDATE dvobject SET storageidentifier=REPLACE(storageidentifier,'file://','s3://')
+  UPDATE dvobject SET storageidentifier=REPLACE(storageidentifier,'file://','<id>://')
     WHERE dtype='Dataset';
 
 To Update Datafile Location to your-s3-bucket, Assuming a ``file://`` Prefix
@@ -126,17 +128,17 @@ To Update Datafile Location to your-s3-bucket, Assuming a ``file://`` Prefix
 ::
 
   UPDATE dvobject
-    SET storageidentifier=REPLACE(storageidentifier,'file://','s3://your-s3-bucket:')
+    SET storageidentifier=REPLACE(storageidentifier,'file://','<id>://your-s3-bucket:')
     WHERE id IN (SELECT o.id FROM dvobject o, dataset s WHERE o.dtype = 'DataFile'
     AND s.id = o.owner_id AND s.harvestingclient_id IS null
-    AND o.storageidentifier NOT LIKE 's3://%');
+    AND o.storageidentifier NOT LIKE '<id>://%');
 
 To Update Datafile Location to your-s3-bucket, Assuming no ``file://`` Prefix
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 ::
 
-  UPDATE dvobject SET storageidentifier=CONCAT('s3://your-s3-bucket:', storageidentifier)
+  UPDATE dvobject SET storageidentifier=CONCAT('<id>://your-s3-bucket:', storageidentifier)
 	  WHERE id IN (SELECT o.id FROM dvobject o, dataset s WHERE o.dtype = 'DataFile'
 	  AND s.id = o.owner_id AND s.harvestingclient_id IS null
 	  AND o.storageidentifier NOT LIKE '%://%');

doc/sphinx-guides/source/installation/config.rst

@@ -3797,6 +3797,9 @@ please find all known feature flags below. Any of these flags can be activated u
     * - shibboleth-use-localhost
       - A Shibboleth-using Dataverse instance needs to make network calls to the locally-running ``shibd`` service. The default behavior is to use the address configured via the ``siteUrl`` setting. There are however situations (firewalls, etc.) where localhost would be preferable.
       - ``Off``
+    * - add-local-contexts-permission-check
+      - Adds a permission check to ensure that the user calling the /api/localcontexts/datasets/{id} API can edit the dataset with that id. This is currently the only use case - see https://github.com/gdcc/dataverse-external-vocab-support/tree/main/packages/local_contexts. The flag adds additional security to stop other uses, but would currently have to be used in conjunction with the api-session-auth feature flag (the security implications of which have not been fully investigated) to still allow adding Local Contexts metadata to a dataset.
+      - ``Off``
     * - enable-pid-failure-log
       - Turns on creation of a monthly log file (logs/PIDFailures_<yyyy-MM>.log) showing failed requests for dataset/file PIDs. Can be used directly or with scripts at https://github.com/gdcc/dataverse-recipes/python/pid_reports to alert admins.
       - ``Off``

src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java

@@ -22,6 +22,7 @@
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.FileUtil;
 import jakarta.ejb.EJB;
 import org.apache.commons.io.IOUtils;
 import org.apache.tika.Tika;
@@ -60,13 +61,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
         try {
             File fileIn = physicalPath.toFile();
             if (fileIn != null) {
-                Tika tika = new Tika();
-                try {
-                    String mimeType = tika.detect(fileIn);
-                    response.setContentType(mimeType);
-                } catch (Exception e) {
-                    logger.info("Error getting MIME Type for " + filePath + " : " + e.getMessage());
-                }
+                String filename = physicalPath.getFileName().toString();
+                int dotIndex = filename.lastIndexOf('.');
+                String ext = dotIndex >= 0 ? filename.substring(dotIndex) : "";
+                String mimeType = FileUtil.lookupFileTypeByExtension(ext);
+                response.setContentType(mimeType);
                 inputStream = new FileInputStream(fileIn);
 
                 in = new BufferedReader(new InputStreamReader(inputStream));

src/main/java/edu/harvard/iq/dataverse/api/dto/NewTemplateDTO.java

@@ -38,6 +38,7 @@ public Template toTemplate() {
         template.updateInstructions();
         template.setCreateTime(new Timestamp(new Date().getTime()));
         template.setUsageCount(0L);
+        template.setIsDefaultForDataverse(isDefault);
 
         return template;
     }

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateTemplateCommand.java

@@ -48,7 +48,14 @@ public Template execute(CommandContext ctxt) throws CommandException {
             DatasetFieldUtil.tidyUpFields(template.getDatasetFields(), false);
         }
 
-        return ctxt.templates().save(template);
+        Template createdTemplate = ctxt.templates().save(template);
+
+        if (initialize && template.isIsDefaultForDataverse()) {
+            dataverse.setDefaultTemplate(createdTemplate);
+            ctxt.em().merge(dataverse);
+        }
+
+        return template;
     }
 
     private static void updateTermsOfUseAndAccess(CommandContext ctxt, Template template) {

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseTemplatesCommand.java

@@ -26,14 +26,22 @@ public ListDataverseTemplatesCommand(DataverseRequest request, Dataverse dataver
 
     @Override
     public List<Template> execute(CommandContext ctxt) throws CommandException {
-        List<Template> templates = new ArrayList<>();
+        List<Template> availableTemplates = new ArrayList<>(dataverse.getTemplates());
 
-        if (dataverse.getOwner() != null) {
-            templates.addAll(dataverse.getParentTemplates());
+        if (!dataverse.isTemplateRoot() && dataverse.getOwner() != null) {
+            availableTemplates.addAll(dataverse.getParentTemplates());
         }
 
-        templates.addAll(dataverse.getTemplates());
+        setDefaultTemplate(availableTemplates);
 
-        return templates;
+        return availableTemplates;
+    }
+
+    private void setDefaultTemplate(List<Template> availableTemplates) {
+        Optional.ofNullable(dataverse.getDefaultTemplate())
+                .map(Template::getId).flatMap(defaultTemplateId -> availableTemplates.stream()
+                        .filter(template -> template.getId().equals(defaultTemplateId))
+                        .findFirst())
+                .ifPresent(template -> template.setIsDefaultForDataverse(true));
     }
 }

src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java

@@ -598,7 +598,10 @@ public static String determineFileTypeByNameAndExtension(final String fileName)
         return lookupFileTypeByExtension(fileName);
     }
 
-    private static String lookupFileTypeByExtension(final String fileName) {
+    /** determineFileTypeByNameAndExtension should be used instead for any user supplied content. 
+     *
+     */
+    public static String lookupFileTypeByExtension(final String fileName) {
         final String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName);
         logger.fine("MimetypesFileTypeMap type by extension, for " + fileName + ": " + mimetypesFileTypeMapResult);
         if (mimetypesFileTypeMapResult == null) {

src/main/java/edu/harvard/iq/dataverse/util/json/InAppNotificationsJsonPrinter.java

@@ -195,9 +195,11 @@ private void addDatasetCreatedFields(final NullSafeJsonBuilder notificationJson,
     }
 
     private void addRequestorFields(final NullSafeJsonBuilder notificationJson, final AuthenticatedUser requestor) {
-        notificationJson.add(KEY_REQUESTOR_FIRST_NAME, requestor.getFirstName());
-        notificationJson.add(KEY_REQUESTOR_LAST_NAME, requestor.getLastName());
-        notificationJson.add(KEY_REQUESTOR_EMAIL, requestor.getEmail());
+        if (requestor != null) {
+            notificationJson.add(KEY_REQUESTOR_FIRST_NAME, requestor.getFirstName());
+            notificationJson.add(KEY_REQUESTOR_LAST_NAME, requestor.getLastName());
+            notificationJson.add(KEY_REQUESTOR_EMAIL, requestor.getEmail());
+        }
     }
 
     private void addDatasetFields(final NullSafeJsonBuilder notificationJson, final UserNotification userNotification) {

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -1592,6 +1592,7 @@ public static JsonObjectBuilder jsonTemplate(Template template) {
         return jsonObjectBuilder()
                 .add("id", template.getId())
                 .add("name", template.getName())
+                .add("isDefault", template.isIsDefaultForDataverse())
                 .add("usageCount", template.getUsageCount())
                 .add("createTime", template.getCreateTime().toString())
                 .add("createDate", template.getCreateDate())
@@ -1602,9 +1603,10 @@ public static JsonObjectBuilder jsonTemplate(Template template) {
     }
 
     public static JsonObjectBuilder jsonTermsOfUseAndAccess(TermsOfUseAndAccess termsOfUseAndAccess) {
+        License license = termsOfUseAndAccess.getLicense();
         return jsonObjectBuilder()
                 .add("id", termsOfUseAndAccess.getId())
-                .add("license", json(termsOfUseAndAccess.getLicense()))
+                .add("license", license != null ? json(license) : null)
                 .add("termsOfUse", termsOfUseAndAccess.getTermsOfUse())
                 .add("termsOfAccess", termsOfUseAndAccess.getTermsOfAccess())
                 .add("confidentialityDeclaration", termsOfUseAndAccess.getConfidentialityDeclaration())
@@ -1618,7 +1620,9 @@ public static JsonObjectBuilder jsonTermsOfUseAndAccess(TermsOfUseAndAccess term
                 .add("originalArchive", termsOfUseAndAccess.getOriginalArchive())
                 .add("availabilityStatus", termsOfUseAndAccess.getAvailabilityStatus())
                 .add("sizeOfCollection", termsOfUseAndAccess.getSizeOfCollection())
-                .add("studyCompletion", termsOfUseAndAccess.getStudyCompletion());
+                .add("studyCompletion", termsOfUseAndAccess.getStudyCompletion())
+                .add("contactForAccess", termsOfUseAndAccess.getContactForAccess())
+                .add("fileAccessRequest", termsOfUseAndAccess.isFileAccessRequest());
     }
 
     public static JsonArrayBuilder jsonTemplateInstructions(Map<String, String> templateInstructions) {