Merge pull request #11429 from IQSS/11382-featured-items-unicode-filename

Allow images with unicode filenames to be uploaded for featured items

Author: ofahimIQSS

scripts/issues/11429/add-featured-items.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+export API_TOKEN=e62ac03b-3bc8-4194-bcd6-d0381fe84e39
+export SERVER_URL=http://localhost:8080
+export ID=:root
+export FILENAME='π.png'
+export CONTENT='test1'
+export DISPLAY_ORDER=0
+
+curl -H "X-Dataverse-key:$API_TOKEN" \
+     -X POST \
+     -F "id=0" \
+     -F "content=$CONTENT" \
+     -F "displayOrder=$DISPLAY_ORDER" \
+     -F "fileName=$FILENAME" \
+     -F "keepFile=false" \
+     -F "file=@$FILENAME" \
+     "$SERVER_URL/api/dataverses/$ID/featuredItems"

scripts/issues/11429/update-featured-item.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -x
+export API_TOKEN=e62ac03b-3bc8-4194-bcd6-d0381fe84e39
+export SERVER_URL=http://localhost:8080
+export ID=78
+export FILENAME='π.png'
+export CONTENT='test1'
+export DISPLAY_ORDER=0
+
+curl -H "X-Dataverse-key:$API_TOKEN" \
+     -X PUT \
+     -F "id=0" \
+     -F "content=$CONTENT" \
+     -F "displayOrder=$DISPLAY_ORDER" \
+     -F "fileName=$FILENAME" \
+     -F "keepFile=false" \
+     -F "file=@$FILENAME" \
+     "$SERVER_URL/api/dataverseFeaturedItems/$ID"

scripts/issues/11429/update-featured-items.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+export API_TOKEN=e62ac03b-3bc8-4194-bcd6-d0381fe84e39
+export SERVER_URL=http://localhost:8080
+export ID=:root
+export FILENAME='π.png'
+export CONTENT='test1'
+export DISPLAY_ORDER=0
+
+curl -H "X-Dataverse-key:$API_TOKEN" \
+     -X PUT \
+     -F "id=0" \
+     -F "content=$CONTENT" \
+     -F "displayOrder=$DISPLAY_ORDER" \
+     -F "fileName=$FILENAME" \
+     -F "keepFile=false" \
+     -F "file=@$FILENAME" \
+     "$SERVER_URL/api/dataverses/$ID/featuredItems"

scripts/issues/11429/π.png

@@ -29,6 +29,7 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
+import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*;
@@ -1863,7 +1864,7 @@ public Response updateFeaturedItems(
 
                 if (files != null) {
                     Optional<FormDataBodyPart> matchingFile = files.stream()
-                            .filter(file -> file.getFormDataContentDisposition().getFileName().equals(fileName))
+                            .filter(file -> fileName.equals(FileUtil.decodeFileName(file.getFormDataContentDisposition().getFileName())))
                             .findFirst();
 
                     if (matchingFile.isPresent()) {

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.api.dto;
 
+import edu.harvard.iq.dataverse.util.FileUtil;
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
 
 import java.io.InputStream;
@@ -21,7 +22,7 @@ public static NewDataverseFeaturedItemDTO fromFormData(String content,
 
         if (imageFileInputStream != null) {
             newDataverseFeaturedItemDTO.imageFileInputStream = imageFileInputStream;
-            newDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName();
+            newDataverseFeaturedItemDTO.imageFileName = FileUtil.decodeFileName(contentDispositionHeader.getFileName());
         }
 
         return newDataverseFeaturedItemDTO;

src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java

@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.api.dto;
 
+import edu.harvard.iq.dataverse.util.FileUtil;
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
 
 import java.io.InputStream;
@@ -24,7 +25,7 @@ public static UpdatedDataverseFeaturedItemDTO fromFormData(String content,
 
         if (imageFileInputStream != null) {
             updatedDataverseFeaturedItemDTO.imageFileInputStream = imageFileInputStream;
-            updatedDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName();
+            updatedDataverseFeaturedItemDTO.imageFileName = FileUtil.decodeFileName(contentDispositionHeader.getFileName());
         }
 
         return updatedDataverseFeaturedItemDTO;

src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java

@@ -101,6 +101,7 @@
 import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -1881,4 +1882,19 @@ public static boolean isFileOfImageType(File file) throws IOException {
         String mimeType = tika.detect(file);
         return mimeType != null && mimeType.startsWith("image/");
     }
+
+    /**
+     * Converts a filename from ISO_8859_1 to UTF_8 to handle encoding issues.
+     * Ref: https://github.com/eclipse-ee4j/jersey/issues/1700
+     *
+     * @param originalFileName the original filename
+     * @return the filename converted to UTF_8
+     */
+    public static String decodeFileName(String originalFileName) {
+        if (originalFileName == null) {
+            return null;
+        }
+        return new String(originalFileName.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java

@@ -10,6 +10,14 @@
 import java.text.MessageFormat;
 
 import static jakarta.ws.rs.core.Response.Status.*;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class DataverseFeaturedItemsIT {
@@ -84,6 +92,71 @@ public void testUpdateFeaturedItem() {
         verifyUpdatedFeaturedItem(updateFeatureItemResponse, sanitizedContent, "coffeeshop.png", 2);
     }
 
+    @Test
+    public void testUpdateFeaturedItemUnicode() {
+        String apiToken = createUserAndGetApiToken();
+        String dataverseAlias = createDataverseAndGetAlias(apiToken);
+
+        String coffeeShopEnglish = "src/test/resources/images/coffeeshop.png";
+        String coffeeShopGreek = System.getProperty("java.io.tmpdir") + File.separator + "καφενείο.png";
+        Path pathToCoffeeShopGreek = java.nio.file.Paths.get(coffeeShopGreek);
+        System.out.println("path to coffee show in Greek: " + pathToCoffeeShopGreek);
+        try {
+            java.nio.file.Files.copy(java.nio.file.Paths.get(coffeeShopEnglish), pathToCoffeeShopGreek, java.nio.file.StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException ex) {
+            Logger.getLogger(DataverseFeaturedItemsIT.class.getName()).log(Level.SEVERE, null, ex);
+        }
+
+        Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 0, coffeeShopGreek);
+        createFeatureItemResponse.prettyPrint();
+        /**
+         * TODO: Fix this REST Assured test. Sending unicode works fine in curl
+         * (see scripts/issues/11429/add-featured-items.sh) and we suspect we
+         * aren't sending Unicode properly through REST Assured (or Unicode
+         * isn't supported, which isn't likely). For now we assert
+         * "????????.png" but once we fix the test, "καφενείο.png" should be
+         * asserted.
+         */
+        verifyUpdatedFeaturedItem(createFeatureItemResponse, "test", "????????.png", 0);
+
+        long featuredItemId = JsonPath.from(createFeatureItemResponse.body().asString()).getLong("data.id");
+
+        // update content
+        Response updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 1, true, null, apiToken);
+        updateFeatureItemResponse.prettyPrint();
+        // TODO: Fix this REST Assured assertion too (see above).
+        // The equivalent curl command: scripts/issues/11429/update-featured-item.sh
+        verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", "????????.png", 1);
+
+        // remove image
+        updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 2, false, null, apiToken);
+        verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", null, 2);
+
+        // add non-unicode image
+        updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 2, false, coffeeShopEnglish, apiToken);
+        verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", "coffeeshop.png", 2);
+
+        updateFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, apiToken);
+        updateFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        List<Long> ids = Arrays.asList(0L);
+        List<String> contents = Arrays.asList("Greek filename");
+        List<Integer> orders = Arrays.asList(0);
+        List<Boolean> keepFiles = Arrays.asList(false);
+        List<String> pathsToFiles = Arrays.asList(coffeeShopGreek);
+
+        Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken);
+        updateDataverseFeaturedItemsResponse.prettyPrint();
+        updateDataverseFeaturedItemsResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].content", equalTo("Greek filename"))
+                // TODO: Fix this REST Assured assertion too (see above).
+                // The equivalent curl command: scripts/issues/11429/update-featured-items.sh
+                .body("data[0].imageFileName", equalTo("????????.png"))
+                .body("data[0].imageFileUrl", containsString("/api/access/dataverseFeaturedItemImage/"))
+                .body("data[0].displayOrder", equalTo(0));
+    }
+
     private String createUserAndGetApiToken() {
         Response createUserResponse = UtilIT.createRandomUser();
         return UtilIT.getApiTokenFromResponse(createUserResponse);