test(files): refactor tabular ingest size limit tests with parameterized approach #11639

Simplified and improved test structure by introducing nested classes, parameterized tests, and utility methods for TabularIngestSizeLimit configurations. Enhanced code readability and reusability.

Author: poikilotherm

src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java

@@ -9,17 +9,22 @@
 import io.restassured.RestAssured;
 import io.restassured.response.Response;
 
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.logging.Logger;
 
 import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism;
 import jakarta.json.JsonObject;
+import jakarta.ws.rs.core.Response.Status;
 import org.assertj.core.util.Lists;
+import org.hamcrest.Matcher;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.BeforeAll;
 import io.restassured.path.json.JsonPath;
 
 import static edu.harvard.iq.dataverse.api.ApiConstants.*;
+import static edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key;
 import static io.restassured.path.json.JsonPath.with;
 import io.restassured.path.xml.XmlPath;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -45,6 +50,10 @@
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import static org.hamcrest.CoreMatchers.*;
 import static org.junit.jupiter.api.Assertions.*;
@@ -59,15 +68,11 @@ public static void setUpClass() {
 
         Response removePublicInstall = UtilIT.deleteSetting(SettingsServiceBean.Key.PublicInstall);
         removePublicInstall.then().assertThat().statusCode(200);
-
-        Response removeLimit = UtilIT.deleteSetting(SettingsServiceBean.Key.TabularIngestSizeLimit);
-        removeLimit.then().assertThat().statusCode(OK.getStatusCode());
     }
 
     @AfterAll
     public static void tearDownClass() {
         UtilIT.deleteSetting(SettingsServiceBean.Key.PublicInstall);
-        UtilIT.deleteSetting(SettingsServiceBean.Key.TabularIngestSizeLimit);
     }
 
     /**
@@ -1211,183 +1216,102 @@ public void test_AddFileBadUploadFormat() {
         }
 
     }
-
-    @Test
-    public void testIngestSizeLimits() throws InterruptedException, IOException {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String username = UtilIT.getUsernameFromResponse(createUser);
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-        Response makeSuperUser = UtilIT.setSuperuserStatus(username, true);
-        makeSuperUser.then().assertThat().statusCode(OK.getStatusCode());
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.prettyPrint();
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.prettyPrint();
-        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        String tinyCsvOnly = """
-{
-  "csv": "50"
-}
-""";
-
-        Response setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, tinyCsvOnly);
-        setLimit.then().assertThat().statusCode(OK.getStatusCode());
-
-        Path pathToDataFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv");
-        String contentOfCsv = ""
-                + "name,pounds,species,treats\n"
-                + "Midnight,15,dog,milkbones\n"
-                + "Tiger,17,cat,cat grass\n"
-                + "Panther,21,cat,cat nip\n";
-        java.nio.file.Files.write(pathToDataFile, contentOfCsv.getBytes());
-
-        Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
-        uploadFile.prettyPrint();
-        uploadFile.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data.files[0].label", equalTo("data.csv"));
-
-        String fileId1 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id");
-
-        Response getTabularFails = UtilIT.getFileDataTables(fileId1, apiToken);
-        getTabularFails.prettyPrint();
-        getTabularFails.then().assertThat()
-                .statusCode(BAD_REQUEST.getStatusCode())
-                .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported")));
-
-        String largerCsv = """
-{
-  "csv": "123456"
-}
-""";
-
-        setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, largerCsv);
-        setLimit.then().assertThat().statusCode(OK.getStatusCode());
-
-        uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
-        uploadFile.prettyPrint();
-        uploadFile.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data.files[0].label", equalTo("data-1.csv"));
-
-        assertTrue(UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if Ingest Lock exceeds max duration " + pathToDataFile);
-
-        String fileId2 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id");
-
-        Response getTabularWorks = UtilIT.getFileDataTables(fileId2, apiToken);
-        getTabularWorks.prettyPrint();
-        getTabularWorks.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].varQuantity", equalTo(4));
-
-        String tinyDefaultSize = """
-{
-  "default": "50"
-}
-""";
-
-        setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, tinyDefaultSize);
-        setLimit.then().assertThat().statusCode(OK.getStatusCode());
-
-        uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
-        uploadFile.prettyPrint();
-        uploadFile.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data.files[0].label", equalTo("data-2.csv"));
-
-        String fileId3 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id");
-
-        getTabularFails = UtilIT.getFileDataTables(fileId3, apiToken);
-        getTabularFails.prettyPrint();
-        getTabularFails.then().assertThat()
-                .statusCode(BAD_REQUEST.getStatusCode())
-                .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported")));
-
-        // The behavior of `"default": "-2"` is not documented in the guides
-        // but it acts like `"default": "0"` which disables ingest.
-        String unexpectedNegativeDefault = """
-{
-  "default": "-2"
-}
-""";
-
-        setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, unexpectedNegativeDefault);
-        setLimit.then().assertThat().statusCode(OK.getStatusCode());
-
-        uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
-        uploadFile.prettyPrint();
-        uploadFile.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data.files[0].label", equalTo("data-3.csv"));
-
-        String fileId4 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id");
-
-        getTabularFails = UtilIT.getFileDataTables(fileId4, apiToken);
-        getTabularFails.prettyPrint();
-        getTabularFails.then().assertThat()
-                .statusCode(BAD_REQUEST.getStatusCode())
-                .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported")));
-
-        // As the guides say, you MUST provide a string, not a JSON number.
-        // That is, `"123"` in quotes rather than `123` with no quotes.
-        // If you provide a number (no quotes) rather than a string,
-        // all ingest will be disabled and you'll see an error in server.log
-        // about how the system is misconfigured.
-        String invalidNonString = """
-{
-  "default": 987654321
-}
-""";
-
-        setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, invalidNonString);
-        setLimit.then().assertThat().statusCode(OK.getStatusCode());
-
-        uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
-        uploadFile.prettyPrint();
-        uploadFile.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data.files[0].label", equalTo("data-4.csv"));
-
-        String fileId5 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id");
-
-        getTabularFails = UtilIT.getFileDataTables(fileId5, apiToken);
-        getTabularFails.prettyPrint();
-        getTabularFails.then().assertThat()
-                .statusCode(BAD_REQUEST.getStatusCode())
-                .body("message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported")));
-
-        String defaultDisabledAndLargeCsvLimit = """
-{
-  "default": "0",
-  "csv": "123456"
-}
-""";
-
-        setLimit = UtilIT.setSetting(SettingsServiceBean.Key.TabularIngestSizeLimit, defaultDisabledAndLargeCsvLimit);
-        setLimit.then().assertThat().statusCode(OK.getStatusCode());
-
-        uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
-        uploadFile.prettyPrint();
-        uploadFile.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data.files[0].label", equalTo("data-5.csv"));
-
-        String fileId6 = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id");
-
-        getTabularWorks = UtilIT.getFileDataTables(fileId2, apiToken);
-        getTabularWorks.prettyPrint();
-        getTabularWorks.then().assertThat()
+    
+    @Nested
+    class IngestSizeLimits {
+        
+        static String apiToken;
+        static int datasetId;
+        @TempDir
+        static Path tempDir;
+        static String csvFileName = "data.csv";
+        static Path csvFile;
+        static final String csvData =
+            """
+            name,pounds,species,treats
+            Midnight,15,dog,milkbones
+            Tiger,17,cat,cat grass
+            Panther,21,cat,cat nip
+            """;
+        
+        @BeforeAll
+        static void setup() throws IOException {
+            // Create random user, collection and dataset to work with
+            Response createUser = UtilIT.createRandomUser();
+            createUser.then().assertThat().statusCode(OK.getStatusCode());
+            String username = UtilIT.getUsernameFromResponse(createUser);
+            apiToken = UtilIT.getApiTokenFromResponse(createUser);
+            Response makeSuperUser = UtilIT.setSuperuserStatus(username, true);
+            makeSuperUser.then().assertThat().statusCode(OK.getStatusCode());
+            
+            Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+            createDataverseResponse.prettyPrint();
+            String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+            Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+            createDatasetResponse.prettyPrint();
+            datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+            
+            // Create CSV datafile to work with
+            csvFile = tempDir.resolve(csvFileName);
+            java.nio.file.Files.writeString(csvFile, csvData, StandardCharsets.UTF_8);
+        }
+        
+        @AfterAll
+        static void teardown() {
+            // Remove the setting for test isolation purposes
+            Response removeLimit = UtilIT.deleteSetting(Key.TabularIngestSizeLimit);
+            removeLimit.then().assertThat().statusCode(OK.getStatusCode());
+        }
+        
+        static List<Arguments> configurations() {
+            return List.of(
+                // Too small for 134 chars
+                Arguments.of("{\"csv\": 50}", BAD_REQUEST, "message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))),
+                Arguments.of("{\"default\": 50.0}", BAD_REQUEST, "message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))),
+                
+                // The behavior of `"default": "-2"` is not documented in the guides
+                // but it acts like `"default": "0"` which disables ingest.
+                Arguments.of("{\"default\": -2}", BAD_REQUEST, "message", equalTo(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"))),
+                
+                // Large enough :-)
+                Arguments.of("{\"csv\": 123456}", OK, "data[0].varQuantity", equalTo(4)),
+                // Default is disabled, but exception for CSV
+                Arguments.of("{\"default\": 0,\"csv\": 123456}", OK, "data[0].varQuantity", equalTo(4))
+            );
+        }
+        
+        @ParameterizedTest
+        @MethodSource("configurations")
+        void testIngestSizeLimits(String ingestSizeLimitConfig, Status expectedStatus, String jsonPath, Matcher matcher) throws IOException {
+            // given
+            Response setLimit = UtilIT.setSetting(Key.TabularIngestSizeLimit, ingestSizeLimitConfig);
+            setLimit.then().assertThat().statusCode(OK.getStatusCode());
+            
+            // when
+            Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), csvFile.toString(), apiToken);
+            //uploadResponse.prettyPrint();
+            uploadResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
-                .body("data[0].varQuantity", equalTo(4));
-
-        Response removeLimit = UtilIT.deleteSetting(SettingsServiceBean.Key.TabularIngestSizeLimit);
-        removeLimit.then().assertThat().statusCode(OK.getStatusCode());
+                .body("data.files[0].label", equalTo(csvFileName));
+            String fileId = JsonPath.from(uploadResponse.body().asString()).getString("data.files[0].dataFile.id");
+            
+            // Wait for ingest to complete
+            assertTrue(UtilIT.sleepForLock(datasetId, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION),
+                "Failed test if Ingest Lock exceeds max duration");
+            
+            // then
+            Response getTabularFails = UtilIT.getFileDataTables(fileId, apiToken);
+            //getTabularFails.prettyPrint();
+            getTabularFails.then().assertThat()
+                .statusCode(expectedStatus.getStatusCode())
+                .body(jsonPath, matcher);
+            
+            // delete the file for the next test
+            UtilIT.deleteFile(Integer.valueOf(fileId), apiToken);
+        }
     }
 
+
     @Test
     public void testUningestFileViaApi() throws InterruptedException {
         Response createUser = UtilIT.createRandomUser();