fix(config): support numeric values for TabularIngestSizeLimit, improve validation and logging

Enhanced `getTabularIngestSizeLimits` to accept numeric values in addition to strings for size limits. Improved validation to handle invalid types, numbers, or decimal values. Updated related tests and configuration documentation.

This was done because the limitation to long numbers as string was artificial. Users can choose which way they like best. Also, the data migration uses numbers, so this lead to errors.

Author: poikilotherm

doc/sphinx-guides/source/installation/config.rst

@@ -4517,7 +4517,11 @@ Using a JSON-based setting, you can set a global default and per-format limits f
 
 (In previous releases of Dataverse, a colon-separated form was used to specify per-format limits, such as ``:TabularIngestSizeLimit:Rdata``, but this is no longer supported. Now JSON is used.)
 
-The expected JSON is an object with key/value pairs like the following. Format names are case-insensitive, and all fields are optional. The size limits must be strings with double quotes around them (e.g. ``"10"``) rather than numbers (e.g. ``10``).
+The expected JSON is an object with key/value pairs like the following.
+Format names are case-insensitive, and all fields are optional (an empty JSON object equals not restricted).
+The size limits must be whole numbers, either presented as strings with double quotes around them (e.g. ``"10"``) or numeric values (e.g. ``10`` or ``10.0``).
+Note that decimal numbers like ``10.5`` are invalid.
+Any invalid setting will temporarily disable tabular ingest until corrected.
 
 .. code:: json
 

src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java

@@ -527,19 +527,30 @@ public Map<String, Long> getTabularIngestSizeLimits() {
                     limitsMap.put(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, -1L);
 
                     for (String formatName : limits.keySet()) {
-                        // We deliberatly do not validate the formatNames here for backward compatibility.
-                        // But we transform to lowercase here, so the casing doesn't matter for lookups.
                         String lowercaseFormatName = formatName.toLowerCase();
 
                         try {
-                            Long sizeOption = Long.valueOf(limits.getString(formatName));
+                            JsonValue value = limits.get(formatName);
+                            Long sizeOption;
+                            
+                            // We want to be able to use either numbers or string values, so detect which one it is.
+                            if (value.getValueType() == JsonValue.ValueType.STRING) {
+                                sizeOption = Long.valueOf(limits.getString(formatName));
+                            } else if (value.getValueType() == JsonValue.ValueType.NUMBER) {
+                                sizeOption = limits.getJsonNumber(formatName).longValueExact();
+                            } else {
+                                logger.warning("Invalid value type for format " + formatName + ": expected string or number");
+                                logger.warning("Disabling all tabular ingest completely until fixed!");
+                                return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L);
+                            }
+                            
                             limitsMap.put(lowercaseFormatName, sizeOption);
-                        } catch (ClassCastException cce) {
-                            logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long from JSON integer. You must provide the long number as string (use quotes) for format " + formatName);
+                        } catch (NumberFormatException nfe) {
+                            logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long for format " + formatName + " (not a valid number)");
                             logger.warning("Disabling all tabular ingest completely until fixed!");
                             return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L);
-                        } catch (NumberFormatException nfe) {
-                            logger.warning("Could not convert " + SettingsServiceBean.Key.TabularIngestSizeLimit + " to long for format " + formatName + " (not a number)");
+                        } catch (ArithmeticException ae) {
+                            logger.warning("Number too large or has fractional part for format " + formatName);
                             logger.warning("Disabling all tabular ingest completely until fixed!");
                             return Map.of(TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY, 0L);
                         }

src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java

@@ -8,9 +8,11 @@
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.beans.factory.annotation.Value;
 
 import java.util.Map;
 
@@ -202,10 +204,10 @@ void testGetTabularIngestSizeLimitsWithSingleInvalidValue() {
         assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY));
     }
 
-    @Test
-    void testGetTabularIngestSizeLimitsWithJsonButUnsupportedJsonInt() {
+    @ParameterizedTest
+    @ValueSource(strings = {"", "{ invalid: }"})
+    void testGetTabularIngestSizeLimitsWithInvalidJson(String invalidJson) {
         // given
-        String invalidJson = "{\"default\": 0}";
         doReturn(invalidJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
 
         // when
@@ -216,10 +218,10 @@ void testGetTabularIngestSizeLimitsWithJsonButUnsupportedJsonInt() {
         assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY));
     }
 
-    @Test
-    void testGetTabularIngestSizeLimitsWithInvalidJson() {
+    @ParameterizedTest
+    @ValueSource(strings = {"[]", "{[{\"tsv\": 1}]}", "{ \"invalid\": \"foobar\" }", "{ \"tsv\": 1.5}", "{ \"tsv\": \"1.0\"}"})
+    void testGetTabularIngestSizeLimitsWithInvalidNumbersInValidJson(String invalidJson) {
         // given
-        String invalidJson = "{invalid:}";
         doReturn(invalidJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
 
         // when
@@ -230,17 +232,33 @@ void testGetTabularIngestSizeLimitsWithInvalidJson() {
         assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY));
     }
 
-    @Test
-    void testGetTabularIngestSizeLimitsWithInvalidNumberInValidJson() {
+    @ParameterizedTest
+    @ValueSource(strings = {"{ \"tSv\": 1.0 }", "{ \"tsv\": \"1\"}", "{ \"tsv\": 1 }"})
+    void testGetTabularIngestSizeLimitsWithValidNumbersInValidJson(String validConfig) {
         // given
-        String invalidJson = "{\"csv\": \"this-is-not-a-number\", \"tSv\": \"10000\"}";
-        doReturn(invalidJson).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
+        doReturn(validConfig).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
 
         // when
         Map<String, Long> result = systemConfig.getTabularIngestSizeLimits();
 
         // then
-        assertEquals(1, result.size());
-        assertEquals(0L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY));
+        assertEquals(2, result.size());
+        assertEquals(-1L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY));
+        assertEquals(1L, (long) result.get("tsv"));
+    }
+    
+    @ParameterizedTest
+    @ValueSource(strings = {"{ \"tSv\": 429496729600.0 }", "{ \"tsv\": \"429496729600\"}", "{ \"tsv\": 429496729600 }"})
+    void testGetTabularIngestSizeLimitsWithValidLongNumbers(String validConfig) {
+        // given
+        doReturn(validConfig).when(settingsService).getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
+        
+        // when
+        Map<String, Long> result = systemConfig.getTabularIngestSizeLimits();
+        
+        // then
+        assertEquals(2, result.size());
+        assertEquals(-1L, (long) result.get(SystemConfig.TABULAR_INGEST_SIZE_LIMITS_DEFAULT_KEY));
+        assertEquals(429496729600L, (long) result.get("tsv"));
     }
 }