FINERACT-2682: Fix bulk client upload failure when custom datatables …

fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java

@@ -34,18 +34,21 @@ public final class DatatableData implements Serializable {
     private final String entitySubType;
     @SuppressWarnings("unused")
     private final List<ResultsetColumnHeaderData> columnHeaderData;
+    @SuppressWarnings("unused")
+    private final boolean multiRow;
 
     public static DatatableData create(final String applicationTableName, final String registeredTableName, final String entitySubType,
-            final List<ResultsetColumnHeaderData> columnHeaderData) {
-        return new DatatableData(applicationTableName, registeredTableName, entitySubType, columnHeaderData);
+            final List<ResultsetColumnHeaderData> columnHeaderData, final boolean multiRow) {
+        return new DatatableData(applicationTableName, registeredTableName, entitySubType, columnHeaderData, multiRow);
     }
 
     private DatatableData(final String applicationTableName, final String registeredTableName, final String entitySubType,
-            final List<ResultsetColumnHeaderData> columnHeaderData) {
+            final List<ResultsetColumnHeaderData> columnHeaderData, final boolean multiRow) {
         this.applicationTableName = applicationTableName;
         this.registeredTableName = registeredTableName;
         this.entitySubType = entitySubType;
         this.columnHeaderData = columnHeaderData;
+        this.multiRow = multiRow;
 
     }
 
@@ -65,4 +68,12 @@ public String getRegisteredTableName() {
         return registeredTableName;
     }
 
+    public List<ResultsetColumnHeaderData> getColumnHeaderData() {
+        return columnHeaderData;
+    }
+
+    public boolean isMultiRow() {
+        return multiRow;
+    }
+
 }

fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java

@@ -149,6 +149,10 @@ public boolean hasColumnValues() {
         return columnValues != null && !columnValues.isEmpty();
     }
 
+    public List<ResultsetColumnValueData> getColumnValues() {
+        return columnValues;
+    }
+
     public boolean isColumnValueAllowed(final String match) {
         for (final ResultsetColumnValueData allowedValue : this.columnValues) {
             if (allowedValue.matches(match)) {

fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java

@@ -49,4 +49,12 @@ public boolean matches(final String match) {
     public boolean codeMatches(final Integer match) {
         return match.intValue() == this.id;
     }
+
+    public int getId() {
+        return id;
+    }
+
+    public String getValue() {
+        return value;
+    }
 }

fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java

@@ -271,7 +271,15 @@ public Object parseColumnValue(@NonNull ResultsetColumnHeaderData columnHeader,
                 }
                 return columnValue;
             } else if (columnHeader.isCodeLookupDisplayType()) {
-                final Integer codeLookup = Integer.valueOf(columnValue);
+                // Extract ID from display value format: "Label (ID)" if present
+                String extractedId = extractIdFromDisplayValue(columnValue);
+                Integer codeLookup;
+                try {
+                    codeLookup = Integer.valueOf(extractedId);
+                } catch (NumberFormatException e) {
+                    // If extraction didn't change the value or parsing failed, try original value
+                    codeLookup = Integer.valueOf(columnValue);
+                }
                 if (!columnHeader.isColumnCodeAllowed(codeLookup)) {
                     ApiParameterError error = ApiParameterError.parameterError("error.msg.invalid.columnValue",
                             "Value not found in Allowed Value list", columnHeader.getColumnName(), columnValue);
@@ -293,6 +301,17 @@ public Object parseColumnValue(@NonNull ResultsetColumnHeaderData columnHeader,
             return JsonParserHelper.convertDateTimeFrom(columnValue, columnHeader.getColumnName(), format, locale);
         }
         if (colType.isAnyIntegerType()) {
+            // Extract ID from display value format: "Label (ID)" if present
+            // This handles FK/INTEGER columns that show human-readable values in Excel
+            String extractedId = extractIdFromDisplayValue(columnValue);
+            if (!extractedId.equals(columnValue)) {
+                // ID was extracted, try to parse it
+                try {
+                    return Integer.parseInt(extractedId);
+                } catch (NumberFormatException e) {
+                    // If parsing fails, fall back to original conversion
+                }
+            }
             return helper.convertToInteger(columnValue, columnHeader.getColumnName(), locale);
         }
         if (colType.isDecimalType()) {
@@ -323,4 +342,28 @@ public String camelToSnake(final String camelStr) {
         return camelStr == null ? null
                 : camelStr.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase();
     }
+
+    /**
+     * Extracts numeric ID from strings ending with "(id)" format.
+     * Example: "Permanent (32)" -> "32"
+     * If the string doesn't match the pattern, returns the original value.
+     * 
+     * @param value The string value that may contain an ID in parentheses
+     * @return The extracted ID as a string, or the original value if no ID pattern is found
+     */
+    private static String extractIdFromDisplayValue(String value) {
+        if (value == null || value.trim().isEmpty()) {
+            return value;
+        }
+        // Match pattern: ".*(\d+)$" where the number is in parentheses at the end
+        String trimmed = value.trim();
+        int lastParenIndex = trimmed.lastIndexOf('(');
+        if (lastParenIndex > 0 && trimmed.endsWith(")")) {
+            String idPart = trimmed.substring(lastParenIndex + 1, trimmed.length() - 1);
+            if (idPart.matches("\\d+")) {
+                return idPart;
+            }
+        }
+        return value;
+    }
 }

fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java

@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.data;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.fineract.infrastructure.core.service.database.DatabaseType;
+import org.junit.jupiter.api.Test;
+
+class DatatableDataTest {
+
+    @Test
+    void testSingleRowDatatable() {
+        // Given: A single-row datatable (no "id" column)
+        String appTableName = "m_client";
+        String registeredTableName = "extra_client_details";
+        String entitySubType = null;
+        List<ResultsetColumnHeaderData> columnHeaders = new ArrayList<>();
+        // Single-row datatable has client_id as primary key, no "id" column
+        columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL));
+        columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL));
+        boolean multiRow = false;
+
+        // When: Creating DatatableData
+        DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType,
+                columnHeaders, multiRow);
+
+        // Then: It should be identified as single-row
+        assertFalse(datatableData.isMultiRow(), "Single-row datatable should return false for isMultiRow()");
+        assertEquals(registeredTableName, datatableData.getRegisteredTableName());
+        assertEquals(columnHeaders, datatableData.getColumnHeaderData());
+    }
+
+    @Test
+    void testMultiRowDatatable() {
+        // Given: A multi-row datatable (has "id" column)
+        String appTableName = "m_client";
+        String registeredTableName = "extra_family_details";
+        String entitySubType = null;
+        List<ResultsetColumnHeaderData> columnHeaders = new ArrayList<>();
+        // Multi-row datatable has "id" as primary key
+        columnHeaders.add(ResultsetColumnHeaderData.basic("id", "bigint", DatabaseType.POSTGRESQL));
+        columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL));
+        columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL));
+        boolean multiRow = true;
+
+        // When: Creating DatatableData
+        DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType,
+                columnHeaders, multiRow);
+
+        // Then: It should be identified as multi-row
+        assertTrue(datatableData.isMultiRow(), "Multi-row datatable should return true for isMultiRow()");
+        assertEquals(registeredTableName, datatableData.getRegisteredTableName());
+        assertEquals(columnHeaders, datatableData.getColumnHeaderData());
+    }
+
+    @Test
+    void testHasColumn() {
+        // Given: A datatable with columns
+        List<ResultsetColumnHeaderData> columnHeaders = new ArrayList<>();
+        columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL));
+        columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL));
+        DatatableData datatableData = DatatableData.create("m_client", "test_table", null, columnHeaders, false);
+
+        // When/Then: Checking for column existence
+        assertTrue(datatableData.hasColumn("client_id"), "Should find existing column");
+        assertTrue(datatableData.hasColumn("field1"), "Should find existing column");
+        assertFalse(datatableData.hasColumn("nonexistent"), "Should not find non-existent column");
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java

@@ -51,15 +51,55 @@ private ClientEntityConstants() {
     public static final int COUNTRY_COL = 24;// Y
     public static final int POSTAL_CODE_COL = 25;// Z
     public static final int IS_ACTIVE_ADDRESS_COL = 26;// AA
-    public static final int WARNING_COL = 26;// AA
     public static final int STATUS_COL = 27;// AB
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
+    public static final int WARNING_COL = 26;// AA
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int RELATIONAL_OFFICE_NAME_COL = 35;// AJ
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int RELATIONAL_OFFICE_OPENING_DATE_COL = 36;// AK
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_CONSTITUTION_COL = 37;// AL
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_CLIENT_CLASSIFICATION = 38;// AM
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_CLIENT_TYPES = 39;// AN
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_ADDRESS_TYPE = 40;// AO
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_STATE_PROVINCE = 41;// AP
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_COUNTRY = 42;// AQ
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_MAIN_BUSINESS_LINE = 43;// AR
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java

@@ -50,14 +50,50 @@ private ClientPersonConstants() {
     public static final int COUNTRY_COL = 23;// X
     public static final int POSTAL_CODE_COL = 24;// Y
     public static final int IS_ACTIVE_ADDRESS_COL = 25;// Z
-    public static final int WARNING_COL = 26;// AA
     public static final int STATUS_COL = 27;// AB
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
+    public static final int WARNING_COL = 26;// AA
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int RELATIONAL_OFFICE_NAME_COL = 35;// AJ
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int RELATIONAL_OFFICE_OPENING_DATE_COL = 36;// AK
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_GENDER_COL = 37;// AL
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_CLIENT_CLASSIFICATION_COL = 38;// AM
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_CLIENT_TYPES_COL = 39;// AN
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_ADDRESS_TYPE_COL = 40;// AO
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_STATE_PROVINCE_COL = 41;// AP
+    /**
+     * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead.
+     */
+    @Deprecated
     public static final int LOOKUP_COUNTRY_COL = 42;// AQ
 }

fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java

@@ -57,6 +57,7 @@ private TemplatePopulateImportConstants() {
     public static final String EMPLOYEE_SHEET_NAME = "Employee";
     public static final String ROLES_SHEET_NAME = "Roles";
     public static final String USER_SHEET_NAME = "Users";
+    public static final String CLIENT_LOOKUPS_SHEET_NAME = "_CLIENT_LOOKUPS";
 
     public static final int ROWHEADER_INDEX = 0;
     public static final short ROW_HEADER_HEIGHT = 500;