fix(ContentImport) Import multiple versions of 1 single content Refs:dotCMS#32008 (dotCMS#32175)

Here, we're replacing the use of the index to retrieve existing content
for the database.
The use of the index brings stale info, and there is no guarantee that
this info is up to date while in a transaction

Author: fabrizzio-dotCMS

dotCMS/src/main/java/com/dotmarketing/util/ImportUtil.java

@@ -1,9 +1,12 @@
 package com.dotmarketing.util;
 
+import static com.dotmarketing.portlets.contentlet.model.Contentlet.IDENTIFIER_KEY;
+import static com.dotmarketing.portlets.contentlet.model.Contentlet.STRUCTURE_INODE_KEY;
 import static com.dotmarketing.util.importer.HeaderValidationCodes.HEADERS_NOT_FOUND;
 import static com.dotmarketing.util.importer.ImportLineValidationCodes.LANGUAGE_NOT_FOUND;
 
 import com.dotcms.content.elasticsearch.util.ESUtils;
+import com.dotcms.contenttype.business.ContentTypeAPI;
 import com.dotcms.contenttype.model.field.BinaryField;
 import com.dotcms.contenttype.model.field.HostFolderField;
 import com.dotcms.contenttype.model.type.BaseContentType;
@@ -18,7 +21,6 @@
 import com.dotmarketing.beans.Identifier;
 import com.dotmarketing.beans.Permission;
 import com.dotmarketing.business.APILocator;
-import com.dotmarketing.business.CacheLocator;
 import com.dotmarketing.business.DotValidationException;
 import com.dotmarketing.business.PermissionAPI;
 import com.dotmarketing.cache.FieldsCache;
@@ -83,6 +85,7 @@
 import com.liferay.portal.model.User;
 import com.liferay.util.StringPool;
 import io.vavr.Lazy;
+import io.vavr.control.Try;
 import java.io.File;
 import java.io.IOException;
 import java.io.Reader;
@@ -110,7 +113,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
-import org.apache.juli.logging.Log;
 
 /**
  * Provides utility methods to import content into dotCMS. The data source is a
@@ -134,6 +136,7 @@ public class ImportUtil {
     private final static HostAPI hostAPI = APILocator.getHostAPI();
     private final static FolderAPI folderAPI = APILocator.getFolderAPI();
     private final static WorkflowAPI workflowAPI = APILocator.getWorkflowAPI();
+    private final static ContentTypeAPI contentTypeAPI = APILocator.getContentTypeAPI(APILocator.systemUser());
 
     public static final String KEY_WARNINGS = "warnings";
     public static final String KEY_ERRORS = "errors";
@@ -294,8 +297,12 @@ public static HashMap<String, List<String>> importFile(
     public static ImportResult importFileResult(final ImportFileParams params)
             throws DotRuntimeException, DotDataException {
 
-        Structure contentType = CacheLocator.getContentTypeCache()
-                .getStructureByInode(params.contentTypeInode());
+        final ContentType type = Try.of(()->contentTypeAPI.find(params.contentTypeInode())).getOrNull();
+        if (type == null) {
+            throw new DotDataValidationException("Content type not found for inode: " + params.contentTypeInode());
+        }
+        Structure contentType = new StructureTransformer(type).asStructure();
+
         List<Permission> contentTypePermissions = permissionAPI.getPermissions(contentType);
         List<UniqueFieldBean> uniqueFieldBeans = new ArrayList<>();
 
@@ -1666,10 +1673,8 @@ private static void importLine(
         fieldResults.categories().forEach(resultBuilder::addCategory);
         uniqueFieldBeans.addAll(fieldResults.uniqueFields());
 
-        final Map<Integer, Object> values = new HashMap<>();
-        final Set<Category> categories = new HashSet<>();
-        values.putAll(fieldResults.values());
-        categories.addAll(fieldResults.categories());
+        final Map<Integer, Object> values = new HashMap<>(fieldResults.values());
+        final Set<Category> categories = new HashSet<>(fieldResults.categories());
 
         if (fieldResults.ignoreLine()) {
             resultBuilder.setIgnoreLine(true);
@@ -1808,7 +1813,7 @@ private static void importLine(
             }
         }
 
-        ProcessedContentResult processResult = processContent(
+        final ProcessedContentResult processResult = processContent(
                 lineNumber,
                 contentlets,
                 resultBuilder.isNewContent(),
@@ -2572,7 +2577,7 @@ private static ContentletSearchResult searchExistingContentlets(
 
         if (UtilMethods.isSet(identifier)) {
             contentlets.addAll(
-                    searchByIdentifier(identifier, contentType, user));
+                    searchByIdentifierFromDB(identifier, contentType, user));
         } else if (urlValue != null && keyFields.isEmpty()) {
             // For HTMLPageAsset, we need to search by URL to math existing pages
             contentlets.addAll(searchByUrl(contentType, urlValue, siteAndFolder, language, user));
@@ -2608,29 +2613,27 @@ private static ContentletSearchResult searchExistingContentlets(
      * @throws DotSecurityException If the user does not have permission to access the requested
      *                              contentlet.
      */
-    private static List<Contentlet> searchByIdentifier(
+    private static List<Contentlet> searchByIdentifierFromDB(
             final String identifier,
             final Structure contentType,
             final User user
     ) throws DotDataException, DotSecurityException {
 
-        StringBuilder query = new StringBuilder()
-                .append("+structureName:").append(contentType.getVelocityVarName())
-                .append(" +working:true +deleted:false")
-                .append(" +identifier:").append(identifier);
+        final Contentlet contentlet = new Contentlet(Map.of(
+                IDENTIFIER_KEY, identifier,
+                STRUCTURE_INODE_KEY, contentType.getInode()
+        ));
 
-        List<ContentletSearch> contentsSearch = conAPI.searchIndex(query.toString(), 0, -1, null,
-                user, true);
-
-        if (contentsSearch == null || contentsSearch.isEmpty()) {
+        final List<Contentlet> allLanguages = conAPI.getAllLanguages(contentlet, false, user, true);
+        if (allLanguages == null || allLanguages.isEmpty()) {
             throw ImportLineException.builder()
                     .message("Content not found with identifier")
                     .code(ImportLineValidationCodes.CONTENT_NOT_FOUND.name())
                     .invalidValue(identifier)
                     .build();
         }
 
-        return convertSearchResults(contentsSearch, user);
+        return allLanguages.stream().map(Contentlet::new).collect(Collectors.toList());
     }
 
     /**
@@ -3051,7 +3054,7 @@ private static ProcessedContentResult processContent(
             final String[] line
     ) throws DotDataException, DotSecurityException, IOException, LanguageException {
 
-        ProcessedContentResultBuilder resultBuilder = new ProcessedContentResultBuilder();
+        final ProcessedContentResultBuilder resultBuilder = new ProcessedContentResultBuilder();
 
         for (Contentlet cont : contentlets) {
 
@@ -3333,14 +3336,11 @@ private static void saveContent(
 
         cont.setLowIndexPriority(true);
 
-        final var validationResult = validateWorkflowExecution(lineNumber, wfActionId, cont, user,
-                resultBuilder);
-        if (validationResult.getLeft()) {
-            executeWorkflowAction(cont, categories, validationResult.getRight(), relationships,
-                    user);
+        final var validationResult = validateWorkflowExecution(lineNumber, wfActionId, cont, user, resultBuilder);
+        if (Boolean.TRUE.equals(validationResult.getLeft())) {
+            cont = executeWorkflowAction(cont, categories, validationResult.getRight(), relationships, user);
         } else {
-            cont = runWorkflowIfCould(user, contentTypePermissions, categories, cont,
-                    relationships);
+            cont = runWorkflowIfCould(user, contentTypePermissions, categories, cont, relationships);
         }
 
         processTagFields(cont, headers, values, siteAndFolder);
@@ -3623,7 +3623,7 @@ private static Pair<Boolean, WorkflowAction> validateWorkflowExecution(
      * @throws DotDataException     If an error occurs during workflow execution or processing.
      * @throws DotSecurityException If a security-related exception occurs during execution.
      */
-    private static void executeWorkflowAction(
+    private static Contentlet executeWorkflowAction(
             final Contentlet cont,
             final List<Category> categories,
             final WorkflowAction executeWfAction,
@@ -3635,17 +3635,17 @@ private static void executeWorkflowAction(
         cont.setBoolProperty(Contentlet.SKIP_RELATIONSHIPS_VALIDATION,
                 relationships == null || relationships.getRelationshipsRecords().isEmpty());
 
-        workflowAPI.fireContentWorkflow(cont,
-                new ContentletDependencies.Builder()
-                        .respectAnonymousPermissions(Boolean.FALSE)
-                        .modUser(user)
-                        .relationships(relationships)
-                        .workflowActionId(executeWfAction.getId())
-                        .workflowActionComments("")
-                        .workflowAssignKey("")
-                        .categories(categories)
-                        .generateSystemEvent(Boolean.FALSE)
-                        .build());
+        return workflowAPI.fireContentWorkflow(cont,
+                    new ContentletDependencies.Builder()
+                            .respectAnonymousPermissions(Boolean.FALSE)
+                            .modUser(user)
+                            .relationships(relationships)
+                            .workflowActionId(executeWfAction.getId())
+                            .workflowActionComments("")
+                            .workflowAssignKey("")
+                            .categories(categories)
+                            .generateSystemEvent(Boolean.FALSE)
+                            .build());
     }
 
     /**

dotcms-integration/src/test/java/com/dotcms/util/ImportUtilTest.java

@@ -1,6 +1,7 @@
 package com.dotcms.util;
 
 import static com.dotcms.util.CollectionsUtils.list;
+import static com.dotmarketing.portlets.workflows.business.SystemWorkflowConstants.WORKFLOW_PUBLISH_ACTION_ID;
 import static com.dotmarketing.util.importer.ImportLineValidationCodes.INVALID_LOCATION;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -3235,6 +3236,34 @@ public void importFile_withHighGranularityAndStopOnError_shouldStopAtFirstInvali
         }
     }
 
+    /**
+     * Helper method to perform the import and basic validation
+     * @param contentType
+     * @param titleField
+     * @param reader
+     * @param stopOnError
+     * @param commitGranularity
+     * @return
+     * @throws IOException
+     * @throws DotDataException
+     */
+    private ImportResult importAndValidate(
+            final ContentType contentType,
+            final com.dotcms.contenttype.model.field.Field titleField,
+            final Reader reader,
+            final boolean stopOnError,
+            final int commitGranularity
+    ) throws IOException, DotDataException{
+        return importAndValidate(
+                contentType,
+                titleField,
+                reader,
+                stopOnError,
+                commitGranularity,
+                null
+        );
+    }
+
     /**
      * Helper method to perform the import and basic validation
      */
@@ -3243,7 +3272,9 @@ private ImportResult importAndValidate(
             final com.dotcms.contenttype.model.field.Field titleField,
             final Reader reader,
             final boolean stopOnError,
-            final int commitGranularity) throws IOException, DotDataException {
+            final int commitGranularity,
+            final String workflowActionId
+    ) throws IOException, DotDataException {
 
         CsvReader csvreader = new CsvReader(reader);
         csvreader.setSafetySwitch(false);
@@ -3260,6 +3291,7 @@ private ImportResult importAndValidate(
                 .request(getHttpRequest())
                 .stopOnError(stopOnError)
                 .commitGranularityOverride(commitGranularity)
+                .workflowActionId(workflowActionId)
                 .build();
         return ImportUtil.importFileResult(importFileParams);
     }
@@ -3473,4 +3505,68 @@ public void testingImportPreviewWithUniqueFields() throws DotSecurityException,
             ESContentletAPIImpl.setFeatureFlagDbUniqueFieldValidation(oldEnabledDataBaseValidation);
         }
     }
+
+    /**
+     * Method to test: {@link ImportUtil#importFile(Long, String, String, String[], boolean, boolean, User, long, String[], CsvReader, int, int, Reader, String, HttpServletRequest)}
+     * When: We have one contentlet with identifier "A" and we try to import 10 contentlets with the same identifier but different title
+     *  We should end up with 10 contentlets with the same identifier and different titles and inode as we are importing versions
+     * @throws DotSecurityException
+     * @throws DotDataException
+     * @throws IOException
+     */
+    @Test
+    public void importVersionsSharingSameIdentifier()
+            throws DotSecurityException, DotDataException, IOException {
+
+        String contentTypeName = "ImportFileManyItemsSharingIdentifier_" + System.currentTimeMillis();
+        String contentTypeVarName = contentTypeName.replaceAll("_", "Var_");
+        com.dotcms.contenttype.model.field.Field titleField = new FieldDataGen()
+                .name("title")
+                .velocityVarName("title")
+                .type(TextField.class)
+                .next();
+
+        ContentType contentType = new ContentTypeDataGen()
+                .name(contentTypeName)
+                .velocityVarName(contentTypeVarName)
+                .host(APILocator.systemHost())
+                .fields(List.of(titleField))
+                .nextPersisted();
+
+        final Contentlet contentlet = new ContentletDataGen(contentType)
+                .host(defaultSite)
+                .setProperty(titleField.variable(), "Here we go 0")
+                .nextPersisted();
+
+        final ContentType saved = contentTypeApi.find(contentType.inode());
+        titleField = saved.fields().get(0);
+
+        final Reader reader = createTempFile("identifier,title \r\n" +
+                contentlet.getIdentifier()+ ", Here we go 10" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 9" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 8" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 7" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 6" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 5" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 4" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 3" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 2" + "\r\n" +
+                contentlet.getIdentifier()+ ", Here we go 1" + "\r\n"
+        );
+
+        final ImportResult result = importAndValidate(contentType, titleField, reader, true, 1, WORKFLOW_PUBLISH_ACTION_ID);
+        // Validate results
+        final List<Contentlet> savedData = contentletAPI.findByStructure(contentType.inode(), user, false, 0, 0);
+        assertNotNull(savedData);
+
+        // Should only have rows before the error (1)
+        assertEquals(1, savedData.size());
+        final Identifier identifier = new Identifier(contentlet.getIdentifier());
+        final List<Contentlet> allVersions = contentletAPI.findAllVersions(identifier, user, false);
+        assertEquals(11, allVersions.size());
+
+        final ResultData data = result.data().orElse(null);
+        assertNotNull(data);
+    }
+
 }