Merge pull request #11779 from QualitativeDataRepository/IQSS-11776-indexing_new_files

IQSS/11776 indexing new files

Author: ofahimIQSS

doc/release-notes/11776- index fix.md

@@ -0,0 +1 @@
+A bug, introduced in v6.7, that caused files in draft versions that were added after the initial dataset version was published, has been fixed.

src/main/java/edu/harvard/iq/dataverse/FileMetadata.java

@@ -94,9 +94,10 @@
                 "        )",
                 resultSetMapping = "IdToLongMapping"
     )
+/* When this mapping was to Long.class, Postgres was still returning an Integer, causing indexing failures - see #11776 */ 
 @SqlResultSetMapping(
         name = "IdToLongMapping",
-        columns = @ColumnResult(name = "id", type = Long.class)
+        columns = @ColumnResult(name = "id", type = Integer.class)
     )
 @Entity
 public class FileMetadata implements Serializable {

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -69,6 +69,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Future;
@@ -1428,7 +1429,31 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                 query.setParameter(1, dataset.getReleasedVersion().getId());
                 query.setParameter(2, datasetVersion.getId());
 
-                changedFileMetadataIds.addAll(query.getResultList());
+                /*
+                 * When the query was configured to return Long, it was returning Integer. The query has been changed to return Integer now. The code here is robust if that changes in the future.
+                 */
+                List<Object> queryResults = query.getResultList();
+                for (Object result : queryResults) {
+                    if (result != null) {
+                        // Ensure we're adding Long objects to the list
+                        if (result instanceof Integer intResult) {
+                            logger.finest("Converted Integer result to Long: " + result);
+                            changedFileMetadataIds.add(Long.valueOf(intResult));
+                        } else if (result instanceof Long longResult) {
+                            // Already a Long, add directly
+                            logger.finest("Added existing Long to list: " + result);
+                            changedFileMetadataIds.add(longResult);
+                        } else {
+                            // If it's not a Long, convert it to one via String
+                            try {
+                                changedFileMetadataIds.add(Long.valueOf(result.toString()));
+                                logger.finest("Converted non-Long result to Long: " + result + " of type " + result.getClass().getName());
+                            } catch (NumberFormatException e) {
+                                logger.warning("Could not convert query result to Long: " + result);
+                            }
+                        }
+                    }
+                }
                 logger.fine(
                         "We are indexing a draft version of a dataset that has a released version. We'll be checking file metadatas if they are exact clones of the released versions.");
             } else if (datasetVersion.isDraft()) {
@@ -1502,19 +1527,25 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                 if (indexThisMetadata && (isReleasedVersion || changedFileMetadataIds.contains(fileMetadata.getId()))) {
                     indexThisFile = true;
                 } else if (indexThisMetadata) {
+                    // Draft version, file is not new or all file metadata matches the released version
+                    // The only thing left to check is variable-level metadata, index if there is a difference
                     logger.fine("Checking if this file metadata is a duplicate.");
                     FileMetadata getFromMap = fileMap.get(datafile.getId());
                     if (getFromMap != null) {
                         if (!VariableMetadataUtil.compareVariableMetadata(getFromMap, fileMetadata)) {
                             indexThisFile = true;
                             logger.fine("This file metadata hasn't changed since the released version; skipping indexing.");
                         }
+                    } else {
+                        logger.warning("File is not in released version when trying to compare variable metadata, fileId: " + datafile.getId());
                     }
                 }
                 if (indexThisFile) {
 
                     SolrInputDocument datafileSolrInputDocument = new SolrInputDocument();
                     Long fileEntityId = datafile.getId();
+                    logger.finest("Indexing file " + fileEntityId);
+
                     datafileSolrInputDocument.addField(SearchFields.ENTITY_ID, fileEntityId);
                     datafileSolrInputDocument.addField(SearchFields.DATAVERSE_VERSION_INDEXED_BY, dataverseVersion);
                     datafileSolrInputDocument.addField(SearchFields.IDENTIFIER, fileEntityId);

src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java

@@ -2328,5 +2328,97 @@ public void testShowCollections() {
                 ));
 
     }
+    
+    @Test
+    public void testFileAddedAfterPublicationIsIndexed() {
+        // Create user
+        Response createUser = UtilIT.createRandomUser();
+        createUser.prettyPrint();
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        // Create dataverse
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        // Create dataset
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.prettyPrint();
+        Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
+        String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse);
+
+        // Publish dataverse and dataset
+        Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken);
+        publishDataverse.prettyPrint();
+        publishDataverse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDataset.prettyPrint();
+        publishDataset.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // Verify no files in search results initially
+        Response searchBeforeFileUpload = UtilIT.search("parentId:" + datasetId, apiToken);
+        searchBeforeFileUpload.prettyPrint();
+        searchBeforeFileUpload.then().assertThat()
+                .body("data.total_count", CoreMatchers.is(0))
+                .statusCode(OK.getStatusCode());
+
+        // Upload a file after publication
+        String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";
+        Response uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
+        uploadFileResponse.prettyPrint();
+        uploadFileResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // Get file ID from the upload response
+        Integer fileId = JsonPath.from(uploadFileResponse.getBody().asString()).getInt("data.files[0].dataFile.id");
+
+        // Wait for indexing to complete
+        String searchQuery = "entityId:" + fileId;
+        assertTrue(UtilIT.sleepForSearch(searchQuery, apiToken, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), 
+                   "Failed test if search exceeds max duration " + searchQuery);
+
+        // Search for the file and verify it's indexed
+        Response searchAfterFileUpload = UtilIT.search(searchQuery, apiToken);
+        searchAfterFileUpload.prettyPrint();
+        searchAfterFileUpload.then().assertThat()
+                .body("data.total_count", CoreMatchers.is(1))
+                .body("data.items[0].name", is("dataverseproject.png"))
+                .body("data.items[0].file_content_type", CoreMatchers.is("image/png"))
+                .statusCode(OK.getStatusCode());
+
+        // Clean up - delete dataset, dataverse, and user
+        
+        //Superuser to delete published dataset
+        Response makeSuperUser = UtilIT.setSuperuserStatus(username, true);
+        assertEquals(200, makeSuperUser.getStatusCode());
+        
+        // Delete the dataset
+        Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken);
+        deleteDatasetResponse.prettyPrint();
+        deleteDatasetResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        try {
+            // give the bag time to generate
+            Thread.sleep(3000);
+        } catch (InterruptedException ex) {
+        }
+
+        // Delete the dataverse
+        Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
+        deleteDataverseResponse.prettyPrint();
+        deleteDataverseResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // Delete the user
+        Response deleteUserResponse = UtilIT.deleteUser(username);
+        deleteUserResponse.prettyPrint();
+        deleteUserResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+    }
 
 }