Skip to content

Commit 2b050dd

Browse files
GPortasGPortas
committed
Merge branch 'develop' of github.com:IQSS/dataverse into 11703-return-isDefault-property-get-dataset-templates
2 parents 79b21a4 + 338dfa2 commit 2b050dd

File tree

6 files changed

+138
-7
lines changed

6 files changed

+138
-7
lines changed

‎doc/release-notes/11776- index fix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
A bug, introduced in v6.7, that caused files in draft versions that were added after the initial dataset version was published, has been fixed.

‎src/main/java/edu/harvard/iq/dataverse/FileMetadata.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,10 @@
9494
" )",
9595
resultSetMapping = "IdToLongMapping"
9696
)
97+
/* When this mapping was to Long.class, Postgres was still returning an Integer, causing indexing failures - see #11776 */
9798
@SqlResultSetMapping(
9899
name = "IdToLongMapping",
99-
columns = @ColumnResult(name = "id", type = Long.class)
100+
columns = @ColumnResult(name = "id", type = Integer.class)
100101
)
101102
@Entity
102103
public class FileMetadata implements Serializable {

‎src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -334,12 +334,16 @@ private void updateParentDataversesSubjectsField(Dataset savedDataset, CommandCo
334334
while (dv != null) {
335335
boolean newSubjectsAdded = false;
336336
for (ControlledVocabularyValue cvv : dsf.getControlledVocabularyValues()) {
337-
if (!dv.getDataverseSubjects().contains(cvv)) {
338-
logger.fine("dv "+dv.getAlias()+" does not have subject "+cvv.getStrValue());
339-
newSubjectsAdded = true;
340-
dv.getDataverseSubjects().add(cvv);
337+
if (!cvv.getStrValue().equals(DatasetField.NA_VALUE)) {
338+
if (!dv.getDataverseSubjects().contains(cvv)) {
339+
logger.fine("dv "+dv.getAlias()+" does not have subject "+cvv.getStrValue());
340+
newSubjectsAdded = true;
341+
dv.getDataverseSubjects().add(cvv);
342+
} else {
343+
logger.fine("dv "+dv.getAlias()+" already has subject "+cvv.getStrValue());
344+
}
341345
} else {
342-
logger.fine("dv "+dv.getAlias()+" already has subject "+cvv.getStrValue());
346+
logger.fine("Subject is not recognized : " + cvv.getStrValue());
343347
}
344348
}
345349
if (newSubjectsAdded) {

‎src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.List;
7070
import java.util.Locale;
7171
import java.util.Map;
72+
import java.util.Objects;
7273
import java.util.Set;
7374
import java.util.concurrent.ConcurrentHashMap;
7475
import java.util.concurrent.Future;
@@ -1428,7 +1429,31 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
14281429
query.setParameter(1, dataset.getReleasedVersion().getId());
14291430
query.setParameter(2, datasetVersion.getId());
14301431

1431-
changedFileMetadataIds.addAll(query.getResultList());
1432+
/*
1433+
* When the query was configured to return Long, it was returning Integer. The query has been changed to return Integer now. The code here is robust if that changes in the future.
1434+
*/
1435+
List<Object> queryResults = query.getResultList();
1436+
for (Object result : queryResults) {
1437+
if (result != null) {
1438+
// Ensure we're adding Long objects to the list
1439+
if (result instanceof Integer intResult) {
1440+
logger.finest("Converted Integer result to Long: " + result);
1441+
changedFileMetadataIds.add(Long.valueOf(intResult));
1442+
} else if (result instanceof Long longResult) {
1443+
// Already a Long, add directly
1444+
logger.finest("Added existing Long to list: " + result);
1445+
changedFileMetadataIds.add(longResult);
1446+
} else {
1447+
// If it's not a Long, convert it to one via String
1448+
try {
1449+
changedFileMetadataIds.add(Long.valueOf(result.toString()));
1450+
logger.finest("Converted non-Long result to Long: " + result + " of type " + result.getClass().getName());
1451+
} catch (NumberFormatException e) {
1452+
logger.warning("Could not convert query result to Long: " + result);
1453+
}
1454+
}
1455+
}
1456+
}
14321457
logger.fine(
14331458
"We are indexing a draft version of a dataset that has a released version. We'll be checking file metadatas if they are exact clones of the released versions.");
14341459
} else if (datasetVersion.isDraft()) {
@@ -1502,19 +1527,25 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
15021527
if (indexThisMetadata && (isReleasedVersion || changedFileMetadataIds.contains(fileMetadata.getId()))) {
15031528
indexThisFile = true;
15041529
} else if (indexThisMetadata) {
1530+
// Draft version, file is not new or all file metadata matches the released version
1531+
// The only thing left to check is variable-level metadata, index if there is a difference
15051532
logger.fine("Checking if this file metadata is a duplicate.");
15061533
FileMetadata getFromMap = fileMap.get(datafile.getId());
15071534
if (getFromMap != null) {
15081535
if (!VariableMetadataUtil.compareVariableMetadata(getFromMap, fileMetadata)) {
15091536
indexThisFile = true;
15101537
logger.fine("This file metadata hasn't changed since the released version; skipping indexing.");
15111538
}
1539+
} else {
1540+
logger.warning("File is not in released version when trying to compare variable metadata, fileId: " + datafile.getId());
15121541
}
15131542
}
15141543
if (indexThisFile) {
15151544

15161545
SolrInputDocument datafileSolrInputDocument = new SolrInputDocument();
15171546
Long fileEntityId = datafile.getId();
1547+
logger.finest("Indexing file " + fileEntityId);
1548+
15181549
datafileSolrInputDocument.addField(SearchFields.ENTITY_ID, fileEntityId);
15191550
datafileSolrInputDocument.addField(SearchFields.DATAVERSE_VERSION_INDEXED_BY, dataverseVersion);
15201551
datafileSolrInputDocument.addField(SearchFields.IDENTIFIER, fileEntityId);

‎src/main/resources/db/migration/V6.7.1.2.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- remove dataversesubjects entries with a controlledvocabularyvalue_id matching "N/A"
2+
DELETE FROM dataversesubjects WHERE controlledvocabularyvalue_id = 1 ;

‎src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2328,5 +2328,97 @@ public void testShowCollections() {
23282328
));
23292329

23302330
}
2331+
2332+
@Test
2333+
public void testFileAddedAfterPublicationIsIndexed() {
2334+
// Create user
2335+
Response createUser = UtilIT.createRandomUser();
2336+
createUser.prettyPrint();
2337+
String username = UtilIT.getUsernameFromResponse(createUser);
2338+
String apiToken = UtilIT.getApiTokenFromResponse(createUser);
2339+
2340+
// Create dataverse
2341+
Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
2342+
createDataverseResponse.prettyPrint();
2343+
String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
2344+
2345+
// Create dataset
2346+
Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
2347+
createDatasetResponse.prettyPrint();
2348+
Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);
2349+
String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse);
2350+
2351+
// Publish dataverse and dataset
2352+
Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken);
2353+
publishDataverse.prettyPrint();
2354+
publishDataverse.then().assertThat()
2355+
.statusCode(OK.getStatusCode());
2356+
2357+
Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
2358+
publishDataset.prettyPrint();
2359+
publishDataset.then().assertThat()
2360+
.statusCode(OK.getStatusCode());
2361+
2362+
// Verify no files in search results initially
2363+
Response searchBeforeFileUpload = UtilIT.search("parentId:" + datasetId, apiToken);
2364+
searchBeforeFileUpload.prettyPrint();
2365+
searchBeforeFileUpload.then().assertThat()
2366+
.body("data.total_count", CoreMatchers.is(0))
2367+
.statusCode(OK.getStatusCode());
2368+
2369+
// Upload a file after publication
2370+
String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";
2371+
Response uploadFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
2372+
uploadFileResponse.prettyPrint();
2373+
uploadFileResponse.then().assertThat()
2374+
.statusCode(OK.getStatusCode());
2375+
2376+
// Get file ID from the upload response
2377+
Integer fileId = JsonPath.from(uploadFileResponse.getBody().asString()).getInt("data.files[0].dataFile.id");
2378+
2379+
// Wait for indexing to complete
2380+
String searchQuery = "entityId:" + fileId;
2381+
assertTrue(UtilIT.sleepForSearch(searchQuery, apiToken, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION),
2382+
"Failed test if search exceeds max duration " + searchQuery);
2383+
2384+
// Search for the file and verify it's indexed
2385+
Response searchAfterFileUpload = UtilIT.search(searchQuery, apiToken);
2386+
searchAfterFileUpload.prettyPrint();
2387+
searchAfterFileUpload.then().assertThat()
2388+
.body("data.total_count", CoreMatchers.is(1))
2389+
.body("data.items[0].name", is("dataverseproject.png"))
2390+
.body("data.items[0].file_content_type", CoreMatchers.is("image/png"))
2391+
.statusCode(OK.getStatusCode());
2392+
2393+
// Clean up - delete dataset, dataverse, and user
2394+
2395+
//Superuser to delete published dataset
2396+
Response makeSuperUser = UtilIT.setSuperuserStatus(username, true);
2397+
assertEquals(200, makeSuperUser.getStatusCode());
2398+
2399+
// Delete the dataset
2400+
Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken);
2401+
deleteDatasetResponse.prettyPrint();
2402+
deleteDatasetResponse.then().assertThat()
2403+
.statusCode(OK.getStatusCode());
2404+
2405+
try {
2406+
// give the bag time to generate
2407+
Thread.sleep(3000);
2408+
} catch (InterruptedException ex) {
2409+
}
2410+
2411+
// Delete the dataverse
2412+
Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
2413+
deleteDataverseResponse.prettyPrint();
2414+
deleteDataverseResponse.then().assertThat()
2415+
.statusCode(OK.getStatusCode());
2416+
2417+
// Delete the user
2418+
Response deleteUserResponse = UtilIT.deleteUser(username);
2419+
deleteUserResponse.prettyPrint();
2420+
deleteUserResponse.then().assertThat()
2421+
.statusCode(OK.getStatusCode());
2422+
}
23312423

23322424
}

0 commit comments

Comments
 (0)