Skip to content

Commit 86c8dc5

Browse files
josegar74josegar74
authored
Harvesters / Check if the metadata schema of a metadata to update has changed (#9052)
Not a usual case, but some metadata schemas like Dutch datasets / services have different versions of the schemas and users can upgrade to the newer versions the existing metadata. The harvesters code when updating a metadata, was not checking if the schema version changed, only updating the XML. This pull request updates the following harvesters to deal with this use case: * CSW * Database * GeoNetwork * OAI-PMH * SFTP * WebDav
1 parent a55959a commit 86c8dc5

File tree

6 files changed

+73
-28
lines changed

6 files changed

+73
-28
lines changed

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//=============================================================================
2-
//=== Copyright (C) 2001-2024 Food and Agriculture Organization of the
2+
//=== Copyright (C) 2001-2025 Food and Agriculture Organization of the
33
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
44
//=== and United Nations Environment Programme (UNEP)
55
//===
@@ -66,8 +66,6 @@
6666
import javax.transaction.Transactional;
6767
import javax.transaction.Transactional.TxType;
6868
import java.io.IOException;
69-
import java.nio.file.Files;
70-
import java.nio.file.Path;
7169
import java.util.*;
7270
import java.util.concurrent.atomic.AtomicBoolean;
7371

@@ -309,7 +307,7 @@ private void addMetadata(RecordInfo ri, String uuidToAssign) throws Exception {
309307
// If the xslfilter process changes the metadata uuid,
310308
// use that uuid (newMdUuid) for the new metadata to add to the catalogue.
311309
String newMdUuid = null;
312-
if (!params.xslfilter.equals("")) {
310+
if (!params.xslfilter.isEmpty()) {
313311
md = applyXSLTProcessToMetadata(context, md, processName, processParams, log);
314312
schema = dataMan.autodetectSchema(md);
315313
// Get new uuid if modified by XSLT process
@@ -467,6 +465,11 @@ boolean updatingLocalMetadata(RecordInfo ri, String id, boolean force) throws Ex
467465
String newSchema = dataMan.autodetectSchema(md);
468466
updateSchema = !newSchema.equals(schema);
469467
schema = newSchema;
468+
} else {
469+
if (!ri.schema.equals(schema)) {
470+
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + ri.schema + "'. Using the detected one.");
471+
updateSchema = true;
472+
}
470473
}
471474

472475
applyBatchEdits(ri.uuid, md, schema, params.getBatchEdits(), context, log);

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/database/DatabaseHarvesterAligner.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ private String processMetadata(Element metadataElement) throws Exception {
236236

237237
switch (params.getOverrideUuid()) {
238238
case OVERRIDE:
239-
updateMetadata(metadataElement, Integer.toString(metadataUtils.findOneByUuid(uuid).getId()), true);
239+
updateMetadata(metadataElement, metadataUtils.findOneByUuid(uuid), true);
240240
log.debug(String.format("Overriding record with uuid %s", uuid));
241241
result.updatedMetadata++;
242242
break;
@@ -253,14 +253,15 @@ private String processMetadata(Element metadataElement) throws Exception {
253253
}
254254
} else {
255255
//record exists and belongs to this harvester
256-
updateMetadata(metadataElement, id, false);
256+
updateMetadata(metadataElement, metadataUtils.findOne(id), false);
257257
result.updatedMetadata++;
258258
}
259259

260260
return id;
261261
}
262262

263-
private void updateMetadata(Element xml, String id, boolean force) throws Exception {
263+
private void updateMetadata(Element xml, AbstractMetadata originalMetadata, boolean force) throws Exception {
264+
String id = Integer.toString(originalMetadata.getId());
264265
log.info("Updating metadata with id: " + id);
265266

266267
//
@@ -288,6 +289,11 @@ private void updateMetadata(Element xml, String id, boolean force) throws Except
288289
String newSchema = metadataSchemaUtils.autodetectSchema(xml);
289290
updateSchema = (newSchema != null) && !newSchema.equals(schema);
290291
schema = newSchema;
292+
} else {
293+
if (!originalMetadata.getDataInfo().getSchemaId().equals(schema)) {
294+
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + originalMetadata.getDataInfo().getSchemaId() + "'. Using the detected one.");
295+
updateSchema = true;
296+
}
291297
}
292298

293299
applyBatchEdits(uuid, xml, schema, params.getBatchEdits(), context, log);

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/BaseGeoNetworkAligner.java‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -435,10 +435,22 @@ private void updateMetadata(RecordInfo ri, String id, Element md,
435435
MetadataResourceDatabaseMigration.updateMetadataResourcesLink(md, null, settingManager);
436436
}
437437

438+
String schema = dataMan.autodetectSchema(md, null);
439+
boolean updateSchema = false;
440+
438441
if (!params.xslfilter.isEmpty()) {
439442
md = HarvesterUtil.processMetadata(metadataSchemaUtils.getSchema(ri.schema),
440443
md, processName, processParams);
444+
String newSchema = dataMan.autodetectSchema(md);
445+
updateSchema = !newSchema.equals(schema);
446+
schema = newSchema;
447+
} else {
448+
if (!ri.schema.equals(schema)) {
449+
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + ri.schema + "'. Using the detected one.");
450+
updateSchema = true;
451+
}
441452
}
453+
442454
// update metadata
443455
if (log.isDebugEnabled()) {
444456
log.debug(" - Updating local metadata with id=" + id);
@@ -452,10 +464,16 @@ private void updateMetadata(RecordInfo ri, String id, Element md,
452464
updateDateStamp, IndexingMode.none);
453465
metadata = metadataRepository.findOneById(Integer.parseInt(id));
454466
result.updatedMetadata++;
455-
if (force) {
456-
//change ownership of metadata to new harvester
457-
metadata.getHarvestInfo().setUuid(params.getUuid());
458-
metadata.getSourceInfo().setSourceId(params.getUuid());
467+
if (force || updateSchema) {
468+
if (force) {
469+
//change ownership of metadata to new harvester
470+
metadata.getHarvestInfo().setUuid(params.getUuid());
471+
metadata.getSourceInfo().setSourceId(params.getUuid());
472+
}
473+
474+
if (updateSchema) {
475+
metadata.getDataInfo().setSchemaId(schema);
476+
}
459477

460478
metadataManager.save(metadata);
461479
}

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//=============================================================================
2-
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
2+
//=== Copyright (C) 2001-2025 Food and Agriculture Organization of the
33
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
44
//=== and United Nations Environment Programme (UNEP)
55
//===
@@ -28,7 +28,6 @@
2828
import java.nio.file.Path;
2929
import java.util.Collections;
3030
import java.util.HashSet;
31-
import java.util.LinkedList;
3231
import java.util.List;
3332
import java.util.Map;
3433
import java.util.Set;
@@ -64,7 +63,6 @@
6463
import org.fao.geonet.lib.Lib;
6564
import org.fao.geonet.repository.MetadataValidationRepository;
6665
import org.fao.geonet.repository.OperationAllowedRepository;
67-
import org.fao.geonet.repository.Updater;
6866
import org.fao.geonet.repository.specification.MetadataValidationSpecs;
6967
import org.fao.geonet.utils.GeonetHttpRequestFactory;
7068
import org.fao.geonet.utils.Xml;
@@ -81,7 +79,6 @@
8179

8280
import jeeves.server.context.ServiceContext;
8381

84-
import javax.annotation.Nonnull;
8582

8683
//=============================================================================
8784

@@ -317,7 +314,7 @@ private void align(XmlRequest t, Set<RecordInfo> records) throws Exception {
317314
switch (params.getOverrideUuid()) {
318315
case OVERRIDE:
319316
processParams.put("mdChangeDate", ri.changeDate);
320-
updateMetadata(t, ri, Integer.toString(metadataUtils.findOneByUuid(ri.id).getId()),
317+
updateMetadata(t, ri, metadataUtils.findOneByUuid(ri.id),
321318
processName, processParams, true);
322319
result.updatedMetadata++;
323320
break;
@@ -340,7 +337,7 @@ private void align(XmlRequest t, Set<RecordInfo> records) throws Exception {
340337
//record exists and belongs to this harvester
341338
String id = localUuids.getID(ri.id);
342339
processParams.put("mdChangeDate", ri.changeDate);
343-
updateMetadata(t, ri, id, processName, processParams, false);
340+
updateMetadata(t, ri, metadataUtils.findOne(id), processName, processParams, false);
344341
}
345342
result.totalMetadata++;
346343
} catch (Throwable tr) {
@@ -534,7 +531,8 @@ private Element toDublinCore(Element md) {
534531
}
535532
}
536533

537-
private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String processName, Map<String, Object> processParams, boolean force) throws Exception {
534+
private void updateMetadata(XmlRequest t, RecordInfo ri, AbstractMetadata originalMetadata, String processName, Map<String, Object> processParams, boolean force) throws Exception {
535+
String id = Integer.toString(originalMetadata.getId());
538536
String date = localUuids.getChangeDate(ri.id);
539537

540538
if (!force && !ri.isMoreRecentThan(date)) {
@@ -565,6 +563,11 @@ private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String proce
565563

566564
schema = dataMan.autodetectSchema(md);
567565
updateSchema = true;
566+
} else {
567+
if (!originalMetadata.getDataInfo().getSchemaId().equals(schema)) {
568+
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + originalMetadata.getDataInfo().getSchemaId() + "'. Using the detected one.");
569+
updateSchema = true;
570+
}
568571
}
569572

570573
// Translate metadata
@@ -585,12 +588,7 @@ private void updateMetadata(XmlRequest t, RecordInfo ri, String id, String proce
585588
context.getBean(MetadataValidationRepository.class);
586589

587590
final String newSchema = schema;
588-
metadataManager.update(Integer.parseInt(id), new Updater<AbstractMetadata>() {
589-
@Override
590-
public void apply(@Nonnull AbstractMetadata entity) {
591-
entity.getDataInfo().setSchemaId(newSchema);
592-
}
593-
});
591+
metadataManager.update(Integer.parseInt(id), entity -> entity.getDataInfo().setSchemaId(newSchema));
594592

595593
metadataValidationRepository.deleteAll(MetadataValidationSpecs.hasMetadataId(Integer.parseInt(id)));
596594
}

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/sftp/Aligner.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,11 @@ boolean updatingLocalMetadata(RecordInfo ri, String id, Element md, boolean forc
367367
String newSchema = metadataSchemaUtils.autodetectSchema(md);
368368
updateSchema = !newSchema.equals(schema);
369369
schema = newSchema;
370+
} else {
371+
if (!ri.schema.equals(schema)) {
372+
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + ri.schema + "'. Using the detected one.");
373+
updateSchema = true;
374+
}
370375
}
371376

372377
boolean validate = false;

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -489,9 +489,18 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force)
489489
md = translateMetadataContent(context, md, schema);
490490
}
491491

492+
boolean updateSchema = false;
492493
if (StringUtils.isNotEmpty(params.xslfilter)) {
493494
md = HarvesterUtil.processMetadata(dataMan.getSchema(schema),
494495
md, processName, processParams);
496+
String newSchema = dataMan.autodetectSchema(md);
497+
updateSchema = !newSchema.equals(schema);
498+
schema = newSchema;
499+
} else {
500+
if (!recordInfo.schema.equals(schema)) {
501+
log.warning(" - Detected schema '" + schema + "' is different from the one of the metadata in the catalog '" + recordInfo.schema + "'. Using the detected one.");
502+
updateSchema = true;
503+
}
495504
}
496505

497506
//
@@ -504,10 +513,16 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force)
504513
final AbstractMetadata metadata = metadataManager.updateMetadata(context, recordInfo.id, md, validate, ufo, language,
505514
date, true, IndexingMode.none);
506515

507-
if(force) {
508-
//change ownership of metadata to new harvester
509-
metadata.getHarvestInfo().setUuid(params.getUuid());
510-
metadata.getSourceInfo().setSourceId(params.getUuid());
516+
if(force || updateSchema) {
517+
if (force) {
518+
//change ownership of metadata to new harvester
519+
metadata.getHarvestInfo().setUuid(params.getUuid());
520+
metadata.getSourceInfo().setSourceId(params.getUuid());
521+
}
522+
523+
if (updateSchema) {
524+
metadata.getDataInfo().setSchemaId(schema);
525+
}
511526

512527
context.getBean(IMetadataManager.class).save(metadata);
513528
}

0 commit comments

Comments
 (0)