Skip to content

Commit ecc2413

Browse files
manerowmanerow
committed
fix(dq-dashboard): cascade certification to all table-child search indices
1 parent 98c2266 commit ecc2413

23 files changed

Lines changed: 808 additions & 3 deletions

‎openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,18 @@ public interface SearchClient
179179
}
180180
""";
181181

182+
// Cascade variant: full-object replace (handles add/update) plus removal on
183+
// null params, so child docs stay in sync when a parent's cert is added,
184+
// changed, or removed.
185+
String CASCADE_CERTIFICATION_SCRIPT =
186+
"""
187+
if (params.certification == null) {
188+
ctx._source.remove('certification');
189+
} else {
190+
ctx._source.certification = params.certification;
191+
}
192+
""";
193+
182194
String UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT =
183195
"""
184196
if (ctx._source.containsKey('tags')) {

‎openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import static org.openmetadata.service.Entity.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA;
1919
import static org.openmetadata.service.Entity.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA;
2020
import static org.openmetadata.service.search.SearchClient.ADD_FOLLOWERS_SCRIPT;
21+
import static org.openmetadata.service.search.SearchClient.CASCADE_CERTIFICATION_SCRIPT;
2122
import static org.openmetadata.service.search.SearchClient.DATA_ASSET_SEARCH_ALIAS;
2223
import static org.openmetadata.service.search.SearchClient.DEFAULT_UPDATE_SCRIPT;
2324
import static org.openmetadata.service.search.SearchClient.GLOBAL_SEARCH_ALIAS;
@@ -1816,6 +1817,47 @@ private void handleEntityCertificationUpdate(EntityInterface entity, ChangeDescr
18161817

18171818
AssetCertification certification = getCertificationFromEntity(entity);
18181819
updateEntityCertificationInSearch(entity, certification);
1820+
cascadeCertificationToChildren(entity, certification);
1821+
}
1822+
1823+
// Pushes the cert change onto every child search doc denormalized from this
1824+
// entity. Without this the cert filter on the DQ dashboard (which queries
1825+
// children like test_case/test_case_result/test_case_resolution_status by
1826+
// `certification.tagLabel.tagFQN`) would silently use the stale cert until a
1827+
// reindex. RAW_REPLACE in PropagationDescriptor can't be used because it
1828+
// restores the old value on delete; we drive a dedicated script instead.
1829+
private void cascadeCertificationToChildren(
1830+
EntityInterface entity, AssetCertification certification) {
1831+
String type = entity.getEntityReference().getType();
1832+
if (!Entity.TABLE.equalsIgnoreCase(type)) {
1833+
// Scope: Table only. Dashboard/ApiCollection children also have cert in
1834+
// their mappings; extend here when those denormalization paths are added.
1835+
return;
1836+
}
1837+
IndexMapping indexMapping = entityIndexMap.get(Entity.TABLE);
1838+
if (indexMapping == null) {
1839+
return;
1840+
}
1841+
List<String> childAliases = indexMapping.getChildAliases(clusterAlias);
1842+
if (nullOrEmpty(childAliases)) {
1843+
return;
1844+
}
1845+
1846+
Map<String, Object> params = new HashMap<>();
1847+
params.put("certification", certification); // null when cert was removed
1848+
1849+
Pair<String, String> parentMatch = new ImmutablePair<>("table.id", entity.getId().toString());
1850+
1851+
try {
1852+
searchClient.updateChildren(
1853+
childAliases, parentMatch, new ImmutablePair<>(CASCADE_CERTIFICATION_SCRIPT, params));
1854+
} catch (Exception e) {
1855+
LOG.error(
1856+
"Failed to cascade certification for table [{}]: {}",
1857+
entity.getFullyQualifiedName(),
1858+
e.getMessage(),
1859+
e);
1860+
}
18191861
}
18201862

18211863
private boolean isCertificationUpdated(ChangeDescription change) {

‎openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnSearchIndex.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,10 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
135135
}
136136
}
137137

138+
if (parentTable.getCertification() != null) {
139+
doc.put("certification", parentTable.getCertification());
140+
}
141+
138142
if (column.getExtension() != null) {
139143
doc.put("extension", column.getExtension());
140144
doc.put(

‎openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import java.util.HashMap;
66
import java.util.Map;
7+
import org.openmetadata.schema.entity.data.Table;
78
import org.openmetadata.schema.tests.TestCase;
89
import org.openmetadata.schema.tests.TestSuite;
910
import org.openmetadata.schema.tests.type.TestCaseResolutionStatus;
@@ -62,7 +63,12 @@ private void setParentRelationships(Map<String, Object> doc) {
6263
if (testSuite == null) return;
6364
doc.put("testSuite", testSuite.getEntityReference());
6465
if (testSuite.getBasicEntityReference() != null) {
65-
TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getBasicEntityReference(), doc);
66+
Table linkedTable =
67+
TestSuiteIndex.addTestSuiteParentEntityRelations(
68+
testSuite.getBasicEntityReference(), doc);
69+
if (linkedTable != null && linkedTable.getCertification() != null) {
70+
doc.put("certification", linkedTable.getCertification());
71+
}
6672
}
6773
}
6874

‎openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResultIndex.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ private void setTableEntityParentRelations(
118118
Entity.getEntityByName(
119119
Entity.TABLE,
120120
entityLink.getEntityFQN(),
121-
"database,databaseSchema,service",
121+
"database,databaseSchema,service,certification",
122122
Include.ALL);
123123
esDoc.put("database", table.getDatabase());
124124
esDoc.put("databaseSchema", table.getDatabaseSchema());
@@ -127,6 +127,9 @@ private void setTableEntityParentRelations(
127127
esDoc.put("serviceType", table.getServiceType());
128128
}
129129
esDoc.put("table", table.getEntityReference());
130+
if (table.getCertification() != null) {
131+
esDoc.put("certification", table.getCertification());
132+
}
130133
} catch (EntityNotFoundException ex) {
131134
LOG.warn(
132135
"Table [{}] not found during search indexing: {}",

‎openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,10 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
4747
private void setParentRelationships(Map<String, Object> doc, TestSuite testSuite) {
4848
EntityReference entityReference = testSuite.getBasicEntityReference();
4949
if (entityReference == null) return;
50-
addTestSuiteParentEntityRelations(entityReference, doc);
50+
Table linkedTable = addTestSuiteParentEntityRelations(entityReference, doc);
51+
if (linkedTable != null && linkedTable.getCertification() != null) {
52+
doc.put("certification", linkedTable.getCertification());
53+
}
5154
}
5255

5356
static Table addTestSuiteParentEntityRelations(

‎openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -849,6 +849,101 @@ void propagateCertificationTagsUsesOldFqnWhenTagRenamed() {
849849
assertEquals("Certification.Gold", keyCaptor.getValue().getRight());
850850
}
851851

852+
@Test
853+
void propagateCertificationTagsCascadesToTableChildrenOnAdd() throws IOException {
854+
Table table = mock(Table.class);
855+
UUID entityId = UUID.randomUUID();
856+
when(table.getId()).thenReturn(entityId);
857+
when(table.getEntityReference())
858+
.thenReturn(new EntityReference().withId(entityId).withType(Entity.TABLE));
859+
AssetCertification cert =
860+
new AssetCertification()
861+
.withTagLabel(
862+
new TagLabel()
863+
.withName("Gold")
864+
.withDescription("Certified")
865+
.withTagFQN("Certification.Gold"));
866+
when(table.getCertification()).thenReturn(cert);
867+
868+
ChangeDescription changeDescription =
869+
changeDescription(
870+
List.of(),
871+
List.of(
872+
new FieldChange().withName("certification").withOldValue("{}").withNewValue("{}")),
873+
List.of());
874+
875+
repository.propagateCertificationTags(Entity.TABLE, table, changeDescription);
876+
877+
@SuppressWarnings("unchecked")
878+
ArgumentCaptor<Pair<String, Map<String, Object>>> updatesCaptor =
879+
ArgumentCaptor.forClass(Pair.class);
880+
@SuppressWarnings("unchecked")
881+
ArgumentCaptor<Pair<String, String>> matchCaptor = ArgumentCaptor.forClass(Pair.class);
882+
verify(searchClient)
883+
.updateChildren(
884+
eq(List.of("cluster_column_search_index")),
885+
matchCaptor.capture(),
886+
updatesCaptor.capture());
887+
assertEquals("table.id", matchCaptor.getValue().getLeft());
888+
assertEquals(entityId.toString(), matchCaptor.getValue().getRight());
889+
assertEquals(SearchClient.CASCADE_CERTIFICATION_SCRIPT, updatesCaptor.getValue().getLeft());
890+
assertSame(cert, updatesCaptor.getValue().getRight().get("certification"));
891+
}
892+
893+
@Test
894+
void propagateCertificationTagsCascadesNullToTableChildrenOnRemove() throws IOException {
895+
Table table = mock(Table.class);
896+
UUID entityId = UUID.randomUUID();
897+
when(table.getId()).thenReturn(entityId);
898+
when(table.getEntityReference())
899+
.thenReturn(new EntityReference().withId(entityId).withType(Entity.TABLE));
900+
when(table.getCertification()).thenReturn(null);
901+
902+
ChangeDescription changeDescription =
903+
changeDescription(
904+
List.of(),
905+
List.of(),
906+
List.of(new FieldChange().withName("certification").withOldValue("{}")));
907+
908+
repository.propagateCertificationTags(Entity.TABLE, table, changeDescription);
909+
910+
@SuppressWarnings("unchecked")
911+
ArgumentCaptor<Pair<String, Map<String, Object>>> updatesCaptor =
912+
ArgumentCaptor.forClass(Pair.class);
913+
verify(searchClient)
914+
.updateChildren(
915+
eq(List.of("cluster_column_search_index")), any(Pair.class), updatesCaptor.capture());
916+
assertEquals(SearchClient.CASCADE_CERTIFICATION_SCRIPT, updatesCaptor.getValue().getLeft());
917+
assertNull(updatesCaptor.getValue().getRight().get("certification"));
918+
}
919+
920+
@Test
921+
void propagateCertificationTagsDoesNotCascadeForNonTableEntities() throws IOException {
922+
// Pipelines carry a native certification but DQ dashboard cascade is
923+
// scoped to Table — children of Pipeline aren't part of the test_case
924+
// family. Verify we don't blast an updateByQuery against unrelated
925+
// child indices.
926+
Pipeline pipeline = mock(Pipeline.class);
927+
UUID entityId = UUID.randomUUID();
928+
when(pipeline.getId()).thenReturn(entityId);
929+
when(pipeline.getEntityReference())
930+
.thenReturn(new EntityReference().withId(entityId).withType(Entity.PIPELINE));
931+
when(pipeline.getCertification())
932+
.thenReturn(
933+
new AssetCertification().withTagLabel(new TagLabel().withTagFQN("Certification.Gold")));
934+
935+
ChangeDescription changeDescription =
936+
changeDescription(
937+
List.of(),
938+
List.of(
939+
new FieldChange().withName("certification").withOldValue("{}").withNewValue("{}")),
940+
List.of());
941+
942+
repository.propagateCertificationTags(Entity.PIPELINE, pipeline, changeDescription);
943+
944+
verify(searchClient, never()).updateChildren(any(List.class), any(Pair.class), any(Pair.class));
945+
}
946+
852947
@Test
853948
void propagateCertificationTagsUsesQuotedOldNameWhenTagHasNoParentFqn() {
854949
Tag tag = mock(Tag.class);

‎openmetadata-spec/src/main/resources/elasticsearch/en/column_index_mapping.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,46 @@
535535
}
536536
}
537537
},
538+
"certification": {
539+
"type": "object",
540+
"properties": {
541+
"tagLabel": {
542+
"type": "object",
543+
"properties": {
544+
"tagFQN": {
545+
"type": "keyword",
546+
"normalizer": "lowercase_normalizer",
547+
"fields": {
548+
"text": {
549+
"type": "text",
550+
"analyzer": "om_analyzer"
551+
}
552+
}
553+
},
554+
"labelType": {
555+
"type": "keyword"
556+
},
557+
"description": {
558+
"type": "text"
559+
},
560+
"source": {
561+
"type": "keyword"
562+
},
563+
"state": {
564+
"type": "keyword"
565+
}
566+
}
567+
},
568+
"appliedDate": {
569+
"type": "date",
570+
"format": "strict_date_optional_time||epoch_millis"
571+
},
572+
"expiryDate": {
573+
"type": "date",
574+
"format": "strict_date_optional_time||epoch_millis"
575+
}
576+
}
577+
},
538578
"classificationTags": {
539579
"type": "keyword",
540580
"normalizer": "lowercase_normalizer"

‎openmetadata-spec/src/main/resources/elasticsearch/en/test_case_resolution_status_index_mapping.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,46 @@
606606
}
607607
}
608608
},
609+
"certification": {
610+
"type": "object",
611+
"properties": {
612+
"tagLabel": {
613+
"type": "object",
614+
"properties": {
615+
"tagFQN": {
616+
"type": "keyword",
617+
"normalizer": "lowercase_normalizer",
618+
"fields": {
619+
"text": {
620+
"type": "text",
621+
"analyzer": "om_analyzer"
622+
}
623+
}
624+
},
625+
"labelType": {
626+
"type": "keyword"
627+
},
628+
"description": {
629+
"type": "text"
630+
},
631+
"source": {
632+
"type": "keyword"
633+
},
634+
"state": {
635+
"type": "keyword"
636+
}
637+
}
638+
},
639+
"appliedDate": {
640+
"type": "date",
641+
"format": "strict_date_optional_time||epoch_millis"
642+
},
643+
"expiryDate": {
644+
"type": "date",
645+
"format": "strict_date_optional_time||epoch_millis"
646+
}
647+
}
648+
},
609649
"testSuite": {
610650
"properties": {
611651
"id": {

‎openmetadata-spec/src/main/resources/elasticsearch/en/test_case_result_index_mapping.json‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,46 @@
471471
}
472472
}
473473
},
474+
"certification": {
475+
"type": "object",
476+
"properties": {
477+
"tagLabel": {
478+
"type": "object",
479+
"properties": {
480+
"tagFQN": {
481+
"type": "keyword",
482+
"normalizer": "lowercase_normalizer",
483+
"fields": {
484+
"text": {
485+
"type": "text",
486+
"analyzer": "om_analyzer"
487+
}
488+
}
489+
},
490+
"labelType": {
491+
"type": "keyword"
492+
},
493+
"description": {
494+
"type": "text"
495+
},
496+
"source": {
497+
"type": "keyword"
498+
},
499+
"state": {
500+
"type": "keyword"
501+
}
502+
}
503+
},
504+
"appliedDate": {
505+
"type": "date",
506+
"format": "strict_date_optional_time||epoch_millis"
507+
},
508+
"expiryDate": {
509+
"type": "date",
510+
"format": "strict_date_optional_time||epoch_millis"
511+
}
512+
}
513+
},
474514
"service": {
475515
"properties": {
476516
"id": {

0 commit comments

Comments
 (0)