Skip to content

Commit e42ec99

Browse files
karanh37karanh37
authored
Merge branch 'main' into import-export-improvements
2 parents 76d2b73 + fce9b05 commit e42ec99

25 files changed

+2443
-24
lines changed

‎ingestion/src/metadata/ingestion/models/patch_request.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -416,15 +416,6 @@ def build_patch(
416416
updated_operations = JsonPatchUpdater.from_restrict_update_fields(
417417
restrict_update_fields
418418
).update(patch, override_metadata=override_metadata)
419-
420-
# if the only operation is to update the sourceHash, we'll skip the patch
421-
# since this will only happen when we filter out the REMOVE and REPLACE ops
422-
# and if the sourceHash is updated in this case further updates will not be processed
423-
if (
424-
len(updated_operations) == 1
425-
and updated_operations[0].get("path") == "/sourceHash"
426-
):
427-
return None
428419
patch.patch = updated_operations
429420

430421
return patch

‎ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -999,6 +999,93 @@ def _parse_redshift_source(self, source_expression: str) -> Optional[List[dict]]
999999
logger.debug(traceback.format_exc())
10001000
return None
10011001

1002+
def _parse_bigquery_source(
1003+
self, source_expression: str, datamodel_entity: DashboardDataModel
1004+
) -> Optional[List[dict]]:
1005+
"""
1006+
Parse BigQuery source from Power Query M expressions.
1007+
Handles both direct BigQuery connections and references to dataset expressions.
1008+
1009+
Examples:
1010+
1. Direct: GoogleBigQuery.Database()[Name="project"][Data][Name="dataset",Kind="Schema"][Data][Name="table",Kind="Table"][Data]
1011+
2. Via expression reference: Source = S_PJ_CODE (where S_PJ_CODE is a dataset expression)
1012+
"""
1013+
try:
1014+
# Check if source expression references a dataset expression
1015+
# Pattern: Source = <expression_name>
1016+
source_ref_match = re.search(
1017+
r'Source\s*=\s*([A-Za-z0-9_#"&\s]+?)\s*,',
1018+
source_expression,
1019+
re.MULTILINE,
1020+
)
1021+
1022+
if source_ref_match:
1023+
ref_name = (
1024+
source_ref_match.group(1).strip().strip('"').strip("#").strip('"')
1025+
)
1026+
logger.debug(
1027+
f"Table source references expression: {ref_name}, resolving..."
1028+
)
1029+
1030+
# Fetch the dataset to get its expressions
1031+
dataset = self._fetch_dataset_from_workspace(datamodel_entity.name.root)
1032+
if dataset and dataset.expressions:
1033+
for dexpression in dataset.expressions:
1034+
if dexpression.name == ref_name and dexpression.expression:
1035+
logger.debug(
1036+
f"Found referenced expression '{ref_name}', checking for BigQuery"
1037+
)
1038+
# Recursively parse the referenced expression
1039+
return self._parse_bigquery_source(
1040+
dexpression.expression, datamodel_entity
1041+
)
1042+
1043+
# Check if this is a direct BigQuery connection
1044+
if "GoogleBigQuery.Database" not in source_expression:
1045+
return None
1046+
1047+
logger.debug(f"Found GoogleBigQuery.Database in expression")
1048+
1049+
# Extract project, dataset (schema), and table from BigQuery M expression
1050+
# Pattern: [Name="project"][Data][Name="dataset",Kind="Schema"][Data][Name="table",Kind="Table"]
1051+
1052+
# Extract all Name= patterns
1053+
name_matches = re.findall(
1054+
r'\[Name="([^"]+)"(?:,Kind="([^"]+)")?\]', source_expression
1055+
)
1056+
1057+
if not name_matches:
1058+
logger.debug("No Name patterns found in BigQuery expression")
1059+
return None
1060+
1061+
# BigQuery structure: project -> dataset (Schema) -> table (Table/View)
1062+
project = None
1063+
dataset = None
1064+
table_name = None
1065+
1066+
for name, kind in name_matches:
1067+
if kind == "Schema":
1068+
dataset = name
1069+
elif kind == "Table" or kind == "View":
1070+
table_name = name
1071+
elif not kind and not project:
1072+
# First Name without Kind is likely the project
1073+
project = name
1074+
1075+
logger.debug(
1076+
f"Extracted BigQuery info: project={project}, dataset={dataset}, table={table_name}"
1077+
)
1078+
1079+
if table_name:
1080+
return [{"database": project, "schema": dataset, "table": table_name}]
1081+
1082+
return None
1083+
1084+
except Exception as exc:
1085+
logger.debug(f"Error to parse BigQuery table source: {exc}")
1086+
logger.debug(traceback.format_exc())
1087+
return None
1088+
10021089
def _parse_snowflake_query_source(
10031090
self, source_expression: str
10041091
) -> Optional[List[dict]]:
@@ -1244,6 +1331,13 @@ def _parse_table_info_from_source_exp(
12441331
if isinstance(table_info_list, List):
12451332
return table_info_list
12461333

1334+
# parse bigquery source
1335+
table_info_list = self._parse_bigquery_source(
1336+
source_expression, datamodel_entity
1337+
)
1338+
if isinstance(table_info_list, List):
1339+
return table_info_list
1340+
12471341
# parse databricks source
12481342
table_info_list = self._parse_databricks_source(
12491343
source_expression, datamodel_entity

‎ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from datetime import datetime
1515
from typing import List, Optional, Union
1616

17-
from pydantic import BaseModel, Field, field_validator
17+
from pydantic import BaseModel, Field, field_validator, model_validator
1818
from typing_extensions import Annotated
1919

2020

@@ -162,6 +162,16 @@ def normalize_expression(cls, v):
162162
return v
163163

164164

165+
class PowerBIPartition(BaseModel):
166+
"""
167+
PowerBI Table Partition (.pbit files)
168+
"""
169+
170+
name: Optional[str] = None
171+
mode: Optional[str] = None
172+
source: Optional[PowerBITableSource] = None
173+
174+
165175
class PowerBiTable(BaseModel):
166176
"""
167177
PowerBI Table Model
@@ -173,6 +183,20 @@ class PowerBiTable(BaseModel):
173183
measures: Optional[List[PowerBiMeasures]] = None
174184
description: Optional[str] = None
175185
source: Optional[List[PowerBITableSource]] = None
186+
partitions: Optional[List[PowerBIPartition]] = None
187+
188+
@model_validator(mode="before")
189+
@classmethod
190+
def extract_source_from_partitions(cls, values):
191+
if isinstance(values, dict):
192+
if values.get("source") is None and values.get("partitions"):
193+
partitions = values.get("partitions", [])
194+
if partitions and len(partitions) > 0:
195+
partition_source = partitions[0].get("source")
196+
if partition_source:
197+
values["source"] = [partition_source]
198+
199+
return values
176200

177201

178202
class TablesResponse(BaseModel):

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.openmetadata.schema.type.change.ChangeSource;
3131
import org.openmetadata.service.Entity;
3232
import org.openmetadata.service.resources.apis.APICollectionResource;
33+
import org.openmetadata.service.util.EntityUtil;
3334
import org.openmetadata.service.util.EntityUtil.Fields;
3435
import org.openmetadata.service.util.EntityUtil.RelationIncludes;
3536
import org.openmetadata.service.util.FullyQualifiedName;
@@ -221,7 +222,13 @@ public APICollectionUpdater(
221222
@Transaction
222223
@Override
223224
public void entitySpecificUpdate(boolean consolidatingChanges) {
224-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
225+
recordChange(
226+
"sourceHash",
227+
original.getSourceHash(),
228+
updated.getSourceHash(),
229+
false,
230+
EntityUtil.objectMatch,
231+
false);
225232
}
226233
}
227234
}

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIEndpointRepository.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -539,7 +539,13 @@ public void entitySpecificUpdate(boolean consolidatingChanges) {
539539
listOrEmpty(updated.getResponseSchema().getSchemaFields()),
540540
EntityUtil.schemaFieldMatch);
541541
}
542-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
542+
recordChange(
543+
"sourceHash",
544+
original.getSourceHash(),
545+
updated.getSourceHash(),
546+
false,
547+
EntityUtil.objectMatch,
548+
false);
543549
}
544550

545551
private void updateSchemaFields(

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ChartRepository.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,13 @@ public ChartUpdater(Chart chart, Chart updated, Operation operation) {
208208
public void entitySpecificUpdate(boolean consolidatingChanges) {
209209
recordChange("chartType", original.getChartType(), updated.getChartType());
210210
recordChange("sourceUrl", original.getSourceUrl(), updated.getSourceUrl());
211-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
211+
recordChange(
212+
"sourceHash",
213+
original.getSourceHash(),
214+
updated.getSourceHash(),
215+
false,
216+
EntityUtil.objectMatch,
217+
false);
212218
update(
213219
Entity.DASHBOARD,
214220
"dashboards",

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -561,7 +561,13 @@ public void entitySpecificUpdate(boolean consolidatingChanges) {
561561
recordChange("sourceUrl", original.getSourceUrl(), updated.getSourceUrl());
562562
recordChange("fullPath", original.getFullPath(), updated.getFullPath());
563563
recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod());
564-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
564+
recordChange(
565+
"sourceHash",
566+
original.getSourceHash(),
567+
updated.getSourceHash(),
568+
false,
569+
EntityUtil.objectMatch,
570+
false);
565571
}
566572

567573
private void updateDataModel(Container original, Container updated) {

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,13 @@ public void entitySpecificUpdate(boolean consolidatingChanges) {
323323
DatabaseUtil.validateColumns(original.getColumns());
324324
updateColumns("columns", original.getColumns(), updated.getColumns(), EntityUtil.columnMatch);
325325
recordChange("sourceUrl", original.getSourceUrl(), updated.getSourceUrl());
326-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
326+
recordChange(
327+
"sourceHash",
328+
original.getSourceHash(),
329+
updated.getSourceHash(),
330+
false,
331+
EntityUtil.objectMatch,
332+
false);
327333
recordChange("sql", original.getSql(), updated.getSql());
328334
}
329335
}

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardRepository.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -570,7 +570,13 @@ public void entitySpecificUpdate(boolean consolidatingChanges) {
570570
listOrEmpty(updated.getDataModels()),
571571
listOrEmpty(original.getDataModels()));
572572
updateDashboardUrl(original, updated);
573-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
573+
recordChange(
574+
"sourceHash",
575+
original.getSourceHash(),
576+
updated.getSourceHash(),
577+
false,
578+
EntityUtil.objectMatch,
579+
false);
574580
}
575581

576582
private void update(

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -514,7 +514,13 @@ public DatabaseUpdater(Database original, Database updated, Operation operation)
514514
public void entitySpecificUpdate(boolean consolidatingChanges) {
515515
recordChange("retentionPeriod", original.getRetentionPeriod(), updated.getRetentionPeriod());
516516
recordChange("sourceUrl", original.getSourceUrl(), updated.getSourceUrl());
517-
recordChange("sourceHash", original.getSourceHash(), updated.getSourceHash());
517+
recordChange(
518+
"sourceHash",
519+
original.getSourceHash(),
520+
updated.getSourceHash(),
521+
false,
522+
EntityUtil.objectMatch,
523+
false);
518524
}
519525
}
520526

0 commit comments

Comments
 (0)