Skip to content

Commit a865b6b

Browse files
authored
fix(ingest/dremio): Fix platform_instance URN generation (#15076)
1 parent 5a15697 commit a865b6b

File tree

5 files changed

+3400
-2160
lines changed

5 files changed

+3400
-2160
lines changed

metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -338,10 +338,10 @@ def process_dataset(
338338
return
339339

340340
dataset_urn = make_dataset_urn_with_platform_instance(
341-
platform=make_data_platform_urn(self.get_platform()),
342-
name=f"dremio.{dataset_name}",
343-
env=self.config.env,
341+
platform=self.get_platform(),
342+
name=dataset_name,
344343
platform_instance=self.config.platform_instance,
344+
env=self.config.env,
345345
)
346346

347347
for dremio_mcp in self.dremio_aspects.populate_dataset_mcp(
@@ -421,10 +421,10 @@ def generate_profiles(
421421
schema_str = ".".join(dataset_info.path)
422422
dataset_name = f"{schema_str}.{dataset_info.resource_name}".lower()
423423
dataset_urn = make_dataset_urn_with_platform_instance(
424-
platform=make_data_platform_urn(self.get_platform()),
425-
name=f"dremio.{dataset_name}",
426-
env=self.config.env,
424+
platform=self.get_platform(),
425+
name=dataset_name,
427426
platform_instance=self.config.platform_instance,
427+
env=self.config.env,
428428
)
429429
yield from self.profiler.get_workunits(dataset_info, dataset_urn)
430430

@@ -436,10 +436,10 @@ def generate_view_lineage(
436436
"""
437437
upstream_urns = [
438438
make_dataset_urn_with_platform_instance(
439-
platform=make_data_platform_urn(self.get_platform()),
440-
name=f"dremio.{upstream_table.lower()}",
441-
env=self.config.env,
439+
platform=self.get_platform(),
440+
name=upstream_table.lower(),
442441
platform_instance=self.config.platform_instance,
442+
env=self.config.env,
443443
)
444444
for upstream_table in parents
445445
]
@@ -498,19 +498,19 @@ def process_query(self, query: DremioQuery) -> None:
498498
if query.query and query.affected_dataset:
499499
upstream_urns = [
500500
make_dataset_urn_with_platform_instance(
501-
platform=make_data_platform_urn(self.get_platform()),
502-
name=f"dremio.{ds.lower()}",
503-
env=self.config.env,
501+
platform=self.get_platform(),
502+
name=ds.lower(),
504503
platform_instance=self.config.platform_instance,
504+
env=self.config.env,
505505
)
506506
for ds in query.queried_datasets
507507
]
508508

509509
downstream_urn = make_dataset_urn_with_platform_instance(
510-
platform=make_data_platform_urn(self.get_platform()),
511-
name=f"dremio.{query.affected_dataset.lower()}",
512-
env=self.config.env,
510+
platform=self.get_platform(),
511+
name=query.affected_dataset.lower(),
513512
platform_instance=self.config.platform_instance,
513+
env=self.config.env,
514514
)
515515

516516
# Add query to SqlParsingAggregator

0 commit comments

Comments
 (0)