diff --git a/setup.py b/setup.py index 27d4cd08..d7079857 100755 --- a/setup.py +++ b/setup.py @@ -62,12 +62,12 @@ def read(*names, **kwargs): ], python_requires=">=3.8", install_requires=[ - "click==8.1.7", - "dbt-artifacts-parser==0.6.0", - "ruamel.yaml==0.18.6", - "tabulate==0.9.0", - "requests==2.31.0", - "sqlglot==25.30.0", + "click~=8.1.7", + "dbt-artifacts-parser~=0.8.1", + "ruamel.yaml~=0.18.6", + "tabulate~=0.9.0", + "requests~=2.31.0", + "sqlglot~=25.30.0", ], extras_require={ # eg: diff --git a/src/datapilot/constants.py b/src/datapilot/constants.py new file mode 100644 index 00000000..aed1f350 --- /dev/null +++ b/src/datapilot/constants.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class Extra(str, Enum): + allow = "allow" + forbid = "forbid" + ignore = "ignore" diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py index 48616e76..e0fce0a6 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py @@ -76,7 +76,7 @@ def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]: status_code = 0 missing_keys = set(self.labels_keys) - set(self.get_node(node_id).label) - config = self.get_node(node_id).config.dict() if self.get_node(node_id).config else {} + config = self.get_node(node_id).config.model_dump() if self.get_node(node_id).config else {} labels = config.get("labels", {}) label_keys = set(labels.keys()) extra_keys = set() diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py index f45d2e2e..fa66d0ba 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py @@ -72,7 +72,7 @@ def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str def _check_meta_keys(self, node_id) -> Tuple[int, Set[str], Set[str]]: status_code = 0 model = self.get_node(node_id) - meta = model.meta.dict() if model.meta else {} + meta = model.meta if model.meta else {} model_meta_keys = set(meta.keys()) missing_keys = None extra_keys = None diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py index 5b593f12..b4f3eaee 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py @@ -61,7 +61,7 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: def _check_source_has_freshness(self, source_id: str) -> List[str]: source = self.get_node(source_id) - freshness = source.freshness.dict() if source.freshness else {} + freshness = source.freshness.model_dump() if source.freshness else {} if not freshness: return False diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py index 7caaf1ff..c7b96b44 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py @@ -74,7 +74,7 @@ def _build_failure_result(self, model_unique_id: str, missing_keys: Sequence[str def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]: status_code = 0 missing_keys = set(self.labels_keys) - set(self.get_node(node_id).label) - config = self.get_node(node_id).config.dict() if self.get_node(node_id).config else {} + config = self.get_node(node_id).config.model_dump() if self.get_node(node_id).config else {} labels = config.get("labels", {}) label_keys = set(labels.keys()) extra_keys = set() diff --git a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py index 71bfebd7..f988e641 100644 --- a/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py +++ b/src/datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py @@ -81,7 +81,7 @@ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]: def _check_source_has_meta_keys(self, source_unique_id: str): status_code = 0 model = self.get_node(source_unique_id) - meta = model.meta.dict() if model.meta else {} + meta = model.meta if model.meta else {} model_meta_keys = set(meta.keys()) missing_keys = None extra_keys = None diff --git a/src/datapilot/core/platforms/dbt/schemas/catalog.py b/src/datapilot/core/platforms/dbt/schemas/catalog.py index af709fa5..288557f9 100644 --- a/src/datapilot/core/platforms/dbt/schemas/catalog.py +++ b/src/datapilot/core/platforms/dbt/schemas/catalog.py @@ -5,13 +5,16 @@ from typing import Optional from typing import Union -from pydantic.config import Extra +from pydantic import ConfigDict from pydantic.main import BaseModel +from datapilot.constants import Extra + class AltimateCatalogMetadata(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/catalog/v1.json" dbt_version: Optional[str] = "0.19.0" @@ -21,8 +24,9 @@ class Config: class AltimateCatalogTableMetadata(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) type: str database: Optional[Optional[str]] = None @@ -33,8 +37,9 @@ class Config: class AltimateCatalogColumnMetadata(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) type: str comment: Optional[Optional[str]] = None @@ -43,8 +48,9 @@ class Config: class AltimateCatalogStatsItem(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) id: str label: str @@ -54,8 +60,9 @@ class Config: class AltimateCatalogTable(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) metadata: AltimateCatalogTableMetadata columns: Dict[str, AltimateCatalogColumnMetadata] @@ -64,8 +71,9 @@ class Config: class AltimateCatalogCatalogV1(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) metadata: AltimateCatalogMetadata nodes: Dict[str, AltimateCatalogTable] diff --git a/src/datapilot/core/platforms/dbt/schemas/manifest.py b/src/datapilot/core/platforms/dbt/schemas/manifest.py index 0a1aa85f..bee2a07a 100644 --- a/src/datapilot/core/platforms/dbt/schemas/manifest.py +++ b/src/datapilot/core/platforms/dbt/schemas/manifest.py @@ -19,7 +19,9 @@ from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11 from dbt_artifacts_parser.parsers.manifest.manifest_v11 import SupportedLanguage from pydantic import BaseModel -from pydantic import Extra +from pydantic import ConfigDict + +from datapilot.constants import Extra class DBTVersion(BaseModel): @@ -46,16 +48,17 @@ class DBTVersion(BaseModel): class AltimateDocs(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) show: Optional[bool] = True node_color: Optional[Optional[str]] = None class AltimateDependsOn(BaseModel): - nodes: Optional[List[str]] - macros: Optional[List[str]] + nodes: Optional[List[str]] = None + macros: Optional[List[str]] = None class AltimateManifestColumnInfo(BaseModel): @@ -98,8 +101,9 @@ class AltimateAccess(Enum): class AltimateDBTContract(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) enforced: Optional[bool] = False alias_types: Optional[bool] = True @@ -107,8 +111,9 @@ class Config: class AltimateHook(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) sql: str transaction: Optional[bool] = True @@ -117,8 +122,9 @@ class Config: # TODO: Need to add the rest of the fields class AltimateNodeConfig(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict( + extra=Extra.allow, + ) _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True @@ -203,8 +209,9 @@ class AltimateSourceConfig(BaseModel): class AltimateDeferRelation(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) database: Optional[str] schema_name: str @@ -213,8 +220,9 @@ class Config: class AltimateSeedConfig(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict( + extra=Extra.allow, + ) _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True @@ -314,8 +322,9 @@ class AltimateExposureType(Enum): class AltimateOwner(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict( + extra=Extra.allow, + ) _extra: Optional[Dict[str, Any]] = None email: Optional[Optional[str]] = None @@ -329,8 +338,9 @@ class AltimateMaturityEnum(Enum): class AltimateRefArgs(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) name: str package: Optional[Optional[str]] = None @@ -338,8 +348,9 @@ class Config: class AltimateExposureConfig(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict( + extra=Extra.allow, + ) _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True @@ -371,8 +382,9 @@ class AltimateManifestExposureNode(BaseModel): class AltimateTestMetadata(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) name: str kwargs: Optional[Dict[str, Any]] = None @@ -380,8 +392,9 @@ class Config: class AltimateTestConfig(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict( + extra=Extra.allow, + ) _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True @@ -433,8 +446,9 @@ class AltimateManifestTestNode(BaseModel): class AltimateMacroArgument(BaseModel): - class Config: - extra = Extra.forbid + model_config = ConfigDict( + extra=Extra.forbid, + ) name: str type: Optional[Optional[str]] = None diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py index 8addc8d3..40a4abf1 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py @@ -116,6 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: contract=contract, meta=node.meta, patch_path=node.patch_path, + access=node.access.value, ) def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: @@ -133,10 +134,10 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: source_description=source.source_description, loader=source.loader, identifier=source.identifier, - quoting=AltimateQuoting(**source.quoting.dict()) if source.quoting else None, + quoting=AltimateQuoting(**source.quoting.model_dump()) if source.quoting else None, loaded_at_field=source.loaded_at_field, - freshness=AltimateFreshnessThreshold(**source.freshness.dict()) if source.freshness else None, - external=AltimateExternalTable(**source.external.dict()) if source.external else None, + freshness=AltimateFreshnessThreshold(**source.freshness.model_dump()) if source.freshness else None, + external=AltimateExternalTable(**source.external.model_dump()) if source.external else None, description=source.description, columns={ name: AltimateManifestColumnInfo( @@ -153,7 +154,7 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: relation_name=source.relation_name, source_meta=source.source_meta, tags=source.tags, - config=AltimateSourceConfig(**source.config.dict()) if source.config else None, + config=AltimateSourceConfig(**source.config.model_dump()) if source.config else None, patch_path=source.patch_path, unrendered_config=source.unrendered_config, created_at=source.created_at, @@ -177,9 +178,9 @@ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: ), description=macro.description, meta=macro.meta, - docs=macro.docs, + docs=macro.docs.model_dump() if macro.docs else None, patch_path=macro.patch_path, - arguments=[AltimateMacroArgument(**arg.dict()) for arg in macro.arguments] if macro.arguments else None, + arguments=[AltimateMacroArgument(**arg.model_dump()) for arg in macro.arguments] if macro.arguments else None, created_at=macro.created_at, supported_languages=macro.supported_languages, ) @@ -194,22 +195,24 @@ def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: unique_id=exposure.unique_id, fqn=exposure.fqn, type=AltimateExposureType(exposure.type.value) if exposure.type else None, - owner=AltimateOwner(**exposure.owner.dict()) if exposure.owner else None, + owner=AltimateOwner(**exposure.owner.model_dump()) if exposure.owner else None, description=exposure.description, label=exposure.label, maturity=AltimateMaturityEnum(exposure.maturity.value) if exposure.maturity else None, meta=exposure.meta, tags=exposure.tags, - config=AltimateSourceConfig(**exposure.config.dict()) if exposure.config else None, + config=AltimateSourceConfig(**exposure.config.model_dump()) if exposure.config else None, unrendered_config=exposure.unrendered_config, url=exposure.url, - depends_on=AltimateDependsOn( - nodes=exposure.depends_on.nodes, - macros=exposure.depends_on.macros, - ) - if exposure.depends_on - else None, - refs=[AltimateRefArgs(**ref.dict()) for ref in exposure.refs] if exposure.refs else None, + depends_on=( + AltimateDependsOn( + nodes=exposure.depends_on.nodes, + macros=exposure.depends_on.macros, + ) + if exposure.depends_on + else None + ), + refs=[AltimateRefArgs(**ref.model_dump()) for ref in exposure.refs] if exposure.refs else None, sources=exposure.sources, metrics=exposure.metrics, created_at=exposure.created_at, @@ -219,7 +222,7 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: test_metadata = None if isinstance(test, GenericTestNode): test_type = GENERIC - test_metadata = AltimateTestMetadata(**test.test_metadata.dict()) if test.test_metadata else None + test_metadata = AltimateTestMetadata(**test.test_metadata.model_dump()) if test.test_metadata else None elif isinstance(test, SingularTestNode): test_type = SINGULAR else: @@ -235,42 +238,48 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: unique_id=test.unique_id, fqn=test.fqn, alias=test.alias, - checksum=AltimateFileHash( - name=test.checksum.name, - checksum=test.checksum.checksum, - ) - if test.checksum - else None, - config=AltimateTestConfig(**test.config.dict()) if test.config else None, + checksum=( + AltimateFileHash( + name=test.checksum.name, + checksum=test.checksum.checksum, + ) + if test.checksum + else None + ), + config=AltimateTestConfig(**test.config.model_dump()) if test.config else None, description=test.description, tags=test.tags, - columns={ - name: AltimateManifestColumnInfo( - name=column.name, - description=column.description, - meta=column.meta, - data_type=column.data_type, - quote=column.quote, - tags=column.tags, - ) - for name, column in test.columns.items() - } - if test.columns - else None, + columns=( + { + name: AltimateManifestColumnInfo( + name=column.name, + description=column.description, + meta=column.meta, + data_type=column.data_type, + quote=column.quote, + tags=column.tags, + ) + for name, column in test.columns.items() + } + if test.columns + else None + ), meta=test.meta, relation_name=test.relation_name, group=test.group, raw_code=test.raw_code, language=test.language, - refs=[AltimateRefArgs(**ref.dict()) for ref in test.refs] if test.refs else None, + refs=[AltimateRefArgs(**ref.model_dump()) for ref in test.refs] if test.refs else None, sources=test.sources, metrics=test.metrics, - depends_on=AltimateDependsOn( - nodes=test.depends_on.nodes, - macros=test.depends_on.macros, - ) - if test.depends_on - else None, + depends_on=( + AltimateDependsOn( + nodes=test.depends_on.nodes, + macros=test.depends_on.macros, + ) + if test.depends_on + else None + ), compiled_path=test.compiled_path, compiled=test.compiled, compiled_code=test.compiled_code, @@ -288,31 +297,35 @@ def _get_seed(self, seed: SeedNodeMap) -> AltimateSeedNode: unique_id=seed.unique_id, fqn=seed.fqn, alias=seed.alias, - checksum=AltimateFileHash( - name=seed.checksum.name, - checksum=seed.checksum.checksum, - ) - if seed.checksum - else None, - config=AltimateSeedConfig(**seed.config.dict()) if seed.config else None, + checksum=( + AltimateFileHash( + name=seed.checksum.name, + checksum=seed.checksum.checksum, + ) + if seed.checksum + else None + ), + config=AltimateSeedConfig(**seed.config.model_dump()) if seed.config else None, description=seed.description, tags=seed.tags, - columns={ - name: AltimateManifestColumnInfo( - name=column.name, - description=column.description, - meta=column.meta, - data_type=column.data_type, - quote=column.quote, - tags=column.tags, - ) - for name, column in seed.columns.items() - } - if seed.columns - else None, + columns=( + { + name: AltimateManifestColumnInfo( + name=column.name, + description=column.description, + meta=column.meta, + data_type=column.data_type, + quote=column.quote, + tags=column.tags, + ) + for name, column in seed.columns.items() + } + if seed.columns + else None + ), meta=seed.meta, group=seed.group, - docs=seed.docs.dict() if seed.docs else None, + docs=seed.docs.model_dump() if seed.docs else None, patch_path=seed.patch_path, build_path=seed.build_path, deferred=seed.deferred, diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py index c9d04731..8acfc0f3 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py @@ -61,7 +61,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: language, contract, ) = ([], [], None, None, None, None, None, "", "", None) - if node.resource_type.value != SEED: + if node.resource_type != SEED: sources = node.sources metrics = node.metrics depends_on_nodes = node.depends_on.nodes if node.depends_on else None @@ -77,7 +77,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: database=node.database, schema_name=node.schema_, name=node.name, - resource_type=AltimateResourceType(node.resource_type.value), + resource_type=AltimateResourceType(node.resource_type), package_name=node.package_name, path=node.path, description=node.description, @@ -116,12 +116,13 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: contract=contract, meta=node.meta, patch_path=node.patch_path, + access=node.access.value, ) def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: return AltimateManifestSourceNode( database=source.database, - resource_type=AltimateResourceType(source.resource_type.value), + resource_type=AltimateResourceType(source.resource_type), schema_name=source.schema_, name=source.name, package_name=source.package_name, @@ -133,10 +134,10 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: source_description=source.source_description, loader=source.loader, identifier=source.identifier, - quoting=AltimateQuoting(**source.quoting.dict()) if source.quoting else None, + quoting=AltimateQuoting(**source.quoting.model_dump()) if source.quoting else None, loaded_at_field=source.loaded_at_field, - freshness=AltimateFreshnessThreshold(**source.freshness.dict()) if source.freshness else None, - external=AltimateExternalTable(**source.external.dict()) if source.external else None, + freshness=AltimateFreshnessThreshold(**source.freshness.model_dump()) if source.freshness else None, + external=AltimateExternalTable(**source.external.model_dump()) if source.external else None, description=source.description, columns={ name: AltimateManifestColumnInfo( @@ -153,7 +154,7 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: relation_name=source.relation_name, source_meta=source.source_meta, tags=source.tags, - config=AltimateSourceConfig(**source.config.dict()) if source.config else None, + config=AltimateSourceConfig(**source.config.model_dump()) if source.config else None, patch_path=source.patch_path, unrendered_config=source.unrendered_config, created_at=source.created_at, @@ -162,7 +163,7 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: return AltimateManifestMacroNode( name=macro.name, - resource_type=AltimateResourceType(macro.resource_type.value), + resource_type=AltimateResourceType(macro.resource_type), package_name=macro.package_name, path=macro.path, original_file_path=macro.original_file_path, @@ -177,9 +178,9 @@ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: ), description=macro.description, meta=macro.meta, - docs=macro.docs, + docs=macro.docs.model_dump() if macro.docs else None, patch_path=macro.patch_path, - arguments=[AltimateMacroArgument(**arg.dict()) for arg in macro.arguments] if macro.arguments else None, + arguments=[AltimateMacroArgument(**arg.model_dump()) for arg in macro.arguments] if macro.arguments else None, created_at=macro.created_at, supported_languages=macro.supported_languages, ) @@ -187,29 +188,31 @@ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: return AltimateManifestExposureNode( name=exposure.name, - resource_type=AltimateResourceType(exposure.resource_type.value), + resource_type=AltimateResourceType(exposure.resource_type), package_name=exposure.package_name, path=exposure.path, original_file_path=exposure.original_file_path, unique_id=exposure.unique_id, fqn=exposure.fqn, type=AltimateExposureType(exposure.type.value) if exposure.type else None, - owner=AltimateOwner(**exposure.owner.dict()) if exposure.owner else None, + owner=AltimateOwner(**exposure.owner.model_dump()) if exposure.owner else None, description=exposure.description, label=exposure.label, maturity=AltimateMaturityEnum(exposure.maturity.value) if exposure.maturity else None, meta=exposure.meta, tags=exposure.tags, - config=AltimateSourceConfig(**exposure.config.dict()) if exposure.config else None, + config=AltimateSourceConfig(**exposure.config.model_dump()) if exposure.config else None, unrendered_config=exposure.unrendered_config, url=exposure.url, - depends_on=AltimateDependsOn( - nodes=exposure.depends_on.nodes, - macros=exposure.depends_on.macros, - ) - if exposure.depends_on - else None, - refs=[AltimateRefArgs(**ref.dict()) for ref in exposure.refs] if exposure.refs else None, + depends_on=( + AltimateDependsOn( + nodes=exposure.depends_on.nodes, + macros=exposure.depends_on.macros, + ) + if exposure.depends_on + else None + ), + refs=[AltimateRefArgs(**ref.model_dump()) for ref in exposure.refs] if exposure.refs else None, sources=exposure.sources, metrics=exposure.metrics, created_at=exposure.created_at, @@ -219,7 +222,7 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: test_metadata = None if isinstance(test, GenericTestNode): test_type = GENERIC - test_metadata = AltimateTestMetadata(**test.test_metadata.dict()) if test.test_metadata else None + test_metadata = AltimateTestMetadata(**test.test_metadata.model_dump()) if test.test_metadata else None elif isinstance(test, SingularTestNode): test_type = SINGULAR else: @@ -228,49 +231,55 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: test_metadata=test_metadata, test_type=test_type, name=test.name, - resource_type=AltimateResourceType(test.resource_type.value), + resource_type=AltimateResourceType(test.resource_type), package_name=test.package_name, path=test.path, original_file_path=test.original_file_path, unique_id=test.unique_id, fqn=test.fqn, alias=test.alias, - checksum=AltimateFileHash( - name=test.checksum.name, - checksum=test.checksum.checksum, - ) - if test.checksum - else None, - config=AltimateTestConfig(**test.config.dict()) if test.config else None, + checksum=( + AltimateFileHash( + name=test.checksum.name, + checksum=test.checksum.checksum, + ) + if test.checksum + else None + ), + config=AltimateTestConfig(**test.config.model_dump()) if test.config else None, description=test.description, tags=test.tags, - columns={ - name: AltimateManifestColumnInfo( - name=column.name, - description=column.description, - meta=column.meta, - data_type=column.data_type, - quote=column.quote, - tags=column.tags, - ) - for name, column in test.columns.items() - } - if test.columns - else None, + columns=( + { + name: AltimateManifestColumnInfo( + name=column.name, + description=column.description, + meta=column.meta, + data_type=column.data_type, + quote=column.quote, + tags=column.tags, + ) + for name, column in test.columns.items() + } + if test.columns + else None + ), meta=test.meta, relation_name=test.relation_name, group=test.group, raw_code=test.raw_code, language=test.language, - refs=[AltimateRefArgs(**ref.dict()) for ref in test.refs] if test.refs else None, + refs=[AltimateRefArgs(**ref.model_dump()) for ref in test.refs] if test.refs else None, sources=test.sources, metrics=test.metrics, - depends_on=AltimateDependsOn( - nodes=test.depends_on.nodes, - macros=test.depends_on.macros, - ) - if test.depends_on - else None, + depends_on=( + AltimateDependsOn( + nodes=test.depends_on.nodes, + macros=test.depends_on.macros, + ) + if test.depends_on + else None + ), compiled_path=test.compiled_path, compiled=test.compiled, compiled_code=test.compiled_code, @@ -281,38 +290,42 @@ def _get_seed(self, seed: SeedNodeMap) -> AltimateSeedNode: database=seed.database, schema_name=seed.schema_, name=seed.name, - resource_type=AltimateResourceType(seed.resource_type.value), + resource_type=AltimateResourceType(seed.resource_type), package_name=seed.package_name, path=seed.path, original_file_path=seed.original_file_path, unique_id=seed.unique_id, fqn=seed.fqn, alias=seed.alias, - checksum=AltimateFileHash( - name=seed.checksum.name, - checksum=seed.checksum.checksum, - ) - if seed.checksum - else None, - config=AltimateSeedConfig(**seed.config.dict()) if seed.config else None, + checksum=( + AltimateFileHash( + name=seed.checksum.name, + checksum=seed.checksum.checksum, + ) + if seed.checksum + else None + ), + config=AltimateSeedConfig(**seed.config.model_dump()) if seed.config else None, description=seed.description, tags=seed.tags, - columns={ - name: AltimateManifestColumnInfo( - name=column.name, - description=column.description, - meta=column.meta, - data_type=column.data_type, - quote=column.quote, - tags=column.tags, - ) - for name, column in seed.columns.items() - } - if seed.columns - else None, + columns=( + { + name: AltimateManifestColumnInfo( + name=column.name, + description=column.description, + meta=column.meta, + data_type=column.data_type, + quote=column.quote, + tags=column.tags, + ) + for name, column in seed.columns.items() + } + if seed.columns + else None + ), meta=seed.meta, group=seed.group, - docs=seed.docs.dict() if seed.docs else None, + docs=seed.docs.model_dump() if seed.docs else None, patch_path=seed.patch_path, build_path=seed.build_path, deferred=seed.deferred, @@ -327,7 +340,7 @@ def get_nodes( nodes = {} for node in self.manifest.nodes.values(): if ( - node.resource_type.value + node.resource_type in [ AltimateResourceType.seed.value, AltimateResourceType.test.value, @@ -350,7 +363,7 @@ def get_sources(self) -> Dict[str, AltimateManifestSourceNode]: def get_macros(self) -> Dict[str, AltimateManifestMacroNode]: macros = {} for macro in self.manifest.macros.values(): - if macro.resource_type.value == AltimateResourceType.macro.value and macro.package_name == self.get_package(): + if macro.resource_type == AltimateResourceType.macro.value and macro.package_name == self.get_package(): macros[macro.unique_id] = self._get_macro(macro) return macros @@ -371,7 +384,7 @@ def get_tests(self, type=None) -> Dict[str, AltimateManifestTestNode]: for node in self.manifest.nodes.values(): # Check if the node is a test and of the correct type - if node.resource_type.value == AltimateResourceType.test.value: + if node.resource_type == AltimateResourceType.test.value: if any(isinstance(node, t) for t in types): tests[node.unique_id] = self._get_tests(node) return tests @@ -379,7 +392,7 @@ def get_tests(self, type=None) -> Dict[str, AltimateManifestTestNode]: def get_seeds(self) -> Dict[str, AltimateSeedNode]: seeds = {} for seed in self.manifest.nodes.values(): - if seed.resource_type.value == AltimateResourceType.seed.value: + if seed.resource_type == AltimateResourceType.seed.value: seeds[seed.unique_id] = self._get_seed(seed) return seeds diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py index bba6ef85..b7f1722f 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py @@ -4,33 +4,33 @@ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Exposures from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Macros -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node1 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node3 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node4 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node5 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node7 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes1 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes2 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes3 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes4 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes5 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes6 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes7 from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Sources from datapilot.core.platforms.dbt.constants import GENERIC from datapilot.core.platforms.dbt.constants import SINGULAR -ManifestNode = Union[Node, Node1, Node2, Node3, Node4, Node5, Node6, Node7] +ManifestNode = Union[Nodes, Nodes1, Nodes2, Nodes3, Nodes4, Nodes5, Nodes6, Nodes7] SourceNode = Sources ExposureNode = Exposures -TestNode = Union[Node6, Node2] +TestNode = Union[Nodes6, Nodes2] MacroNode = Macros TEST_TYPE_TO_NODE_MAP: Dict[str, Type] = { - GENERIC: [Node6], - SINGULAR: [Node2], + GENERIC: [Nodes6], + SINGULAR: [Nodes2], } -SeedNodeMap = Node +SeedNodeMap = Nodes diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py index d694f514..b84715f1 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py @@ -3,8 +3,8 @@ from typing import Set from dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2 -from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes2 +from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Nodes6 from datapilot.core.platforms.dbt.constants import GENERIC from datapilot.core.platforms.dbt.constants import OTHER_TEST_NODE @@ -116,6 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: contract=contract, meta=node.meta, patch_path=node.patch_path, + access=node.access.value, ) def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: @@ -133,10 +134,10 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: source_description=source.source_description, loader=source.loader, identifier=source.identifier, - quoting=AltimateQuoting(**source.quoting.dict()) if source.quoting else None, + quoting=AltimateQuoting(**source.quoting.model_dump()) if source.quoting else None, loaded_at_field=source.loaded_at_field, - freshness=AltimateFreshnessThreshold(**source.freshness.dict()) if source.freshness else None, - external=AltimateExternalTable(**source.external.dict()) if source.external else None, + freshness=AltimateFreshnessThreshold(**source.freshness.model_dump()) if source.freshness else None, + external=AltimateExternalTable(**source.external.model_dump()) if source.external else None, description=source.description, columns={ name: AltimateManifestColumnInfo( @@ -153,7 +154,7 @@ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: relation_name=source.relation_name, source_meta=source.source_meta, tags=source.tags, - config=AltimateSourceConfig(**source.config.dict()) if source.config else None, + config=AltimateSourceConfig(**source.config.model_dump()) if source.config else None, patch_path=source.patch_path, unrendered_config=source.unrendered_config, created_at=source.created_at, @@ -177,9 +178,9 @@ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode: ), description=macro.description, meta=macro.meta, - docs=macro.docs, + docs=macro.docs.model_dump() if macro.docs else None, patch_path=macro.patch_path, - arguments=[AltimateMacroArgument(**arg.dict()) for arg in macro.arguments] if macro.arguments else None, + arguments=[AltimateMacroArgument(**arg.model_dump()) for arg in macro.arguments] if macro.arguments else None, created_at=macro.created_at, supported_languages=macro.supported_languages, ) @@ -194,13 +195,13 @@ def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: unique_id=exposure.unique_id, fqn=exposure.fqn, type=AltimateExposureType(exposure.type.value) if exposure.type else None, - owner=AltimateOwner(**exposure.owner.dict()) if exposure.owner else None, + owner=AltimateOwner(**exposure.owner.model_dump()) if exposure.owner else None, description=exposure.description, label=exposure.label, maturity=AltimateMaturityEnum(exposure.maturity.value) if exposure.maturity else None, meta=exposure.meta, tags=exposure.tags, - config=AltimateSourceConfig(**exposure.config.dict()) if exposure.config else None, + config=AltimateSourceConfig(**exposure.config.model_dump()) if exposure.config else None, unrendered_config=exposure.unrendered_config, url=exposure.url, depends_on=( @@ -211,7 +212,7 @@ def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: if exposure.depends_on else None ), - refs=[AltimateRefArgs(**ref.dict()) for ref in exposure.refs] if exposure.refs else None, + refs=[AltimateRefArgs(**ref.model_dump()) for ref in exposure.refs] if exposure.refs else None, sources=exposure.sources, metrics=exposure.metrics, created_at=exposure.created_at, @@ -219,10 +220,10 @@ def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode: def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: test_metadata = None - if isinstance(test, Node6): + if isinstance(test, Nodes6): test_type = GENERIC - test_metadata = AltimateTestMetadata(**test.test_metadata.dict()) if test.test_metadata else None - elif isinstance(test, Node2): + test_metadata = AltimateTestMetadata(**test.test_metadata.model_dump()) if test.test_metadata else None + elif isinstance(test, Nodes2): test_type = SINGULAR else: test_type = OTHER_TEST_NODE @@ -245,7 +246,7 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: if test.checksum else None ), - config=AltimateTestConfig(**test.config.dict()) if test.config else None, + config=AltimateTestConfig(**test.config.model_dump()) if test.config else None, description=test.description, tags=test.tags, columns=( @@ -268,7 +269,7 @@ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode: group=test.group, raw_code=test.raw_code, language=test.language, - refs=[AltimateRefArgs(**ref.dict()) for ref in test.refs] if test.refs else None, + refs=[AltimateRefArgs(**ref.model_dump()) for ref in test.refs] if test.refs else None, sources=test.sources, metrics=test.metrics, depends_on=( @@ -304,7 +305,7 @@ def _get_seed(self, seed: SeedNodeMap) -> AltimateSeedNode: if seed.checksum else None ), - config=AltimateSeedConfig(**seed.config.dict()) if seed.config else None, + config=AltimateSeedConfig(**seed.config.model_dump()) if seed.config else None, description=seed.description, tags=seed.tags, columns=( @@ -324,7 +325,7 @@ def _get_seed(self, seed: SeedNodeMap) -> AltimateSeedNode: ), meta=seed.meta, group=seed.group, - docs=seed.docs.dict() if seed.docs else None, + docs=seed.docs.model_dump() if seed.docs else None, patch_path=seed.patch_path, build_path=seed.build_path, deferred=False, @@ -375,7 +376,7 @@ def get_exposures(self) -> Dict[str, AltimateManifestExposureNode]: def get_tests(self, type=None) -> Dict[str, AltimateManifestTestNode]: tests = {} # Initialize types_union with TestNode - types = [Node2, Node6] + types = [Nodes2, Nodes6] # Add other types to the union if provided if type: