From 9358e35c56741cf8876bf4dbd605fa28752d3747 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 26 Dec 2025 12:14:56 +0530 Subject: [PATCH 1/6] add new dbt artifacts support --- src/datapilot/clients/altimate/constants.py | 10 ++ src/datapilot/clients/altimate/utils.py | 22 +++++ src/datapilot/core/platforms/dbt/cli/cli.py | 91 +++++++++++++++++-- .../core/platforms/dbt/schemas/run_results.py | 44 +++++++++ .../dbt/schemas/semantic_manifest.py | 12 +++ .../core/platforms/dbt/schemas/sources.py | 26 ++++++ src/datapilot/core/platforms/dbt/utils.py | 54 ++++++++++- src/vendor/dbt_artifacts_parser/parser.py | 18 ++++ .../parsers/semantic_manifest/__init__.py | 19 ++++ .../semantic_manifest/semantic_manifest_v1.py | 44 +++++++++ .../parsers/version_map.py | 3 + tests/clients/__init__.py | 0 tests/clients/altimate/__init__.py | 0 tests/clients/altimate/test_utils.py | 26 ++++++ .../platform/dbt/test_artifact_loaders.py | 53 +++++++++++ tests/core/platform/dbt/test_cli.py | 14 +++ tests/data/run_results_v6.json | 33 +++++++ tests/data/semantic_manifest_v1.json | 58 ++++++++++++ tests/data/sources_v3.json | 39 ++++++++ 19 files changed, 555 insertions(+), 11 deletions(-) create mode 100644 src/datapilot/clients/altimate/constants.py create mode 100644 src/datapilot/core/platforms/dbt/schemas/run_results.py create mode 100644 src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py create mode 100644 src/datapilot/core/platforms/dbt/schemas/sources.py create mode 100644 src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py create mode 100644 src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py create mode 100644 tests/clients/__init__.py create mode 100644 tests/clients/altimate/__init__.py create mode 100644 tests/clients/altimate/test_utils.py create mode 100644 tests/core/platform/dbt/test_artifact_loaders.py create mode 100644 tests/data/run_results_v6.json create mode 100644 tests/data/semantic_manifest_v1.json create mode 100644 tests/data/sources_v3.json diff --git a/src/datapilot/clients/altimate/constants.py b/src/datapilot/clients/altimate/constants.py new file mode 100644 index 00000000..25118b83 --- /dev/null +++ b/src/datapilot/clients/altimate/constants.py @@ -0,0 +1,10 @@ +"""Constants for the Altimate API client.""" + +# Supported dbt artifact file types for onboarding +SUPPORTED_ARTIFACT_TYPES = { + "manifest", + "catalog", + "run_results", + "sources", + "semantic_manifest", +} diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py index 449851e7..87e40113 100644 --- a/src/datapilot/clients/altimate/utils.py +++ b/src/datapilot/clients/altimate/utils.py @@ -7,6 +7,7 @@ from requests import Response from datapilot.clients.altimate.client import APIClient +from datapilot.clients.altimate.constants import SUPPORTED_ARTIFACT_TYPES def check_token_and_instance( @@ -56,6 +57,27 @@ def validate_permissions( def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict: + """ + Upload a dbt artifact file to the Altimate backend. + + Args: + api_token: API authentication token + tenant: Tenant/instance name + dbt_core_integration_id: ID of the dbt integration + dbt_core_integration_environment: Environment type (e.g., PROD) + file_type: Type of artifact - one of: manifest, catalog, run_results, sources, semantic_manifest + file_path: Path to the artifact file + backend_url: URL of the Altimate backend + + Returns: + Dict with 'ok' boolean and optional 'message' on failure + """ + if file_type not in SUPPORTED_ARTIFACT_TYPES: + return { + "ok": False, + "message": f"Unsupported file type: {file_type}. Supported types: {', '.join(sorted(SUPPORTED_ARTIFACT_TYPES))}", + } + api_client = APIClient(api_token, base_url=backend_url, tenant=tenant) params = { diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 5fc8f06b..751cd498 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -18,6 +18,9 @@ from datapilot.core.platforms.dbt.formatting import generate_project_insights_table from datapilot.core.platforms.dbt.utils import load_catalog from datapilot.core.platforms.dbt.utils import load_manifest +from datapilot.core.platforms.dbt.utils import load_run_results +from datapilot.core.platforms.dbt.utils import load_semantic_manifest +from datapilot.core.platforms.dbt.utils import load_sources from datapilot.utils.formatting.utils import tabulate_data from datapilot.utils.utils import map_url_to_instance @@ -155,6 +158,9 @@ def project_health( ) @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") @click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.") +@click.option("--run-results-path", required=False, prompt=False, help="Path to the run_results.json file.") +@click.option("--sources-path", required=False, prompt=False, help="Path to the sources.json file (source freshness results).") +@click.option("--semantic-manifest-path", required=False, prompt=False, help="Path to the semantic_manifest.json file.") def onboard( token, instance_name, @@ -164,6 +170,9 @@ def onboard( dbt_integration_environment, manifest_path, catalog_path, + run_results_path, + sources_path, + semantic_manifest_path, ): """Onboard a manifest file to DBT. You can specify either --dbt_integration_id or --dbt_integration_name.""" @@ -198,34 +207,98 @@ def onboard( elif dbt_integration_name and dbt_integration_id: click.echo("Warning: Both integration ID and name provided. Using ID and ignoring name.") + # Validate manifest (required) try: load_manifest(manifest_path) except Exception as e: click.echo(f"Error: {e}") return + # Validate optional artifacts if provided + if catalog_path: + try: + load_catalog(catalog_path) + except Exception as e: + click.echo(f"Error validating catalog: {e}") + return + + if run_results_path: + try: + load_run_results(run_results_path) + except Exception as e: + click.echo(f"Error validating run_results: {e}") + return + + if sources_path: + try: + load_sources(sources_path) + except Exception as e: + click.echo(f"Error validating sources: {e}") + return + + if semantic_manifest_path: + try: + load_semantic_manifest(semantic_manifest_path) + except Exception as e: + click.echo(f"Error validating semantic_manifest: {e}") + return + + # Onboard manifest (required) response = onboard_file(token, instance_name, dbt_integration_id, dbt_integration_environment, "manifest", manifest_path, backend_url) if response["ok"]: click.echo("Manifest onboarded successfully!") else: click.echo(f"{response['message']}") - - if not catalog_path: return - response = onboard_file(token, instance_name, dbt_integration_id, dbt_integration_environment, "catalog", catalog_path, backend_url) - if response["ok"]: - click.echo("Catalog onboarded successfully!") - else: - click.echo(f"{response['message']}") + # Onboard optional artifacts + artifacts_uploaded = ["manifest"] + + if catalog_path: + response = onboard_file(token, instance_name, dbt_integration_id, dbt_integration_environment, "catalog", catalog_path, backend_url) + if response["ok"]: + click.echo("Catalog onboarded successfully!") + artifacts_uploaded.append("catalog") + else: + click.echo(f"{response['message']}") + + if run_results_path: + response = onboard_file( + token, instance_name, dbt_integration_id, dbt_integration_environment, "run_results", run_results_path, backend_url + ) + if response["ok"]: + click.echo("Run results onboarded successfully!") + artifacts_uploaded.append("run_results") + else: + click.echo(f"{response['message']}") + + if sources_path: + response = onboard_file(token, instance_name, dbt_integration_id, dbt_integration_environment, "sources", sources_path, backend_url) + if response["ok"]: + click.echo("Sources onboarded successfully!") + artifacts_uploaded.append("sources") + else: + click.echo(f"{response['message']}") + + if semantic_manifest_path: + response = onboard_file( + token, instance_name, dbt_integration_id, dbt_integration_environment, "semantic_manifest", semantic_manifest_path, backend_url + ) + if response["ok"]: + click.echo("Semantic manifest onboarded successfully!") + artifacts_uploaded.append("semantic_manifest") + else: + click.echo(f"{response['message']}") + # Start ingestion response = start_dbt_ingestion(token, instance_name, dbt_integration_id, dbt_integration_environment, backend_url) if response["ok"]: url = map_url_to_instance(backend_url, instance_name) + artifacts_str = ", ".join(artifacts_uploaded) if not url: - click.echo("Manifest and catalog ingestion has started.") + click.echo(f"Ingestion has started for: {artifacts_str}") else: url = f"{url}/settings/integrations/{dbt_integration_id}/{dbt_integration_environment}" - click.echo(f"Manifest and catalog ingestion has started. You can check the status at {url}") + click.echo(f"Ingestion has started for: {artifacts_str}. You can check the status at {url}") else: click.echo(f"{response['message']}") diff --git a/src/datapilot/core/platforms/dbt/schemas/run_results.py b/src/datapilot/core/platforms/dbt/schemas/run_results.py new file mode 100644 index 00000000..b6bc8523 --- /dev/null +++ b/src/datapilot/core/platforms/dbt/schemas/run_results.py @@ -0,0 +1,44 @@ +from typing import Union + +from pydantic import ConfigDict + +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v1 import RunResultsV1 as BaseRunResultsV1 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v2 import RunResultsV2 as BaseRunResultsV2 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v3 import RunResultsV3 as BaseRunResultsV3 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 as BaseRunResultsV4 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 as BaseRunResultsV5 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 as BaseRunResultsV6 + + +class RunResultsV1(BaseRunResultsV1): + model_config = ConfigDict(extra="allow") + + +class RunResultsV2(BaseRunResultsV2): + model_config = ConfigDict(extra="allow") + + +class RunResultsV3(BaseRunResultsV3): + model_config = ConfigDict(extra="allow") + + +class RunResultsV4(BaseRunResultsV4): + model_config = ConfigDict(extra="allow") + + +class RunResultsV5(BaseRunResultsV5): + model_config = ConfigDict(extra="allow") + + +class RunResultsV6(BaseRunResultsV6): + model_config = ConfigDict(extra="allow") + + +RunResults = Union[ + RunResultsV6, + RunResultsV5, + RunResultsV4, + RunResultsV3, + RunResultsV2, + RunResultsV1, +] diff --git a/src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py b/src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py new file mode 100644 index 00000000..c92b7b7d --- /dev/null +++ b/src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py @@ -0,0 +1,12 @@ +from typing import Union + +from pydantic import ConfigDict + +from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 as BaseSemanticManifestV1 + + +class SemanticManifestV1(BaseSemanticManifestV1): + model_config = ConfigDict(extra="allow") + + +SemanticManifest = Union[SemanticManifestV1] diff --git a/src/datapilot/core/platforms/dbt/schemas/sources.py b/src/datapilot/core/platforms/dbt/schemas/sources.py new file mode 100644 index 00000000..8d66a3a7 --- /dev/null +++ b/src/datapilot/core/platforms/dbt/schemas/sources.py @@ -0,0 +1,26 @@ +from typing import Union + +from pydantic import ConfigDict + +from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 as BaseSourcesV1 +from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 as BaseSourcesV2 +from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 as BaseSourcesV3 + + +class SourcesV1(BaseSourcesV1): + model_config = ConfigDict(extra="allow") + + +class SourcesV2(BaseSourcesV2): + model_config = ConfigDict(extra="allow") + + +class SourcesV3(BaseSourcesV3): + model_config = ConfigDict(extra="allow") + + +Sources = Union[ + SourcesV3, + SourcesV2, + SourcesV1, +] diff --git a/src/datapilot/core/platforms/dbt/utils.py b/src/datapilot/core/platforms/dbt/utils.py index c94610d0..70560b28 100644 --- a/src/datapilot/core/platforms/dbt/utils.py +++ b/src/datapilot/core/platforms/dbt/utils.py @@ -22,6 +22,9 @@ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestSourceNode from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestTestNode from datapilot.core.platforms.dbt.schemas.manifest import Manifest +from datapilot.core.platforms.dbt.schemas.run_results import RunResults +from datapilot.core.platforms.dbt.schemas.semantic_manifest import SemanticManifest +from datapilot.core.platforms.dbt.schemas.sources import Sources from datapilot.exceptions.exceptions import AltimateFileNotFoundError from datapilot.exceptions.exceptions import AltimateInvalidJSONError from datapilot.utils.utils import extract_dir_name_from_file_path @@ -29,6 +32,9 @@ from datapilot.utils.utils import is_superset_path from datapilot.utils.utils import load_json from vendor.dbt_artifacts_parser.parser import parse_manifest +from vendor.dbt_artifacts_parser.parser import parse_run_results +from vendor.dbt_artifacts_parser.parser import parse_semantic_manifest +from vendor.dbt_artifacts_parser.parser import parse_sources MODEL_TYPE_PATTERNS = { STAGING: r"^stg_.*", # Example: models starting with 'stg_' @@ -94,8 +100,52 @@ def load_catalog(catalog_path: str) -> Catalog: return catalog -def load_run_results(run_results_path: str) -> Manifest: - raise NotImplementedError +def load_run_results(run_results_path: str) -> RunResults: + try: + run_results_dict = load_json(run_results_path) + except FileNotFoundError as e: + raise AltimateFileNotFoundError(f"Run results file not found: {run_results_path}. Error: {e}") from e + except ValueError as e: + raise AltimateInvalidJSONError(f"Invalid JSON file: {run_results_path}. Error: {e}") from e + + try: + run_results: RunResults = parse_run_results(run_results_dict) + except ValueError as e: + raise AltimateInvalidManifestError(f"Invalid run results file: {run_results_path}. Error: {e}") from e + + return run_results + + +def load_sources(sources_path: str) -> Sources: + try: + sources_dict = load_json(sources_path) + except FileNotFoundError as e: + raise AltimateFileNotFoundError(f"Sources file not found: {sources_path}. Error: {e}") from e + except ValueError as e: + raise AltimateInvalidJSONError(f"Invalid JSON file: {sources_path}. Error: {e}") from e + + try: + sources: Sources = parse_sources(sources_dict) + except ValueError as e: + raise AltimateInvalidManifestError(f"Invalid sources file: {sources_path}. Error: {e}") from e + + return sources + + +def load_semantic_manifest(semantic_manifest_path: str) -> SemanticManifest: + try: + semantic_manifest_dict = load_json(semantic_manifest_path) + except FileNotFoundError as e: + raise AltimateFileNotFoundError(f"Semantic manifest file not found: {semantic_manifest_path}. Error: {e}") from e + except ValueError as e: + raise AltimateInvalidJSONError(f"Invalid JSON file: {semantic_manifest_path}. Error: {e}") from e + + try: + semantic_manifest: SemanticManifest = parse_semantic_manifest(semantic_manifest_dict) + except ValueError as e: + raise AltimateInvalidManifestError(f"Invalid semantic manifest file: {semantic_manifest_path}. Error: {e}") from e + + return semantic_manifest # TODO: Add tests! diff --git a/src/vendor/dbt_artifacts_parser/parser.py b/src/vendor/dbt_artifacts_parser/parser.py index 0a7511f5..646c16f9 100644 --- a/src/vendor/dbt_artifacts_parser/parser.py +++ b/src/vendor/dbt_artifacts_parser/parser.py @@ -35,6 +35,7 @@ from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 +from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 @@ -345,3 +346,20 @@ def parse_sources_v3(sources: dict) -> SourcesV3: if dbt_schema_version == ArtifactTypes.SOURCES_V3.value.dbt_schema_version: return SourcesV3(**sources) raise ValueError("Not a sources.json v3") + + +# +# semantic-manifest +# +def parse_semantic_manifest(semantic_manifest: dict) -> SemanticManifestV1: + """Parse semantic_manifest.json + + Args: + semantic_manifest: A dict of semantic_manifest.json + + Returns: + SemanticManifestV1 + """ + # Semantic manifest uses a flexible schema, so we accept any valid dict + # The schema version check is optional since the format may vary + return SemanticManifestV1(**semantic_manifest) diff --git a/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py b/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py new file mode 100644 index 00000000..137cf772 --- /dev/null +++ b/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 + +__all__ = ["SemanticManifestV1"] diff --git a/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py b/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py new file mode 100644 index 00000000..991514aa --- /dev/null +++ b/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from typing import Any +from typing import Optional + +from pydantic import ConfigDict + +from vendor.dbt_artifacts_parser.parsers.base import BaseParserModel + + +class SemanticManifestV1(BaseParserModel): + """ + Semantic Manifest artifact produced by dbt when parsing a project with semantic layer definitions. + + The semantic manifest contains: + - semantic_models: Starting points of data with entities, dimensions, and measures + - metrics: Functions combining measures, constraints to define quantitative indicators + - project_configuration: Project-level settings including time spine configurations + - saved_queries: Pre-built queries for common MetricFlow use cases + """ + + model_config = ConfigDict(extra="allow") + + semantic_models: Optional[list[dict[str, Any]]] = None + metrics: Optional[list[dict[str, Any]]] = None + project_configuration: Optional[dict[str, Any]] = None + saved_queries: Optional[list[dict[str, Any]]] = None diff --git a/src/vendor/dbt_artifacts_parser/parsers/version_map.py b/src/vendor/dbt_artifacts_parser/parsers/version_map.py index 69dbd0fe..7c197b31 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/version_map.py +++ b/src/vendor/dbt_artifacts_parser/parsers/version_map.py @@ -38,6 +38,7 @@ from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 +from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 @@ -78,3 +79,5 @@ class ArtifactTypes(Enum): SOURCES_V1 = ArtifactType("https://schemas.getdbt.com/dbt/sources/v1.json", SourcesV1) SOURCES_V2 = ArtifactType("https://schemas.getdbt.com/dbt/sources/v2.json", SourcesV2) SOURCES_V3 = ArtifactType("https://schemas.getdbt.com/dbt/sources/v3.json", SourcesV3) + # Semantic Manifest + SEMANTIC_MANIFEST_V1 = ArtifactType("https://schemas.getdbt.com/dbt/semantic-manifest/v1.json", SemanticManifestV1) diff --git a/tests/clients/__init__.py b/tests/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/clients/altimate/__init__.py b/tests/clients/altimate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/clients/altimate/test_utils.py b/tests/clients/altimate/test_utils.py new file mode 100644 index 00000000..30df77ff --- /dev/null +++ b/tests/clients/altimate/test_utils.py @@ -0,0 +1,26 @@ +from datapilot.clients.altimate.constants import SUPPORTED_ARTIFACT_TYPES +from datapilot.clients.altimate.utils import onboard_file + + +class TestOnboardFile: + def test_supported_artifact_types(self): + """Test that all expected artifact types are supported.""" + expected_types = {"manifest", "catalog", "run_results", "sources", "semantic_manifest"} + assert SUPPORTED_ARTIFACT_TYPES == expected_types + + def test_unsupported_file_type_returns_error(self): + """Test that unsupported file types return an error without making API calls.""" + test_token = "test_token" # noqa: S105 + result = onboard_file( + api_token=test_token, + tenant="test_tenant", + dbt_core_integration_id="test_id", + dbt_core_integration_environment="PROD", + file_type="unsupported_type", + file_path="test_path.json", + backend_url="http://localhost", + ) + + assert result["ok"] is False + assert "Unsupported file type" in result["message"] + assert "unsupported_type" in result["message"] diff --git a/tests/core/platform/dbt/test_artifact_loaders.py b/tests/core/platform/dbt/test_artifact_loaders.py new file mode 100644 index 00000000..a5f7dbe0 --- /dev/null +++ b/tests/core/platform/dbt/test_artifact_loaders.py @@ -0,0 +1,53 @@ +import pytest + +from datapilot.core.platforms.dbt.utils import load_run_results +from datapilot.core.platforms.dbt.utils import load_semantic_manifest +from datapilot.core.platforms.dbt.utils import load_sources +from datapilot.exceptions.exceptions import AltimateFileNotFoundError + + +class TestLoadRunResults: + def test_load_run_results_v6(self): + run_results_path = "tests/data/run_results_v6.json" + run_results = load_run_results(run_results_path) + + assert run_results is not None + assert run_results.metadata.dbt_schema_version == "https://schemas.getdbt.com/dbt/run-results/v6.json" + assert len(run_results.results) == 1 + assert run_results.results[0].status.value == "success" + assert run_results.results[0].unique_id == "model.jaffle_shop.stg_customers" + + def test_load_run_results_file_not_found(self): + with pytest.raises(AltimateFileNotFoundError): + load_run_results("nonexistent_file.json") + + +class TestLoadSources: + def test_load_sources_v3(self): + sources_path = "tests/data/sources_v3.json" + sources = load_sources(sources_path) + + assert sources is not None + assert sources.metadata.dbt_schema_version == "https://schemas.getdbt.com/dbt/sources/v3.json" + assert len(sources.results) == 1 + assert sources.results[0].unique_id == "source.jaffle_shop.raw.customers" + + def test_load_sources_file_not_found(self): + with pytest.raises(AltimateFileNotFoundError): + load_sources("nonexistent_file.json") + + +class TestLoadSemanticManifest: + def test_load_semantic_manifest_v1(self): + semantic_manifest_path = "tests/data/semantic_manifest_v1.json" + semantic_manifest = load_semantic_manifest(semantic_manifest_path) + + assert semantic_manifest is not None + assert len(semantic_manifest.semantic_models) == 1 + assert semantic_manifest.semantic_models[0]["name"] == "orders" + assert len(semantic_manifest.metrics) == 1 + assert semantic_manifest.metrics[0]["name"] == "revenue" + + def test_load_semantic_manifest_file_not_found(self): + with pytest.raises(AltimateFileNotFoundError): + load_semantic_manifest("nonexistent_file.json") diff --git a/tests/core/platform/dbt/test_cli.py b/tests/core/platform/dbt/test_cli.py index b7f6f45f..4cdb3921 100644 --- a/tests/core/platform/dbt/test_cli.py +++ b/tests/core/platform/dbt/test_cli.py @@ -113,3 +113,17 @@ def test_project_health_with_required_and_optional_args_v12(): # Add more assertions here to validate the behavior of your command, # for example, checking that the output contains expected text. assert "-----------" in result.output + + +def test_onboard_help_shows_all_artifact_options(): + """Test that the onboard command shows all artifact options in help.""" + runner = CliRunner() + + result = runner.invoke(datapilot, ["dbt", "onboard", "--help"]) + + assert result.exit_code == 0 + assert "--manifest-path" in result.output + assert "--catalog-path" in result.output + assert "--run-results-path" in result.output + assert "--sources-path" in result.output + assert "--semantic-manifest-path" in result.output diff --git a/tests/data/run_results_v6.json b/tests/data/run_results_v6.json new file mode 100644 index 00000000..c1fbe2bc --- /dev/null +++ b/tests/data/run_results_v6.json @@ -0,0 +1,33 @@ +{ + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v6.json", + "dbt_version": "1.9.0", + "generated_at": "2024-12-24T10:00:00.000000Z", + "invocation_id": "test-invocation-id", + "env": {} + }, + "results": [ + { + "status": "success", + "timing": [ + { + "name": "compile", + "started_at": "2024-12-24T10:00:00.000000Z", + "completed_at": "2024-12-24T10:00:01.000000Z" + }, + { + "name": "execute", + "started_at": "2024-12-24T10:00:01.000000Z", + "completed_at": "2024-12-24T10:00:02.000000Z" + } + ], + "thread_id": "Thread-1", + "execution_time": 2.0, + "adapter_response": {}, + "message": "CREATE TABLE", + "unique_id": "model.jaffle_shop.stg_customers" + } + ], + "elapsed_time": 5.0, + "args": {} +} diff --git a/tests/data/semantic_manifest_v1.json b/tests/data/semantic_manifest_v1.json new file mode 100644 index 00000000..f8d70b4e --- /dev/null +++ b/tests/data/semantic_manifest_v1.json @@ -0,0 +1,58 @@ +{ + "semantic_models": [ + { + "name": "orders", + "description": "Order model for semantic layer", + "node_relation": { + "schema_name": "jaffle_shop", + "relation_name": "orders" + }, + "entities": [ + { + "name": "order_id", + "type": "primary" + }, + { + "name": "customer_id", + "type": "foreign" + } + ], + "dimensions": [ + { + "name": "order_date", + "type": "time" + }, + { + "name": "status", + "type": "categorical" + } + ], + "measures": [ + { + "name": "order_total", + "agg": "sum", + "expr": "amount" + }, + { + "name": "order_count", + "agg": "count", + "expr": "1" + } + ] + } + ], + "metrics": [ + { + "name": "revenue", + "description": "Total revenue", + "type": "simple", + "type_params": { + "measure": "order_total" + } + } + ], + "project_configuration": { + "time_spine_table_configurations": [] + }, + "saved_queries": [] +} diff --git a/tests/data/sources_v3.json b/tests/data/sources_v3.json new file mode 100644 index 00000000..29de453f --- /dev/null +++ b/tests/data/sources_v3.json @@ -0,0 +1,39 @@ +{ + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/sources/v3.json", + "dbt_version": "1.9.0", + "generated_at": "2024-12-24T10:00:00.000000Z", + "invocation_id": "test-invocation-id", + "env": {} + }, + "results": [ + { + "unique_id": "source.jaffle_shop.raw.customers", + "max_loaded_at": "2024-12-24T09:00:00.000000Z", + "snapshotted_at": "2024-12-24T10:00:00.000000Z", + "max_loaded_at_time_ago_in_s": 3600.0, + "status": "pass", + "criteria": { + "warn_after": { + "count": 24, + "period": "hour" + }, + "error_after": { + "count": 48, + "period": "hour" + } + }, + "adapter_response": {}, + "timing": [ + { + "name": "execute", + "started_at": "2024-12-24T10:00:00.000000Z", + "completed_at": "2024-12-24T10:00:01.000000Z" + } + ], + "thread_id": "Thread-1", + "execution_time": 1.0 + } + ], + "elapsed_time": 2.0 +} From 6d2034fa589d6e03ec1a521d43a1272cce13c0f7 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 26 Dec 2025 16:21:31 +0530 Subject: [PATCH 2/6] remove semantic manifest --- src/datapilot/clients/altimate/constants.py | 1 - src/datapilot/core/platforms/dbt/cli/cli.py | 20 ------- .../dbt/schemas/semantic_manifest.py | 12 ---- src/datapilot/core/platforms/dbt/utils.py | 18 ------ src/vendor/dbt_artifacts_parser/parser.py | 18 ------ .../parsers/semantic_manifest/__init__.py | 19 ------ .../semantic_manifest/semantic_manifest_v1.py | 44 -------------- .../parsers/version_map.py | 3 - tests/clients/altimate/test_utils.py | 2 +- .../platform/dbt/test_artifact_loaders.py | 17 ------ tests/core/platform/dbt/test_cli.py | 1 - tests/data/semantic_manifest_v1.json | 58 ------------------- 12 files changed, 1 insertion(+), 212 deletions(-) delete mode 100644 src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py delete mode 100644 src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py delete mode 100644 src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py delete mode 100644 tests/data/semantic_manifest_v1.json diff --git a/src/datapilot/clients/altimate/constants.py b/src/datapilot/clients/altimate/constants.py index 25118b83..dc69f1bf 100644 --- a/src/datapilot/clients/altimate/constants.py +++ b/src/datapilot/clients/altimate/constants.py @@ -6,5 +6,4 @@ "catalog", "run_results", "sources", - "semantic_manifest", } diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 751cd498..07ba5427 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -19,7 +19,6 @@ from datapilot.core.platforms.dbt.utils import load_catalog from datapilot.core.platforms.dbt.utils import load_manifest from datapilot.core.platforms.dbt.utils import load_run_results -from datapilot.core.platforms.dbt.utils import load_semantic_manifest from datapilot.core.platforms.dbt.utils import load_sources from datapilot.utils.formatting.utils import tabulate_data from datapilot.utils.utils import map_url_to_instance @@ -160,7 +159,6 @@ def project_health( @click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.") @click.option("--run-results-path", required=False, prompt=False, help="Path to the run_results.json file.") @click.option("--sources-path", required=False, prompt=False, help="Path to the sources.json file (source freshness results).") -@click.option("--semantic-manifest-path", required=False, prompt=False, help="Path to the semantic_manifest.json file.") def onboard( token, instance_name, @@ -172,7 +170,6 @@ def onboard( catalog_path, run_results_path, sources_path, - semantic_manifest_path, ): """Onboard a manifest file to DBT. You can specify either --dbt_integration_id or --dbt_integration_name.""" @@ -236,13 +233,6 @@ def onboard( click.echo(f"Error validating sources: {e}") return - if semantic_manifest_path: - try: - load_semantic_manifest(semantic_manifest_path) - except Exception as e: - click.echo(f"Error validating semantic_manifest: {e}") - return - # Onboard manifest (required) response = onboard_file(token, instance_name, dbt_integration_id, dbt_integration_environment, "manifest", manifest_path, backend_url) if response["ok"]: @@ -280,16 +270,6 @@ def onboard( else: click.echo(f"{response['message']}") - if semantic_manifest_path: - response = onboard_file( - token, instance_name, dbt_integration_id, dbt_integration_environment, "semantic_manifest", semantic_manifest_path, backend_url - ) - if response["ok"]: - click.echo("Semantic manifest onboarded successfully!") - artifacts_uploaded.append("semantic_manifest") - else: - click.echo(f"{response['message']}") - # Start ingestion response = start_dbt_ingestion(token, instance_name, dbt_integration_id, dbt_integration_environment, backend_url) if response["ok"]: diff --git a/src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py b/src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py deleted file mode 100644 index c92b7b7d..00000000 --- a/src/datapilot/core/platforms/dbt/schemas/semantic_manifest.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import Union - -from pydantic import ConfigDict - -from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 as BaseSemanticManifestV1 - - -class SemanticManifestV1(BaseSemanticManifestV1): - model_config = ConfigDict(extra="allow") - - -SemanticManifest = Union[SemanticManifestV1] diff --git a/src/datapilot/core/platforms/dbt/utils.py b/src/datapilot/core/platforms/dbt/utils.py index 70560b28..1c439e43 100644 --- a/src/datapilot/core/platforms/dbt/utils.py +++ b/src/datapilot/core/platforms/dbt/utils.py @@ -23,7 +23,6 @@ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestTestNode from datapilot.core.platforms.dbt.schemas.manifest import Manifest from datapilot.core.platforms.dbt.schemas.run_results import RunResults -from datapilot.core.platforms.dbt.schemas.semantic_manifest import SemanticManifest from datapilot.core.platforms.dbt.schemas.sources import Sources from datapilot.exceptions.exceptions import AltimateFileNotFoundError from datapilot.exceptions.exceptions import AltimateInvalidJSONError @@ -33,7 +32,6 @@ from datapilot.utils.utils import load_json from vendor.dbt_artifacts_parser.parser import parse_manifest from vendor.dbt_artifacts_parser.parser import parse_run_results -from vendor.dbt_artifacts_parser.parser import parse_semantic_manifest from vendor.dbt_artifacts_parser.parser import parse_sources MODEL_TYPE_PATTERNS = { @@ -132,22 +130,6 @@ def load_sources(sources_path: str) -> Sources: return sources -def load_semantic_manifest(semantic_manifest_path: str) -> SemanticManifest: - try: - semantic_manifest_dict = load_json(semantic_manifest_path) - except FileNotFoundError as e: - raise AltimateFileNotFoundError(f"Semantic manifest file not found: {semantic_manifest_path}. Error: {e}") from e - except ValueError as e: - raise AltimateInvalidJSONError(f"Invalid JSON file: {semantic_manifest_path}. Error: {e}") from e - - try: - semantic_manifest: SemanticManifest = parse_semantic_manifest(semantic_manifest_dict) - except ValueError as e: - raise AltimateInvalidManifestError(f"Invalid semantic manifest file: {semantic_manifest_path}. Error: {e}") from e - - return semantic_manifest - - # TODO: Add tests! def get_table_name_from_source(source: AltimateManifestSourceNode) -> str: db = source.database diff --git a/src/vendor/dbt_artifacts_parser/parser.py b/src/vendor/dbt_artifacts_parser/parser.py index 646c16f9..0a7511f5 100644 --- a/src/vendor/dbt_artifacts_parser/parser.py +++ b/src/vendor/dbt_artifacts_parser/parser.py @@ -35,7 +35,6 @@ from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 -from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 @@ -346,20 +345,3 @@ def parse_sources_v3(sources: dict) -> SourcesV3: if dbt_schema_version == ArtifactTypes.SOURCES_V3.value.dbt_schema_version: return SourcesV3(**sources) raise ValueError("Not a sources.json v3") - - -# -# semantic-manifest -# -def parse_semantic_manifest(semantic_manifest: dict) -> SemanticManifestV1: - """Parse semantic_manifest.json - - Args: - semantic_manifest: A dict of semantic_manifest.json - - Returns: - SemanticManifestV1 - """ - # Semantic manifest uses a flexible schema, so we accept any valid dict - # The schema version check is optional since the format may vary - return SemanticManifestV1(**semantic_manifest) diff --git a/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py b/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py deleted file mode 100644 index 137cf772..00000000 --- a/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 - -__all__ = ["SemanticManifestV1"] diff --git a/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py b/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py deleted file mode 100644 index 991514aa..00000000 --- a/src/vendor/dbt_artifacts_parser/parsers/semantic_manifest/semantic_manifest_v1.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import annotations - -from typing import Any -from typing import Optional - -from pydantic import ConfigDict - -from vendor.dbt_artifacts_parser.parsers.base import BaseParserModel - - -class SemanticManifestV1(BaseParserModel): - """ - Semantic Manifest artifact produced by dbt when parsing a project with semantic layer definitions. - - The semantic manifest contains: - - semantic_models: Starting points of data with entities, dimensions, and measures - - metrics: Functions combining measures, constraints to define quantitative indicators - - project_configuration: Project-level settings including time spine configurations - - saved_queries: Pre-built queries for common MetricFlow use cases - """ - - model_config = ConfigDict(extra="allow") - - semantic_models: Optional[list[dict[str, Any]]] = None - metrics: Optional[list[dict[str, Any]]] = None - project_configuration: Optional[dict[str, Any]] = None - saved_queries: Optional[list[dict[str, Any]]] = None diff --git a/src/vendor/dbt_artifacts_parser/parsers/version_map.py b/src/vendor/dbt_artifacts_parser/parsers/version_map.py index 7c197b31..69dbd0fe 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/version_map.py +++ b/src/vendor/dbt_artifacts_parser/parsers/version_map.py @@ -38,7 +38,6 @@ from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 -from vendor.dbt_artifacts_parser.parsers.semantic_manifest.semantic_manifest_v1 import SemanticManifestV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 @@ -79,5 +78,3 @@ class ArtifactTypes(Enum): SOURCES_V1 = ArtifactType("https://schemas.getdbt.com/dbt/sources/v1.json", SourcesV1) SOURCES_V2 = ArtifactType("https://schemas.getdbt.com/dbt/sources/v2.json", SourcesV2) SOURCES_V3 = ArtifactType("https://schemas.getdbt.com/dbt/sources/v3.json", SourcesV3) - # Semantic Manifest - SEMANTIC_MANIFEST_V1 = ArtifactType("https://schemas.getdbt.com/dbt/semantic-manifest/v1.json", SemanticManifestV1) diff --git a/tests/clients/altimate/test_utils.py b/tests/clients/altimate/test_utils.py index 30df77ff..0eceb206 100644 --- a/tests/clients/altimate/test_utils.py +++ b/tests/clients/altimate/test_utils.py @@ -5,7 +5,7 @@ class TestOnboardFile: def test_supported_artifact_types(self): """Test that all expected artifact types are supported.""" - expected_types = {"manifest", "catalog", "run_results", "sources", "semantic_manifest"} + expected_types = {"manifest", "catalog", "run_results", "sources"} assert SUPPORTED_ARTIFACT_TYPES == expected_types def test_unsupported_file_type_returns_error(self): diff --git a/tests/core/platform/dbt/test_artifact_loaders.py b/tests/core/platform/dbt/test_artifact_loaders.py index a5f7dbe0..0ed98cd2 100644 --- a/tests/core/platform/dbt/test_artifact_loaders.py +++ b/tests/core/platform/dbt/test_artifact_loaders.py @@ -1,7 +1,6 @@ import pytest from datapilot.core.platforms.dbt.utils import load_run_results -from datapilot.core.platforms.dbt.utils import load_semantic_manifest from datapilot.core.platforms.dbt.utils import load_sources from datapilot.exceptions.exceptions import AltimateFileNotFoundError @@ -35,19 +34,3 @@ def test_load_sources_v3(self): def test_load_sources_file_not_found(self): with pytest.raises(AltimateFileNotFoundError): load_sources("nonexistent_file.json") - - -class TestLoadSemanticManifest: - def test_load_semantic_manifest_v1(self): - semantic_manifest_path = "tests/data/semantic_manifest_v1.json" - semantic_manifest = load_semantic_manifest(semantic_manifest_path) - - assert semantic_manifest is not None - assert len(semantic_manifest.semantic_models) == 1 - assert semantic_manifest.semantic_models[0]["name"] == "orders" - assert len(semantic_manifest.metrics) == 1 - assert semantic_manifest.metrics[0]["name"] == "revenue" - - def test_load_semantic_manifest_file_not_found(self): - with pytest.raises(AltimateFileNotFoundError): - load_semantic_manifest("nonexistent_file.json") diff --git a/tests/core/platform/dbt/test_cli.py b/tests/core/platform/dbt/test_cli.py index 4cdb3921..e4b74aa3 100644 --- a/tests/core/platform/dbt/test_cli.py +++ b/tests/core/platform/dbt/test_cli.py @@ -126,4 +126,3 @@ def test_onboard_help_shows_all_artifact_options(): assert "--catalog-path" in result.output assert "--run-results-path" in result.output assert "--sources-path" in result.output - assert "--semantic-manifest-path" in result.output diff --git a/tests/data/semantic_manifest_v1.json b/tests/data/semantic_manifest_v1.json deleted file mode 100644 index f8d70b4e..00000000 --- a/tests/data/semantic_manifest_v1.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "semantic_models": [ - { - "name": "orders", - "description": "Order model for semantic layer", - "node_relation": { - "schema_name": "jaffle_shop", - "relation_name": "orders" - }, - "entities": [ - { - "name": "order_id", - "type": "primary" - }, - { - "name": "customer_id", - "type": "foreign" - } - ], - "dimensions": [ - { - "name": "order_date", - "type": "time" - }, - { - "name": "status", - "type": "categorical" - } - ], - "measures": [ - { - "name": "order_total", - "agg": "sum", - "expr": "amount" - }, - { - "name": "order_count", - "agg": "count", - "expr": "1" - } - ] - } - ], - "metrics": [ - { - "name": "revenue", - "description": "Total revenue", - "type": "simple", - "type_params": { - "measure": "order_total" - } - } - ], - "project_configuration": { - "time_spine_table_configurations": [] - }, - "saved_queries": [] -} From 9a3499916c1e6290dc4a51454c926612b81a9630 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Mon, 29 Dec 2025 15:53:32 +0530 Subject: [PATCH 3/6] WIP --- src/datapilot/clients/altimate/constants.py | 1 + src/datapilot/core/platforms/dbt/cli/cli.py | 12 ++++++++++++ tests/clients/altimate/test_utils.py | 2 +- tests/core/platform/dbt/test_cli.py | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/datapilot/clients/altimate/constants.py b/src/datapilot/clients/altimate/constants.py index dc69f1bf..25118b83 100644 --- a/src/datapilot/clients/altimate/constants.py +++ b/src/datapilot/clients/altimate/constants.py @@ -6,4 +6,5 @@ "catalog", "run_results", "sources", + "semantic_manifest", } diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 07ba5427..e286e798 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -159,6 +159,7 @@ def project_health( @click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.") @click.option("--run-results-path", required=False, prompt=False, help="Path to the run_results.json file.") @click.option("--sources-path", required=False, prompt=False, help="Path to the sources.json file (source freshness results).") +@click.option("--semantic-manifest-path", required=False, prompt=False, help="Path to the semantic_manifest.json file.") def onboard( token, instance_name, @@ -170,6 +171,7 @@ def onboard( catalog_path, run_results_path, sources_path, + semantic_manifest_path, ): """Onboard a manifest file to DBT. You can specify either --dbt_integration_id or --dbt_integration_name.""" @@ -270,6 +272,16 @@ def onboard( else: click.echo(f"{response['message']}") + if semantic_manifest_path: + response = onboard_file( + token, instance_name, dbt_integration_id, dbt_integration_environment, "semantic_manifest", semantic_manifest_path, backend_url + ) + if response["ok"]: + click.echo("Semantic manifest onboarded successfully!") + artifacts_uploaded.append("semantic_manifest") + else: + click.echo(f"{response['message']}") + # Start ingestion response = start_dbt_ingestion(token, instance_name, dbt_integration_id, dbt_integration_environment, backend_url) if response["ok"]: diff --git a/tests/clients/altimate/test_utils.py b/tests/clients/altimate/test_utils.py index 0eceb206..30df77ff 100644 --- a/tests/clients/altimate/test_utils.py +++ b/tests/clients/altimate/test_utils.py @@ -5,7 +5,7 @@ class TestOnboardFile: def test_supported_artifact_types(self): """Test that all expected artifact types are supported.""" - expected_types = {"manifest", "catalog", "run_results", "sources"} + expected_types = {"manifest", "catalog", "run_results", "sources", "semantic_manifest"} assert SUPPORTED_ARTIFACT_TYPES == expected_types def test_unsupported_file_type_returns_error(self): diff --git a/tests/core/platform/dbt/test_cli.py b/tests/core/platform/dbt/test_cli.py index e4b74aa3..4cdb3921 100644 --- a/tests/core/platform/dbt/test_cli.py +++ b/tests/core/platform/dbt/test_cli.py @@ -126,3 +126,4 @@ def test_onboard_help_shows_all_artifact_options(): assert "--catalog-path" in result.output assert "--run-results-path" in result.output assert "--sources-path" in result.output + assert "--semantic-manifest-path" in result.output From ba68036553674c2dd14504f5db8bb5d325219a56 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 2 Jan 2026 10:30:37 +0530 Subject: [PATCH 4/6] fix issues --- src/datapilot/clients/altimate/utils.py | 2 +- src/datapilot/core/insights/schema.py | 5 ++ .../core/platforms/dbt/schemas/catalog.py | 12 ++++ .../core/platforms/dbt/schemas/manifest.py | 55 +++++++++++++++++++ src/datapilot/schemas/nodes.py | 5 ++ .../parsers/run_results/run_results_v1.py | 8 +-- .../parsers/run_results/run_results_v2.py | 8 +-- .../parsers/run_results/run_results_v3.py | 18 +++--- .../parsers/run_results/run_results_v4.py | 18 +++--- .../parsers/run_results/run_results_v5.py | 8 +-- .../parsers/run_results/run_results_v6.py | 10 ++-- .../parsers/sources/sources_v1.py | 12 ++-- .../parsers/sources/sources_v2.py | 14 ++--- .../parsers/sources/sources_v3.py | 16 +++--- 14 files changed, 134 insertions(+), 57 deletions(-) diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py index 87e40113..ecce6069 100644 --- a/src/datapilot/clients/altimate/utils.py +++ b/src/datapilot/clients/altimate/utils.py @@ -106,7 +106,7 @@ def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integratio api_client.log("Error getting signed URL.") return { "ok": False, - "message": "Error in uploading the manifest. ", + "message": f"Error in uploading the {file_type}.", } diff --git a/src/datapilot/core/insights/schema.py b/src/datapilot/core/insights/schema.py index ea042e4c..dcfa1858 100644 --- a/src/datapilot/core/insights/schema.py +++ b/src/datapilot/core/insights/schema.py @@ -2,6 +2,7 @@ from typing import Dict from pydantic import BaseModel +from pydantic import ConfigDict class Severity(Enum): @@ -11,6 +12,8 @@ class Severity(Enum): class InsightResult(BaseModel): + model_config = ConfigDict(extra="allow") + name: str type: str message: str @@ -20,5 +23,7 @@ class InsightResult(BaseModel): class InsightResponse(BaseModel): + model_config = ConfigDict(extra="allow") + insight: InsightResult severity: Severity = Severity.ERROR diff --git a/src/datapilot/core/platforms/dbt/schemas/catalog.py b/src/datapilot/core/platforms/dbt/schemas/catalog.py index 699635ce..756b39d2 100644 --- a/src/datapilot/core/platforms/dbt/schemas/catalog.py +++ b/src/datapilot/core/platforms/dbt/schemas/catalog.py @@ -13,6 +13,8 @@ class AltimateCatalogMetadata(BaseModel): + model_config = ConfigDict(extra="allow") + dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/catalog/v1.json" dbt_version: Optional[str] = "0.19.0" generated_at: Optional[datetime] = "2021-02-10T04:42:33.680487Z" @@ -21,6 +23,8 @@ class AltimateCatalogMetadata(BaseModel): class AltimateCatalogTableMetadata(BaseModel): + model_config = ConfigDict(extra="allow") + type: str database: Optional[Optional[str]] = None schema_name: str @@ -30,6 +34,8 @@ class AltimateCatalogTableMetadata(BaseModel): class AltimateCatalogColumnMetadata(BaseModel): + model_config = ConfigDict(extra="allow") + type: str comment: Optional[Optional[str]] = None index: int @@ -37,6 +43,8 @@ class AltimateCatalogColumnMetadata(BaseModel): class AltimateCatalogStatsItem(BaseModel): + model_config = ConfigDict(extra="allow") + id: str label: str value: Optional[Optional[Union[bool, str, float]]] = None @@ -45,6 +53,8 @@ class AltimateCatalogStatsItem(BaseModel): class AltimateCatalogTable(BaseModel): + model_config = ConfigDict(extra="allow") + metadata: AltimateCatalogTableMetadata columns: Dict[str, AltimateCatalogColumnMetadata] stats: Dict[str, AltimateCatalogStatsItem] @@ -52,6 +62,8 @@ class AltimateCatalogTable(BaseModel): class AltimateCatalogCatalogV1(BaseModel): + model_config = ConfigDict(extra="allow") + metadata: AltimateCatalogMetadata nodes: Dict[str, AltimateCatalogTable] sources: Dict[str, AltimateCatalogTable] diff --git a/src/datapilot/core/platforms/dbt/schemas/manifest.py b/src/datapilot/core/platforms/dbt/schemas/manifest.py index 3a4b33fe..52b100ab 100644 --- a/src/datapilot/core/platforms/dbt/schemas/manifest.py +++ b/src/datapilot/core/platforms/dbt/schemas/manifest.py @@ -6,6 +6,7 @@ from typing import Union from pydantic import BaseModel +from pydantic import ConfigDict from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v1 import ManifestV1 from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v2 import ManifestV2 @@ -23,6 +24,8 @@ class DBTVersion(BaseModel): + model_config = ConfigDict(extra="allow") + MAJOR: int MINOR: int PATCH: Optional[int] = None @@ -45,16 +48,22 @@ class DBTVersion(BaseModel): class AltimateDocs(BaseModel): + model_config = ConfigDict(extra="allow") + show: Optional[bool] = True node_color: Optional[Optional[str]] = None class AltimateDependsOn(BaseModel): + model_config = ConfigDict(extra="allow") + nodes: Optional[List[str]] = None macros: Optional[List[str]] = None class AltimateManifestColumnInfo(BaseModel): + model_config = ConfigDict(extra="allow") + name: str description: Optional[str] = "" meta: Optional[Dict[str, Any]] = {} @@ -64,6 +73,8 @@ class AltimateManifestColumnInfo(BaseModel): class AltimateFileHash(BaseModel): + model_config = ConfigDict(extra="allow") + name: Optional[str] = None checksum: Optional[str] = None @@ -94,12 +105,16 @@ class AltimateAccess(Enum): class AltimateDBTContract(BaseModel): + model_config = ConfigDict(extra="allow") + enforced: Optional[bool] = False alias_types: Optional[bool] = True checksum: Optional[Optional[str]] = None class AltimateHook(BaseModel): + model_config = ConfigDict(extra="allow") + sql: str transaction: Optional[bool] = True index: Optional[Optional[int]] = None @@ -107,6 +122,8 @@ class AltimateHook(BaseModel): # TODO: Need to add the rest of the fields class AltimateNodeConfig(BaseModel): + model_config = ConfigDict(extra="allow") + _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True alias: Optional[Optional[str]] = None @@ -128,6 +145,8 @@ class AltimateNodeConfig(BaseModel): class AltimateManifestNode(BaseModel): + model_config = ConfigDict(extra="allow") + database: Optional[str] = None resource_type: AltimateResourceType schema_name: str @@ -158,6 +177,8 @@ class AltimateManifestNode(BaseModel): class AltimateQuoting(BaseModel): + model_config = ConfigDict(extra="allow") + database: Optional[Optional[bool]] = None schema_: Optional[Optional[bool]] = None identifier: Optional[Optional[bool]] = None @@ -165,12 +186,16 @@ class AltimateQuoting(BaseModel): class AltimateFreshnessThreshold(BaseModel): + model_config = ConfigDict(extra="allow") + warn_after: Optional[Dict] = None error_after: Optional[Dict] = None filter: Optional[str] = None class AltimateExternalPartition(BaseModel): + model_config = ConfigDict(extra="allow") + name: Optional[str] = "" description: Optional[str] = "" data_type: Optional[str] = "" @@ -178,6 +203,8 @@ class AltimateExternalPartition(BaseModel): class AltimateExternalTable(BaseModel): + model_config = ConfigDict(extra="allow") + location: Optional[Optional[str]] = None file_format: Optional[Optional[str]] = None row_format: Optional[Optional[str]] = None @@ -186,10 +213,14 @@ class AltimateExternalTable(BaseModel): class AltimateSourceConfig(BaseModel): + model_config = ConfigDict(extra="allow") + enabled: Optional[bool] = True class AltimateDeferRelation(BaseModel): + model_config = ConfigDict(extra="allow") + database: Optional[str] = None schema_name: str alias: str @@ -197,6 +228,8 @@ class AltimateDeferRelation(BaseModel): class AltimateSeedConfig(BaseModel): + model_config = ConfigDict(extra="allow") + _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True alias: Optional[Optional[str]] = None @@ -225,6 +258,8 @@ class AltimateSeedConfig(BaseModel): class AltimateSeedNode(BaseModel): + model_config = ConfigDict(extra="allow") + database: Optional[str] = None schema_name: str name: str @@ -257,6 +292,8 @@ class AltimateSeedNode(BaseModel): class AltimateManifestSourceNode(BaseModel): + model_config = ConfigDict(extra="allow") + database: Optional[str] = None resource_type: AltimateResourceType schema_name: str @@ -295,6 +332,8 @@ class AltimateExposureType(Enum): class AltimateOwner(BaseModel): + model_config = ConfigDict(extra="allow") + _extra: Optional[Dict[str, Any]] = None email: Optional[Optional[str]] = None name: Optional[Optional[str]] = None @@ -307,17 +346,23 @@ class AltimateMaturityEnum(Enum): class AltimateRefArgs(BaseModel): + model_config = ConfigDict(extra="allow") + name: str package: Optional[Optional[str]] = None version: Optional[Optional[Union[str, float]]] = None class AltimateExposureConfig(BaseModel): + model_config = ConfigDict(extra="allow") + _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True class AltimateManifestExposureNode(BaseModel): + model_config = ConfigDict(extra="allow") + name: str resource_type: AltimateResourceType package_name: str @@ -343,12 +388,16 @@ class AltimateManifestExposureNode(BaseModel): class AltimateTestMetadata(BaseModel): + model_config = ConfigDict(extra="allow") + name: str kwargs: Optional[Dict[str, Any]] = None namespace: Optional[Optional[str]] = None class AltimateTestConfig(BaseModel): + model_config = ConfigDict(extra="allow") + _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True alias: Optional[Optional[str]] = None @@ -369,6 +418,8 @@ class AltimateTestConfig(BaseModel): class AltimateManifestTestNode(BaseModel): + model_config = ConfigDict(extra="allow") + test_metadata: Optional[AltimateTestMetadata] = None test_type: Optional[str] = None name: str @@ -399,6 +450,8 @@ class AltimateManifestTestNode(BaseModel): class AltimateMacroArgument(BaseModel): + model_config = ConfigDict(extra="allow") + name: str type: Optional[Optional[str]] = None description: Optional[Optional[str]] = "" @@ -408,6 +461,8 @@ class AltimateMacroArgument(BaseModel): class AltimateManifestMacroNode(BaseModel): + model_config = ConfigDict(extra="allow") + name: str resource_type: AltimateResourceType package_name: str diff --git a/src/datapilot/schemas/nodes.py b/src/datapilot/schemas/nodes.py index 83d119b7..21ae5917 100644 --- a/src/datapilot/schemas/nodes.py +++ b/src/datapilot/schemas/nodes.py @@ -1,7 +1,10 @@ from pydantic import BaseModel +from pydantic import ConfigDict class ModelNode(BaseModel): + model_config = ConfigDict(extra="allow") + unique_id: str name: str resource_type: str @@ -11,6 +14,8 @@ class ModelNode(BaseModel): class SourceNode(BaseModel): + model_config = ConfigDict(extra="allow") + unique_id: str name: str resource_type: str diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v1.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v1.py index e5d33a85..a599c560 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v1.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v1.py @@ -16,7 +16,7 @@ class BaseArtifactMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: str dbt_version: Optional[str] = "0.19.0" @@ -47,7 +47,7 @@ class Status2(Enum): class TimingInfo(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[datetime] = None @@ -56,7 +56,7 @@ class TimingInfo(BaseParserModel): class RunResultOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) status: Union[Status, Status1, Status2] timing: list[TimingInfo] @@ -69,7 +69,7 @@ class RunResultOutput(BaseParserModel): class RunResultsV1(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: BaseArtifactMetadata results: list[RunResultOutput] diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v2.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v2.py index 8b248d7c..a16065ae 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v2.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v2.py @@ -16,7 +16,7 @@ class BaseArtifactMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: str dbt_version: Optional[str] = "0.20.0rc1" @@ -47,7 +47,7 @@ class Status2(Enum): class TimingInfo(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[datetime] = None @@ -56,7 +56,7 @@ class TimingInfo(BaseParserModel): class RunResultOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) status: Union[Status, Status1, Status2] timing: list[TimingInfo] @@ -70,7 +70,7 @@ class RunResultOutput(BaseParserModel): class RunResultsV2(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: BaseArtifactMetadata results: list[RunResultOutput] diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v3.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v3.py index 06effdc6..840fddd2 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v3.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v3.py @@ -16,7 +16,7 @@ class BaseArtifactMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: str dbt_version: Optional[str] = "0.21.0rc1" @@ -48,7 +48,7 @@ class Status2(Enum): class TimingInfo(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[datetime] = None @@ -57,7 +57,7 @@ class TimingInfo(BaseParserModel): class FreshnessMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/sources/v2.json" dbt_version: Optional[str] = "0.21.0rc1" @@ -72,7 +72,7 @@ class Status3(Enum): class SourceFreshnessRuntimeError(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str error: Optional[Union[str, int]] = None @@ -94,7 +94,7 @@ class Period(Enum): class Time(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) count: int period: Period @@ -102,7 +102,7 @@ class Time(BaseParserModel): class RunResultOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) status: Union[Status, Status1, Status2] timing: list[TimingInfo] @@ -116,7 +116,7 @@ class RunResultOutput(BaseParserModel): class FreshnessThreshold(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) warn_after: Optional[Time] = None error_after: Optional[Time] = None @@ -125,7 +125,7 @@ class FreshnessThreshold(BaseParserModel): class RunResultsV3(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: BaseArtifactMetadata results: list[RunResultOutput] @@ -135,7 +135,7 @@ class RunResultsV3(BaseParserModel): class SourceFreshnessOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str max_loaded_at: datetime diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v4.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v4.py index 44bd186d..75e1086f 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v4.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v4.py @@ -16,7 +16,7 @@ class BaseArtifactMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: str dbt_version: Optional[str] = "1.0.0b2" @@ -48,7 +48,7 @@ class Status2(Enum): class TimingInfo(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[datetime] = None @@ -57,7 +57,7 @@ class TimingInfo(BaseParserModel): class FreshnessMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/sources/v3.json" dbt_version: Optional[str] = "1.0.0b2" @@ -72,7 +72,7 @@ class Status3(Enum): class SourceFreshnessRuntimeError(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str error: Optional[Union[str, int]] = None @@ -94,7 +94,7 @@ class Period(Enum): class Time(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) count: Optional[int] = None period: Optional[Period] = None @@ -102,7 +102,7 @@ class Time(BaseParserModel): class RunResultOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) status: Union[Status, Status1, Status2] timing: list[TimingInfo] @@ -116,7 +116,7 @@ class RunResultOutput(BaseParserModel): class FreshnessThreshold(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) warn_after: Optional[Time] = {"count": None, "period": None} error_after: Optional[Time] = {"count": None, "period": None} @@ -125,7 +125,7 @@ class FreshnessThreshold(BaseParserModel): class RunResultsV4(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: BaseArtifactMetadata results: list[RunResultOutput] @@ -135,7 +135,7 @@ class RunResultsV4(BaseParserModel): class SourceFreshnessOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str max_loaded_at: datetime diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v5.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v5.py index d5358179..fb505cd6 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v5.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v5.py @@ -15,7 +15,7 @@ class BaseArtifactMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: str dbt_version: Optional[str] = "1.7.0b1" @@ -26,7 +26,7 @@ class BaseArtifactMetadata(BaseParserModel): class TimingInfo(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[str] = None @@ -56,7 +56,7 @@ class Status2(Enum): class RunResultOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) status: Union[Status, Status1, Status2] timing: list[TimingInfo] @@ -73,7 +73,7 @@ class RunResultOutput(BaseParserModel): class RunResultsV5(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: BaseArtifactMetadata results: list[RunResultOutput] diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py index c9b8808f..bb46b86a 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py @@ -16,7 +16,7 @@ class Metadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: str dbt_version: Optional[str] = "1.9.0b2" @@ -49,7 +49,7 @@ class Status2(Enum): class TimingItem(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[str] = None @@ -58,7 +58,7 @@ class TimingItem(BaseParserModel): class BatchResults(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) successful: Optional[list[list]] = None failed: Optional[list[list]] = None @@ -66,7 +66,7 @@ class BatchResults(BaseParserModel): class Result(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) status: Union[Status, Status1, Status2] timing: list[TimingItem] @@ -84,7 +84,7 @@ class Result(BaseParserModel): class RunResultsV6(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: Metadata = Field(..., title="BaseArtifactMetadata") results: list[Result] diff --git a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v1.py b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v1.py index bbba11dc..6f9a036e 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v1.py +++ b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v1.py @@ -16,7 +16,7 @@ class FreshnessMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/sources/v1.json" dbt_version: Optional[str] = "0.19.0" @@ -31,7 +31,7 @@ class Status(Enum): class SourceFreshnessRuntimeError(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str error: Optional[Union[str, int]] = None @@ -53,7 +53,7 @@ class Period(Enum): class Time(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) count: int period: Period @@ -61,7 +61,7 @@ class Time(BaseParserModel): class FreshnessThreshold(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) warn_after: Optional[Time] = None error_after: Optional[Time] = None @@ -70,7 +70,7 @@ class FreshnessThreshold(BaseParserModel): class SourceFreshnessOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str max_loaded_at: datetime @@ -83,7 +83,7 @@ class SourceFreshnessOutput(BaseParserModel): class SourcesV1(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: FreshnessMetadata results: list[Union[SourceFreshnessRuntimeError, SourceFreshnessOutput]] diff --git a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v2.py b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v2.py index 054ef935..a83c3c46 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v2.py +++ b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v2.py @@ -16,7 +16,7 @@ class FreshnessMetadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/sources/v2.json" dbt_version: Optional[str] = "0.21.0rc1" @@ -31,7 +31,7 @@ class Status(Enum): class SourceFreshnessRuntimeError(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str error: Optional[Union[str, int]] = None @@ -53,7 +53,7 @@ class Period(Enum): class Time(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) count: int period: Period @@ -61,7 +61,7 @@ class Time(BaseParserModel): class TimingInfo(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[datetime] = None @@ -70,7 +70,7 @@ class TimingInfo(BaseParserModel): class FreshnessThreshold(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) warn_after: Optional[Time] = None error_after: Optional[Time] = None @@ -79,7 +79,7 @@ class FreshnessThreshold(BaseParserModel): class SourceFreshnessOutput(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str max_loaded_at: datetime @@ -95,7 +95,7 @@ class SourceFreshnessOutput(BaseParserModel): class SourcesV2(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: FreshnessMetadata results: list[Union[SourceFreshnessRuntimeError, SourceFreshnessOutput]] diff --git a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py index 6aaa90ff..2a5c4fce 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py +++ b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py @@ -16,7 +16,7 @@ class Metadata(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) dbt_schema_version: Optional[str] = None dbt_version: Optional[str] = "1.9.0b2" @@ -31,7 +31,7 @@ class Status(Enum): class Results(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str error: Optional[Union[str, int]] = None @@ -53,7 +53,7 @@ class Period(Enum): class WarnAfter(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) count: Optional[int] = None period: Optional[Period] = None @@ -61,7 +61,7 @@ class WarnAfter(BaseParserModel): class ErrorAfter(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) count: Optional[int] = None period: Optional[Period] = None @@ -69,7 +69,7 @@ class ErrorAfter(BaseParserModel): class Criteria(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) warn_after: Optional[WarnAfter] = None error_after: Optional[ErrorAfter] = None @@ -78,7 +78,7 @@ class Criteria(BaseParserModel): class TimingItem(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) name: str started_at: Optional[str] = None @@ -87,7 +87,7 @@ class TimingItem(BaseParserModel): class Results1(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) unique_id: str max_loaded_at: str @@ -103,7 +103,7 @@ class Results1(BaseParserModel): class SourcesV3(BaseParserModel): model_config = ConfigDict( - extra="forbid", + extra="allow", ) metadata: Metadata = Field(..., title="FreshnessMetadata") results: list[Union[Results, Results1]] From 752cf7cd4ab99e0bcd548a00aa1be6e26cf345e8 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Tue, 6 Jan 2026 12:35:17 +0530 Subject: [PATCH 5/6] remove extra allow --- src/datapilot/core/insights/schema.py | 5 -- .../core/platforms/dbt/schemas/catalog.py | 12 ---- .../core/platforms/dbt/schemas/manifest.py | 55 ------------------- .../core/platforms/dbt/schemas/run_results.py | 39 ++----------- .../core/platforms/dbt/schemas/sources.py | 21 +------ src/datapilot/schemas/nodes.py | 5 -- 6 files changed, 9 insertions(+), 128 deletions(-) diff --git a/src/datapilot/core/insights/schema.py b/src/datapilot/core/insights/schema.py index dcfa1858..ea042e4c 100644 --- a/src/datapilot/core/insights/schema.py +++ b/src/datapilot/core/insights/schema.py @@ -2,7 +2,6 @@ from typing import Dict from pydantic import BaseModel -from pydantic import ConfigDict class Severity(Enum): @@ -12,8 +11,6 @@ class Severity(Enum): class InsightResult(BaseModel): - model_config = ConfigDict(extra="allow") - name: str type: str message: str @@ -23,7 +20,5 @@ class InsightResult(BaseModel): class InsightResponse(BaseModel): - model_config = ConfigDict(extra="allow") - insight: InsightResult severity: Severity = Severity.ERROR diff --git a/src/datapilot/core/platforms/dbt/schemas/catalog.py b/src/datapilot/core/platforms/dbt/schemas/catalog.py index 756b39d2..699635ce 100644 --- a/src/datapilot/core/platforms/dbt/schemas/catalog.py +++ b/src/datapilot/core/platforms/dbt/schemas/catalog.py @@ -13,8 +13,6 @@ class AltimateCatalogMetadata(BaseModel): - model_config = ConfigDict(extra="allow") - dbt_schema_version: Optional[str] = "https://schemas.getdbt.com/dbt/catalog/v1.json" dbt_version: Optional[str] = "0.19.0" generated_at: Optional[datetime] = "2021-02-10T04:42:33.680487Z" @@ -23,8 +21,6 @@ class AltimateCatalogMetadata(BaseModel): class AltimateCatalogTableMetadata(BaseModel): - model_config = ConfigDict(extra="allow") - type: str database: Optional[Optional[str]] = None schema_name: str @@ -34,8 +30,6 @@ class AltimateCatalogTableMetadata(BaseModel): class AltimateCatalogColumnMetadata(BaseModel): - model_config = ConfigDict(extra="allow") - type: str comment: Optional[Optional[str]] = None index: int @@ -43,8 +37,6 @@ class AltimateCatalogColumnMetadata(BaseModel): class AltimateCatalogStatsItem(BaseModel): - model_config = ConfigDict(extra="allow") - id: str label: str value: Optional[Optional[Union[bool, str, float]]] = None @@ -53,8 +45,6 @@ class AltimateCatalogStatsItem(BaseModel): class AltimateCatalogTable(BaseModel): - model_config = ConfigDict(extra="allow") - metadata: AltimateCatalogTableMetadata columns: Dict[str, AltimateCatalogColumnMetadata] stats: Dict[str, AltimateCatalogStatsItem] @@ -62,8 +52,6 @@ class AltimateCatalogTable(BaseModel): class AltimateCatalogCatalogV1(BaseModel): - model_config = ConfigDict(extra="allow") - metadata: AltimateCatalogMetadata nodes: Dict[str, AltimateCatalogTable] sources: Dict[str, AltimateCatalogTable] diff --git a/src/datapilot/core/platforms/dbt/schemas/manifest.py b/src/datapilot/core/platforms/dbt/schemas/manifest.py index 52b100ab..3a4b33fe 100644 --- a/src/datapilot/core/platforms/dbt/schemas/manifest.py +++ b/src/datapilot/core/platforms/dbt/schemas/manifest.py @@ -6,7 +6,6 @@ from typing import Union from pydantic import BaseModel -from pydantic import ConfigDict from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v1 import ManifestV1 from vendor.dbt_artifacts_parser.parsers.manifest.manifest_v2 import ManifestV2 @@ -24,8 +23,6 @@ class DBTVersion(BaseModel): - model_config = ConfigDict(extra="allow") - MAJOR: int MINOR: int PATCH: Optional[int] = None @@ -48,22 +45,16 @@ class DBTVersion(BaseModel): class AltimateDocs(BaseModel): - model_config = ConfigDict(extra="allow") - show: Optional[bool] = True node_color: Optional[Optional[str]] = None class AltimateDependsOn(BaseModel): - model_config = ConfigDict(extra="allow") - nodes: Optional[List[str]] = None macros: Optional[List[str]] = None class AltimateManifestColumnInfo(BaseModel): - model_config = ConfigDict(extra="allow") - name: str description: Optional[str] = "" meta: Optional[Dict[str, Any]] = {} @@ -73,8 +64,6 @@ class AltimateManifestColumnInfo(BaseModel): class AltimateFileHash(BaseModel): - model_config = ConfigDict(extra="allow") - name: Optional[str] = None checksum: Optional[str] = None @@ -105,16 +94,12 @@ class AltimateAccess(Enum): class AltimateDBTContract(BaseModel): - model_config = ConfigDict(extra="allow") - enforced: Optional[bool] = False alias_types: Optional[bool] = True checksum: Optional[Optional[str]] = None class AltimateHook(BaseModel): - model_config = ConfigDict(extra="allow") - sql: str transaction: Optional[bool] = True index: Optional[Optional[int]] = None @@ -122,8 +107,6 @@ class AltimateHook(BaseModel): # TODO: Need to add the rest of the fields class AltimateNodeConfig(BaseModel): - model_config = ConfigDict(extra="allow") - _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True alias: Optional[Optional[str]] = None @@ -145,8 +128,6 @@ class AltimateNodeConfig(BaseModel): class AltimateManifestNode(BaseModel): - model_config = ConfigDict(extra="allow") - database: Optional[str] = None resource_type: AltimateResourceType schema_name: str @@ -177,8 +158,6 @@ class AltimateManifestNode(BaseModel): class AltimateQuoting(BaseModel): - model_config = ConfigDict(extra="allow") - database: Optional[Optional[bool]] = None schema_: Optional[Optional[bool]] = None identifier: Optional[Optional[bool]] = None @@ -186,16 +165,12 @@ class AltimateQuoting(BaseModel): class AltimateFreshnessThreshold(BaseModel): - model_config = ConfigDict(extra="allow") - warn_after: Optional[Dict] = None error_after: Optional[Dict] = None filter: Optional[str] = None class AltimateExternalPartition(BaseModel): - model_config = ConfigDict(extra="allow") - name: Optional[str] = "" description: Optional[str] = "" data_type: Optional[str] = "" @@ -203,8 +178,6 @@ class AltimateExternalPartition(BaseModel): class AltimateExternalTable(BaseModel): - model_config = ConfigDict(extra="allow") - location: Optional[Optional[str]] = None file_format: Optional[Optional[str]] = None row_format: Optional[Optional[str]] = None @@ -213,14 +186,10 @@ class AltimateExternalTable(BaseModel): class AltimateSourceConfig(BaseModel): - model_config = ConfigDict(extra="allow") - enabled: Optional[bool] = True class AltimateDeferRelation(BaseModel): - model_config = ConfigDict(extra="allow") - database: Optional[str] = None schema_name: str alias: str @@ -228,8 +197,6 @@ class AltimateDeferRelation(BaseModel): class AltimateSeedConfig(BaseModel): - model_config = ConfigDict(extra="allow") - _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True alias: Optional[Optional[str]] = None @@ -258,8 +225,6 @@ class AltimateSeedConfig(BaseModel): class AltimateSeedNode(BaseModel): - model_config = ConfigDict(extra="allow") - database: Optional[str] = None schema_name: str name: str @@ -292,8 +257,6 @@ class AltimateSeedNode(BaseModel): class AltimateManifestSourceNode(BaseModel): - model_config = ConfigDict(extra="allow") - database: Optional[str] = None resource_type: AltimateResourceType schema_name: str @@ -332,8 +295,6 @@ class AltimateExposureType(Enum): class AltimateOwner(BaseModel): - model_config = ConfigDict(extra="allow") - _extra: Optional[Dict[str, Any]] = None email: Optional[Optional[str]] = None name: Optional[Optional[str]] = None @@ -346,23 +307,17 @@ class AltimateMaturityEnum(Enum): class AltimateRefArgs(BaseModel): - model_config = ConfigDict(extra="allow") - name: str package: Optional[Optional[str]] = None version: Optional[Optional[Union[str, float]]] = None class AltimateExposureConfig(BaseModel): - model_config = ConfigDict(extra="allow") - _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True class AltimateManifestExposureNode(BaseModel): - model_config = ConfigDict(extra="allow") - name: str resource_type: AltimateResourceType package_name: str @@ -388,16 +343,12 @@ class AltimateManifestExposureNode(BaseModel): class AltimateTestMetadata(BaseModel): - model_config = ConfigDict(extra="allow") - name: str kwargs: Optional[Dict[str, Any]] = None namespace: Optional[Optional[str]] = None class AltimateTestConfig(BaseModel): - model_config = ConfigDict(extra="allow") - _extra: Optional[Dict[str, Any]] = None enabled: Optional[bool] = True alias: Optional[Optional[str]] = None @@ -418,8 +369,6 @@ class AltimateTestConfig(BaseModel): class AltimateManifestTestNode(BaseModel): - model_config = ConfigDict(extra="allow") - test_metadata: Optional[AltimateTestMetadata] = None test_type: Optional[str] = None name: str @@ -450,8 +399,6 @@ class AltimateManifestTestNode(BaseModel): class AltimateMacroArgument(BaseModel): - model_config = ConfigDict(extra="allow") - name: str type: Optional[Optional[str]] = None description: Optional[Optional[str]] = "" @@ -461,8 +408,6 @@ class AltimateMacroArgument(BaseModel): class AltimateManifestMacroNode(BaseModel): - model_config = ConfigDict(extra="allow") - name: str resource_type: AltimateResourceType package_name: str diff --git a/src/datapilot/core/platforms/dbt/schemas/run_results.py b/src/datapilot/core/platforms/dbt/schemas/run_results.py index b6bc8523..5b412b7f 100644 --- a/src/datapilot/core/platforms/dbt/schemas/run_results.py +++ b/src/datapilot/core/platforms/dbt/schemas/run_results.py @@ -1,38 +1,11 @@ from typing import Union -from pydantic import ConfigDict - -from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v1 import RunResultsV1 as BaseRunResultsV1 -from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v2 import RunResultsV2 as BaseRunResultsV2 -from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v3 import RunResultsV3 as BaseRunResultsV3 -from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 as BaseRunResultsV4 -from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 as BaseRunResultsV5 -from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 as BaseRunResultsV6 - - -class RunResultsV1(BaseRunResultsV1): - model_config = ConfigDict(extra="allow") - - -class RunResultsV2(BaseRunResultsV2): - model_config = ConfigDict(extra="allow") - - -class RunResultsV3(BaseRunResultsV3): - model_config = ConfigDict(extra="allow") - - -class RunResultsV4(BaseRunResultsV4): - model_config = ConfigDict(extra="allow") - - -class RunResultsV5(BaseRunResultsV5): - model_config = ConfigDict(extra="allow") - - -class RunResultsV6(BaseRunResultsV6): - model_config = ConfigDict(extra="allow") - +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v1 import RunResultsV1 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v2 import RunResultsV2 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v3 import RunResultsV3 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v4 import RunResultsV4 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5 +from vendor.dbt_artifacts_parser.parsers.run_results.run_results_v6 import RunResultsV6 RunResults = Union[ RunResultsV6, diff --git a/src/datapilot/core/platforms/dbt/schemas/sources.py b/src/datapilot/core/platforms/dbt/schemas/sources.py index 8d66a3a7..79eb161a 100644 --- a/src/datapilot/core/platforms/dbt/schemas/sources.py +++ b/src/datapilot/core/platforms/dbt/schemas/sources.py @@ -1,23 +1,8 @@ from typing import Union -from pydantic import ConfigDict - -from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 as BaseSourcesV1 -from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 as BaseSourcesV2 -from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 as BaseSourcesV3 - - -class SourcesV1(BaseSourcesV1): - model_config = ConfigDict(extra="allow") - - -class SourcesV2(BaseSourcesV2): - model_config = ConfigDict(extra="allow") - - -class SourcesV3(BaseSourcesV3): - model_config = ConfigDict(extra="allow") - +from vendor.dbt_artifacts_parser.parsers.sources.sources_v1 import SourcesV1 +from vendor.dbt_artifacts_parser.parsers.sources.sources_v2 import SourcesV2 +from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import SourcesV3 Sources = Union[ SourcesV3, diff --git a/src/datapilot/schemas/nodes.py b/src/datapilot/schemas/nodes.py index 21ae5917..83d119b7 100644 --- a/src/datapilot/schemas/nodes.py +++ b/src/datapilot/schemas/nodes.py @@ -1,10 +1,7 @@ from pydantic import BaseModel -from pydantic import ConfigDict class ModelNode(BaseModel): - model_config = ConfigDict(extra="allow") - unique_id: str name: str resource_type: str @@ -14,8 +11,6 @@ class ModelNode(BaseModel): class SourceNode(BaseModel): - model_config = ConfigDict(extra="allow") - unique_id: str name: str resource_type: str From 32d52c7c67242ec931a6f027b25da526d72409e4 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Wed, 7 Jan 2026 13:57:19 +0530 Subject: [PATCH 6/6] changes --- .../parsers/run_results/run_results_v6.py | 16 ++++++++++++++-- .../parsers/sources/sources_v3.py | 8 +++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py index bb46b86a..c48c71b7 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py +++ b/src/vendor/dbt_artifacts_parser/parsers/run_results/run_results_v6.py @@ -64,6 +64,12 @@ class BatchResults(BaseParserModel): failed: Optional[list[list]] = None +class AdapterResponse(BaseParserModel): + model_config = ConfigDict( + extra="allow", + ) + + class Result(BaseParserModel): model_config = ConfigDict( extra="allow", @@ -72,7 +78,7 @@ class Result(BaseParserModel): timing: list[TimingItem] thread_id: str execution_time: float - adapter_response: dict[str, Any] + adapter_response: AdapterResponse message: Optional[str] = None failures: Optional[int] = None unique_id: str @@ -82,6 +88,12 @@ class Result(BaseParserModel): batch_results: Optional[BatchResults] = None +class Args(BaseParserModel): + model_config = ConfigDict( + extra="allow", + ) + + class RunResultsV6(BaseParserModel): model_config = ConfigDict( extra="allow", @@ -89,4 +101,4 @@ class RunResultsV6(BaseParserModel): metadata: Metadata = Field(..., title="BaseArtifactMetadata") results: list[Result] elapsed_time: float - args: Optional[dict[str, Any]] = None + args: Optional[Args] = None diff --git a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py index 2a5c4fce..b4465c0c 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py +++ b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py @@ -85,6 +85,12 @@ class TimingItem(BaseParserModel): completed_at: Optional[str] = None +class AdapterResponse(BaseParserModel): + model_config = ConfigDict( + extra="allow", + ) + + class Results1(BaseParserModel): model_config = ConfigDict( extra="allow", @@ -95,7 +101,7 @@ class Results1(BaseParserModel): max_loaded_at_time_ago_in_s: float status: Status1 criteria: Criteria = Field(..., title="FreshnessThreshold") - adapter_response: dict[str, Any] + adapter_response: AdapterResponse timing: list[TimingItem] thread_id: str execution_time: float