[CDF-27564] 🙉Test build+deploy for v2 commands. (#2800)

# Description

Note this is stacked on two branches #2796 , and the branch #2792 ->
#2797 .

Adds a unit test that builds and deploy the complete orgs.

Found a few bugs through the testing:

1. CDF Project not set in `lineage.yaml` when running `cdf build`.
2. Unwrapping build variables were not done correctly for iteration
cases.
3. Lineage serialized as relative paths, but requires absolute paths
causing failure when loading.
4. Graphql was found in an incorrect way. 


## Bump

- [ ] Patch
- [x] Skip

Author: doctrino

cognite_toolkit/_cdf_tk/commands/build_v2/_module_parser.py

@@ -44,11 +44,7 @@ def parse(cls, build: BuildSourceFiles) -> BuildSource:
         module_sources: list[ModuleSource] = []
         for module in selected_modules:
             source = source_by_module_id[module]
-            module_specific_variables: dict[int | None, list[BuildVariable]] = defaultdict(list)
-            for path in [module, *module.parents]:
-                if path_variables := build_variables.get(path):
-                    for iteration, variables in path_variables.items():
-                        module_specific_variables[iteration].extend(variables)
+            module_specific_variables = cls._as_module_variables(build_variables, module)
 
             if module_specific_variables:
                 for iteration, variables in module_specific_variables.items():
@@ -70,6 +66,28 @@ def parse(cls, build: BuildSourceFiles) -> BuildSource:
             orphan_yaml_files=orphan_yaml_files,
         )
 
+    @classmethod
+    def _as_module_variables(
+        cls, build_variables: dict[Path, dict[int | None, list[BuildVariable]]], module: Path
+    ) -> dict[int | None, list[BuildVariable]]:
+        module_specific_variables: dict[int | None, list[BuildVariable]] = defaultdict(list)
+        for path in [module, *module.parents]:
+            if path_variables := build_variables.get(path):
+                for iteration, variables in path_variables.items():
+                    module_specific_variables[iteration].extend(variables)
+        if len(module_specific_variables) <= 1 or None not in module_specific_variables:
+            return module_specific_variables
+
+        # There is multiple iterations with shared variables
+        shared_variables = module_specific_variables[None]
+        variables_by_iteration: dict[int | None, list[BuildVariable]] = defaultdict(list)
+        for iteration, variables in module_specific_variables.items():
+            if iteration is None:
+                continue
+            variables_by_iteration[iteration].extend(variables)
+            variables_by_iteration[iteration].extend(shared_variables)
+        return variables_by_iteration
+
     @classmethod
     def _find_modules(
         cls, yaml_files: list[RelativeFilePath], organization_dir: Path

cognite_toolkit/_cdf_tk/commands/build_v2/build_v2.py

@@ -109,7 +109,7 @@ def build(self, parameters: BuildParameters, client: ToolkitClient | None = None
 
         self._display_build_folder(build_folder, parameters.config_yaml_name or "", console, parameters.verbose)
 
-        self._write_results(build_folder)
+        self._write_results(build_folder, client.config.project if client else None)
 
         # Todo: Some mixpanel tracking.
         return build_folder
@@ -889,7 +889,7 @@ def _display_build_folder(
 
         return None
 
-    def _write_results(self, build: BuildFolder) -> None:
+    def _write_results(self, build: BuildFolder, cdf_project: str | None = None) -> None:
         """Write build results including lineage information and insights to the build folder."""
 
         insight_file = build.build_dir / "insights.csv"
@@ -899,5 +899,5 @@ def _write_results(self, build: BuildFolder) -> None:
             safe_write(insight_file, insight_file_content)
 
         lineage_file = build.build_dir / BuildLineage.filename
-        lineage = BuildLineage.from_build(build).to_yaml()
+        lineage = BuildLineage.from_build(build, cdf_project).to_yaml()
         safe_write(lineage_file, lineage)

cognite_toolkit/_cdf_tk/commands/build_v2/data_classes/_lineage.py

@@ -43,14 +43,6 @@ class ResourceLineageItem(_BaseLineageModel):
     type: ResourceType
     built_file: AbsoluteFilePath
 
-    @field_serializer("source_file", "built_file", when_used="json")
-    def serialize_paths(self, value: Path, info: SerializationInfo) -> str:
-        """Serialize absolute paths to strings."""
-        organization_dir = info.context.get("organization_dir") if info.context else None
-        if organization_dir and value.is_relative_to(organization_dir):
-            return value.relative_to(organization_dir).as_posix()
-        return value.as_posix()
-
 
 class ModuleLineageItem(_BaseLineageModel):
     """Tracks a module through the build process."""
@@ -84,30 +76,22 @@ def status(self) -> str:
         else:
             return "FAILED: Unknown reason"
 
-    @field_serializer("module_path", when_used="json")
-    def serialize_module_path(self, value: Path, info: SerializationInfo) -> str:
-        """Serialize module path to relative path."""
-        organization_dir = info.context.get("organization_dir") if info.context else None
-        if organization_dir and value.is_relative_to(organization_dir):
-            return str(value.relative_to(organization_dir))
-        return str(value)
-
     @classmethod
     def from_built_module(cls, module: BuiltModule) -> "ModuleLineageItem":
         """Construct lineage item from built module."""
         resource_lineage = []
         for resource in module.resources:
             resource_lineage.append(
                 ResourceLineageItem(
-                    source_file=resource.source_path,
+                    source_file=resource.source_path.resolve(),
                     source_hash=resource.source_hash,
-                    built_file=resource.build_path,
+                    built_file=resource.build_path.resolve(),
                     type=resource.type,
                 )
             )
         return cls(
             module_id=module.module_id.id.as_posix(),
-            module_path=module.module_id.path,
+            module_path=module.module_id.path.resolve(),
             resource_lineage=resource_lineage,
             insights_summary=module.all_insights.summary,
         )
@@ -150,7 +134,7 @@ def serialize_duration(self, value: float | None, info: SerializationInfo) -> fl
         return round(value, 2) if value is not None else None
 
     @classmethod
-    def from_build(cls, build: BuildFolder) -> "BuildLineage":
+    def from_build(cls, build: BuildFolder, cdf_project: str | None = None) -> "BuildLineage":
         """Construct lineage from build output folder."""
 
         module_lineage = [ModuleLineageItem.from_built_module(module) for module in build.built_modules]
@@ -172,6 +156,7 @@ def from_build(cls, build: BuildFolder) -> "BuildLineage":
             module_lineage=module_lineage,
             modules_summary=modules_summary,
             insights_summary=insights_summary,
+            cdf_project=cdf_project,
         )
 
     def to_yaml(self) -> str:

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/datamodel.py

@@ -1372,7 +1372,8 @@ def get_extra_files(cls, filepath: Path, identifier: DataModelId, item: dict[str
 
         This includes a required .graphql file with the schema.
         """
-        graphql_file = filepath.with_suffix(".graphql")
+        graphql_file = cls._get_graphql_file(filepath)
+
         if not graphql_file.is_file():
             yield FailedReadExtra(
                 code="NOT-EXISTING",
@@ -1391,6 +1392,19 @@ def get_extra_files(cls, filepath: Path, identifier: DataModelId, item: dict[str
             description="GraphQL schema",
         )
 
+    @classmethod
+    def _get_graphql_file(cls, filepath: Path) -> Path:
+        filestem = filepath.stem
+        if filestem.lower().endswith(cls.kind.lower()):
+            filestem = filestem[: -len(cls.kind)].removesuffix(".").rstrip()
+
+        graphql_file = filepath.parent / f"{filestem}.graphql"
+        if not graphql_file.is_file() and filepath.with_suffix(".graphql").exists():
+            # Fallback
+            graphql_file = filepath.with_suffix(".graphql")
+
+        return graphql_file
+
     @classmethod
     def safe_read(cls, filepath: Path | str) -> str:
         # The version is a string, but the user often writes it as an int.
@@ -1413,7 +1427,7 @@ def load_resource_file(
         for item in raw_list:
             model_id = self.get_id(item)
             # Find the GraphQL files adjacent to the DML files
-            graphql_file = filepath.with_suffix(".graphql")
+            graphql_file = self._get_graphql_file(filepath)
             if not graphql_file.is_file():
                 raise ToolkitFileNotFoundError(
                     f"Failed to find GraphQL file. Expected {graphql_file.name} adjacent to {filepath.as_posix()}"

cognite_toolkit/_cdf_tk/cruds/_resource_cruds/file.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
 from collections.abc import Hashable, Iterable, Sequence
 from datetime import date, datetime
 from pathlib import Path
@@ -162,7 +160,9 @@ def load_resource_file(
         for item in raw_files:
             source_file = item.pop("$FILEPATH", None)
             if source_file is None:
-                if candidate := next((filepath.parent.rglob(f"{stem}*")), None):
+                if candidate := next(
+                    (file for file in filepath.parent.glob(f"{stem}*") if file != filepath and file.stem == stem), None
+                ):
                     source_file = candidate
                 elif isinstance(name := item.get("name"), str) and (filepath.parent / name).exists():
                     source_file = filepath.parent / name
@@ -336,7 +336,9 @@ def load_resource_file(
         for item in raw_files:
             source_file = item.pop("$FILEPATH", None)
             if source_file is None:
-                if candidate := next((filepath.parent.rglob(f"{stem}*")), None):
+                if candidate := next(
+                    (file for file in filepath.parent.glob(f"{stem}*") if file != filepath and file.stem == stem), None
+                ):
                     source_file = candidate
                 elif isinstance(name := item.get("name"), str) and (filepath.parent / name).exists():
                     source_file = filepath.parent / name
@@ -360,7 +362,7 @@ def load_resource_file(
 
     def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> CogniteFileRequest:
         request = super().load_resource(resource, is_dry_run)
-        request.filepath = self._filepath_by_node_id[self.get_id(resource)]
+        request.filepath = self._filepath_by_node_id.get(self.get_id(resource))
         return request
 
     def dump_resource(self, resource: CogniteFileResponse, local: dict[str, Any] | None = None) -> dict[str, Any]:

cognite_toolkit/_cdf_tk/utils/hashing.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import zipfile
 from pathlib import Path
 from typing import Any
 
@@ -47,6 +48,8 @@ def calculate_secure_hash(item: dict[str, Any], shorten: bool = False) -> str:
 def calculate_hash(content: str | bytes | Path, shorten: bool = False) -> str:
     sha256_hash = hashlib.sha256()
     if isinstance(content, Path):
+        if content.suffix == ".zip" and zipfile.is_zipfile(content):
+            return calculate_zipfile_hash(content, shorten=shorten)
         # Get rid of Windows line endings to make the hash consistent across platforms.
         content = content.read_bytes().replace(b"\r\n", b"\n")
     elif isinstance(content, str):
@@ -56,3 +59,12 @@ def calculate_hash(content: str | bytes | Path, shorten: bool = False) -> str:
     if shorten:
         return calculated[:8]
     return calculated
+
+
+def calculate_zipfile_hash(filepath: Path, shorten: bool = False) -> str:
+    """Calculate a hash of a zip file based on its contents, ignoring zip metadata.
+
+    It reads the contents directly from the zip file without extracting,
+    which ensures consistent hashing across platforms.
+    """
+    return "zipfil-hash-not-supported"

tests/test_unit/approval_client/client.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import builtins
 import hashlib
 import io
 import itertools
@@ -51,7 +52,7 @@
     VersionedDataModelingId,
     View,
 )
-from cognite.client.data_classes.data_modeling.ids import DataModelIdentifier, InstanceId
+from cognite.client.data_classes.data_modeling.ids import DataModelIdentifier
 from cognite.client.data_classes.functions import FunctionsStatus
 from cognite.client.data_classes.iam import CreatedSession, GroupWrite, ProjectSpec, TokenInspection
 from cognite.client.utils._text import to_camel_case
@@ -60,14 +61,15 @@
 
 from cognite_toolkit._cdf_tk.client import ToolkitClient, ToolkitClientConfig
 from cognite_toolkit._cdf_tk.client._resource_base import RequestResource, ResponseResource
-from cognite_toolkit._cdf_tk.client.identifiers import ExternalId, InternalId
+from cognite_toolkit._cdf_tk.client.identifiers import ExternalId, InstanceId, InternalId
+from cognite_toolkit._cdf_tk.client.resource_classes.cognite_file import CogniteFileRequest
 from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import (
     InstanceDefinition,
     InstanceRequest,
     NodeId,
 )
 from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling._instance import InstanceSlimDefinition
-from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataResponse
+from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataRequest, FileMetadataResponse
 from cognite_toolkit._cdf_tk.client.resource_classes.hosted_extractor_source._base import SourceRequestDefinition
 from cognite_toolkit._cdf_tk.client.resource_classes.project import ProjectStatus, ProjectStatusList
 from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWDatabaseResponse, RAWTableResponse
@@ -77,6 +79,7 @@
 from cognite_toolkit._cdf_tk.cruds import FileCRUD
 from cognite_toolkit._cdf_tk.utils import calculate_hash
 from cognite_toolkit._cdf_tk.utils.auth import CLIENT_NAME
+from tests.constants import CDF_PROJECT
 
 from .config import API_RESOURCES
 from .data_classes import APIResource, AuthGroupCalls
@@ -163,7 +166,7 @@ def __init__(self, mock_client: ToolkitClientMock, allow_reverse_lookup: bool =
         credentials.client_secret = "toolkit-client-secret"
         credentials.token_url = "https://toolkit.auth.com/oauth/token"
         credentials.scopes = ["ttps://pytest-field.cognitedata.com/.default"]
-        project = "test_project"
+        project = CDF_PROJECT
         self.mock_client.config = ToolkitClientConfig(
             client_name=CLIENT_NAME,
             project=project,
@@ -763,6 +766,34 @@ def create_instances_pydantic(items: Sequence[InstanceRequest]) -> list[Instance
                 for item in items
             ]
 
+        def create_cognite_file(items: Sequence[CogniteFileRequest]) -> list[InstanceSlimDefinition]:
+            created_resources[resource_cls.__name__].extend(items)
+            return [
+                InstanceSlimDefinition(
+                    instance_type="node",
+                    version=1,
+                    was_modified=True,
+                    space=item.space,
+                    external_id=item.external_id,
+                    created_time=1,
+                    last_updated_time=2,
+                )
+                for item in items
+            ]
+
+        def create_filemetadata_v2(items: Sequence[FileMetadataRequest], **_) -> list[FileMetadataResponse]:
+            created_resources[resource_cls.__name__].extend(items)
+            return [
+                FileMetadataResponse(
+                    **item.dump(),
+                    created_time=1,
+                    last_updated_time=2,
+                    id=LookUpAPIMock.create_id(item.external_id or "unknown"),
+                    uploaded=True,
+                )
+                for item in items
+            ]
+
         available_create_methods = {
             fn.__name__: fn
             for fn in [
@@ -783,6 +814,8 @@ def create_instances_pydantic(items: Sequence[InstanceRequest]) -> list[Instance
                 create,
                 create_hosted_extractor_source,
                 create_instances_pydantic,
+                create_cognite_file,
+                create_filemetadata_v2,
             ]
         }
         if mock_method not in available_create_methods:
@@ -934,14 +967,27 @@ def list_raw_table(db_name: str, limit: int | None = None) -> list[RAWTableRespo
 
         def retrieve_filemetadata(
             items: Sequence[InternalId | ExternalId | InstanceId], ignore_unknown_ids: bool = False
-        ) -> list[FileMetadataResponse]:
-            if len(items) == 1 and isinstance(items[0], InternalId):
-                return [
+        ) -> builtins.list[FileMetadataResponse]:
+            filemetadata_responses: builtins.list[FileMetadataResponse] = []
+            for item in items:
+                if isinstance(item, InternalId):
+                    id = item.id
+                    external_id = str(id)
+                elif isinstance(item, ExternalId):
+                    id = LookUpAPIMock.create_id(item.external_id or "unknown")
+                    external_id = item.external_id or "unknown"
+                elif isinstance(item, InstanceId):
+                    id = LookUpAPIMock.create_id(item.instance_id.external_id)
+                    external_id = item.instance_id.external_id
+                else:
+                    external_id = "unknown"
+                    id = 37
+                filemetadata_responses.append(
                     FileMetadataResponse(
-                        id=items[0].id, uploaded=True, created_time=0, last_updated_time=1, name="file"
+                        id=id, uploaded=True, created_time=0, last_updated_time=1, external_id=external_id, name="file"
                     )
-                ]
-            return []
+                )
+            return filemetadata_responses
 
         available_retrieve_methods = {
             fn.__name__: fn

tests/test_unit/approval_client/config.py

@@ -803,7 +803,7 @@
         resource_cls=CogniteFileResponse,
         _write_cls=CogniteFileRequest,
         methods={
-            "create": [Method(api_class_method="create", mock_class_method="create")],
+            "create": [Method(api_class_method="create", mock_class_method="create_cognite_file")],
             "retrieve": [
                 Method(api_class_method="retrieve", mock_class_method="retrieve"),
             ],
@@ -881,7 +881,10 @@
         resource_cls=FileMetadataResponse,
         _write_cls=FileMetadataRequest,
         methods={
-            "create": [Method(api_class_method="create", mock_class_method="create")],
+            "create": [
+                Method(api_class_method="create", mock_class_method="create_filemetadata_v2"),
+                Method(api_class_method="update", mock_class_method="create_filemetadata_v2"),
+            ],
             "retrieve": [
                 Method(api_class_method="retrieve", mock_class_method="retrieve_filemetadata"),
             ],