[CDF-27549] Dune App resource (Files API + zip, alpha flag)

cognite_toolkit/_cdf_tk/builders/__init__.py

@@ -3,6 +3,7 @@
 
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
 
+from ._app import AppBuilder
 from ._base import Builder, DefaultBuilder, get_resource_crud
 from ._datamodels import DataModelBuilder
 from ._file import FileBuilder
@@ -27,6 +28,7 @@ def create_builder(
 
 _BUILDER_BY_RESOURCE_FOLDER = {_builder._resource_folder: _builder for _builder in Builder.__subclasses__()}
 __all__ = [
+    "AppBuilder",
     "Builder",
     "DataModelBuilder",
     "DefaultBuilder",

cognite_toolkit/_cdf_tk/builders/_app.py

@@ -0,0 +1,123 @@
+import shutil
+from collections.abc import Callable, Iterable, Sequence
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from cognite_toolkit._cdf_tk.builders._base import Builder
+from cognite_toolkit._cdf_tk.cruds import AppCRUD
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildDestinationFile,
+    BuildSourceFile,
+    BuiltResourceList,
+    ModuleLocation,
+)
+from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError
+from cognite_toolkit._cdf_tk.tk_warnings import (
+    FileReadWarning,
+    HighSeverityWarning,
+    LowSeverityWarning,
+    ToolkitWarning,
+    WarningList,
+)
+from cognite_toolkit._cdf_tk.yaml_classes import AppsYAML
+
+
+class AppBuilder(Builder):
+    _resource_folder = AppCRUD.folder_name
+
+    def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None:
+        super().__init__(build_dir, warn=warn)
+
+    def build(
+        self,
+        source_files: list[BuildSourceFile],
+        module: ModuleLocation,
+        console: Callable[[str], None] | None = None,
+    ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]:
+        for source_file in source_files:
+            if source_file.loaded is None:
+                continue
+            if source_file.source.path.parent.parent != module.dir:
+                continue
+
+            loader, warning = self._get_loader(source_file.source.path)
+            if loader is None:
+                if warning is not None:
+                    yield [warning]
+                continue
+
+            warnings = WarningList[FileReadWarning]()
+            if loader is AppCRUD:
+                warnings = self.copy_app_directory_to_build(source_file)
+
+            destination_path = self._create_destination_path(source_file.source.path, loader.kind)
+
+            yield BuildDestinationFile(
+                path=destination_path,
+                loaded=source_file.loaded,
+                loader=loader,
+                source=source_file.source,
+                extra_sources=None,
+                warnings=warnings,
+            )
+
+    def validate_directory(
+        self,
+        built_resources: BuiltResourceList,
+        module: ModuleLocation,
+    ) -> WarningList[ToolkitWarning]:
+        warnings = WarningList[ToolkitWarning]()
+        has_config_files = any(resource.kind == AppCRUD.kind for resource in built_resources)
+        if has_config_files:
+            return warnings
+        config_files_misplaced = [
+            file
+            for file in module.source_paths_by_resource_folder[AppCRUD.folder_name]
+            if AppCRUD.is_supported_file(file)
+        ]
+        if config_files_misplaced:
+            for yaml_source_path in config_files_misplaced:
+                required_location = module.dir / AppCRUD.folder_name / yaml_source_path.name
+                warning = LowSeverityWarning(
+                    f"The required App resource configuration file "
+                    f"was not found in {required_location.as_posix()!r}. "
+                    f"The file {yaml_source_path.as_posix()!r} is currently "
+                    f"considered part of the App's artifacts and "
+                    f"will not be processed by the Toolkit.",
+                )
+                warnings.append(warning)
+        return warnings
+
+    def copy_app_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]:
+        raw_content = source_file.loaded
+        if raw_content is None:
+            raise ToolkitValueError("App source file should be a YAML file.")
+        raw_apps = raw_content if isinstance(raw_content, list) else [raw_content]
+        warnings = WarningList[FileReadWarning]()
+        for raw_app in raw_apps:
+            try:
+                app_config = AppsYAML.model_validate(raw_app)
+            except ValidationError as e:
+                warnings.append(
+                    HighSeverityWarning(
+                        f"App in {source_file.source.path.as_posix()!r} has invalid configuration: {e}",
+                    ),
+                )
+                continue
+
+            app_directory = source_file.source.path.with_name(app_config.app_external_id)
+
+            if not app_directory.is_dir():
+                raise ToolkitNotADirectoryError(
+                    f"App directory not found for appExternalId {app_config.app_external_id} defined in {source_file.source.path.as_posix()!r}.",
+                )
+
+            destination = self.build_dir / self.resource_folder / app_config.app_external_id
+            if destination.exists():
+                raise ToolkitFileExistsError(
+                    f"App {app_config.app_external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.",
+                )
+            shutil.copytree(app_directory, destination, ignore=shutil.ignore_patterns("__pycache__"))
+
+        return warnings

cognite_toolkit/_cdf_tk/client/_toolkit_client.py

@@ -9,6 +9,7 @@
 
 from .api.agents import AgentsAPI
 from .api.annotations import AnnotationsAPI
+from .api.apps import AppsAPI
 from .api.assets import AssetsAPI
 from .api.canvas import IndustrialCanvasAPI
 from .api.cognite_files import CogniteFilesAPI
@@ -61,6 +62,7 @@ class ToolAPI:
     def __init__(self, http_client: HTTPClient, console: Console) -> None:
         self.http_client = http_client
         self.agents = AgentsAPI(http_client)
+        self.apps = AppsAPI(http_client)
         self.annotations = AnnotationsAPI(http_client)
         self.assets = AssetsAPI(http_client)
         self.cognite_files = CogniteFilesAPI(http_client)

cognite_toolkit/_cdf_tk/client/api/apps.py

@@ -0,0 +1,96 @@
+"""Apps API: Dune apps as classic files under /dune-apps/ (same pattern as Streamlit + /files)."""
+
+from collections.abc import Iterable, Sequence
+from typing import Literal
+
+from cognite_toolkit._cdf_tk.client.cdf_client import CDFResourceAPI, PagedResponse, ResponseItems
+from cognite_toolkit._cdf_tk.client.cdf_client.api import Endpoint
+from cognite_toolkit._cdf_tk.client.http_client import (
+    HTTPClient,
+    ItemsSuccessResponse,
+    RequestMessage,
+    SuccessResponse,
+)
+from cognite_toolkit._cdf_tk.client.identifiers import ExternalId
+from cognite_toolkit._cdf_tk.client.request_classes.filters import DuneAppFilter
+from cognite_toolkit._cdf_tk.client.resource_classes.app import AppRequest, AppResponse
+
+
+class AppsAPI(CDFResourceAPI[AppResponse]):
+    """Dune apps are file metadata objects under ``/dune-apps/`` with a zip uploaded to ``uploadUrl``."""
+
+    def __init__(self, http_client: HTTPClient) -> None:
+        super().__init__(
+            http_client=http_client,
+            method_endpoint_map={
+                "create": Endpoint(method="POST", path="/files", item_limit=1, concurrency_max_workers=1),
+                "retrieve": Endpoint(method="POST", path="/files/byids", item_limit=1000, concurrency_max_workers=1),
+                "update": Endpoint(method="POST", path="/files/update", item_limit=1000, concurrency_max_workers=1),
+                "delete": Endpoint(method="POST", path="/files/delete", item_limit=1000, concurrency_max_workers=1),
+                "list": Endpoint(method="POST", path="/files/list", item_limit=1000),
+            },
+        )
+
+    def _validate_page_response(self, response: SuccessResponse | ItemsSuccessResponse) -> PagedResponse[AppResponse]:
+        return PagedResponse[AppResponse].model_validate_json(response.body)
+
+    def _reference_response(self, response: SuccessResponse) -> ResponseItems[ExternalId]:
+        return ResponseItems[ExternalId].model_validate_json(response.body)
+
+    def create(self, items: Sequence[AppRequest], overwrite: bool = False) -> list[AppResponse]:
+        endpoint = self._method_endpoint_map["create"]
+        results: list[AppResponse] = []
+        for item in items:
+            request = RequestMessage(
+                endpoint_url=self._make_url(endpoint.path),
+                method=endpoint.method,
+                body_content=item.dump(),
+                parameters={"overwrite": overwrite},
+            )
+            response = self._http_client.request_single_retries(request)
+            result = response.get_success_or_raise(request)
+            results.append(AppResponse.model_validate_json(result.body))
+        return results
+
+    def retrieve(self, items: Sequence[ExternalId], ignore_unknown_ids: bool = False) -> list[AppResponse]:
+        return self._request_item_response(
+            items, method="retrieve", extra_body={"ignoreUnknownIds": ignore_unknown_ids}
+        )
+
+    def update(self, items: Sequence[AppRequest], mode: Literal["patch", "replace"] = "replace") -> list[AppResponse]:
+        return self._update(items, mode=mode)
+
+    def delete(self, items: Sequence[ExternalId], ignore_unknown_ids: bool = False) -> None:
+        self._request_no_response(items, "delete", extra_body={"ignoreUnknownIds": ignore_unknown_ids})
+
+    def paginate(
+        self,
+        filter: DuneAppFilter | None = None,
+        limit: int = 100,
+        cursor: str | None = None,
+    ) -> PagedResponse[AppResponse]:
+        return self._paginate(
+            cursor=cursor,
+            limit=limit,
+            body={"filter": (filter or DuneAppFilter()).dump()},
+        )
+
+    def iterate(
+        self,
+        filter: DuneAppFilter | None = None,
+        limit: int | None = 100,
+    ) -> Iterable[list[AppResponse]]:
+        return self._iterate(
+            limit=limit,
+            body={"filter": (filter or DuneAppFilter()).dump()},
+        )
+
+    def list(
+        self,
+        filter: DuneAppFilter | None = None,
+        limit: int | None = 100,
+    ) -> list[AppResponse]:
+        return self._list(
+            limit=limit,
+            body={"filter": (filter or DuneAppFilter()).dump()},
+        )

cognite_toolkit/_cdf_tk/client/api/filemetadata.py

@@ -8,7 +8,6 @@
 from cognite_toolkit._cdf_tk.client.cdf_client.api import Endpoint
 from cognite_toolkit._cdf_tk.client.http_client import (
     HTTPClient,
-    HTTPResult,
     ItemsSuccessResponse,
     RequestMessage,
     SuccessResponse,
@@ -203,22 +202,16 @@ def upload_file_link(
                 _ = response.get_success_or_raise(request)
         return results
 
-    def upload_content(self, data_content: bytes, upload_url: str, mime_type: str | None = None) -> HTTPResult:
-        """Uploads file content to CDF.
-
-        Args:
-            data_content: Content to be uploaded.
-            upload_url: Upload URL.
-            mime_type: MIME type to upload. None for no MIME type.
-        """
-        return self._http_client.request_single_retries(
-            RequestMessage(
-                endpoint_url=upload_url,
-                method="PUT",
-                content_type=mime_type or "application/octet-stream",
-                data_content=data_content,
-            )
+    def upload_content(self, data_content: bytes, upload_url: str, mime_type: str | None = None) -> None:
+        """Uploads file content to the signed upload URL (same flow as ``upload_file`` for in-memory bytes)."""
+        request = RequestMessage(
+            endpoint_url=upload_url,
+            method="PUT",
+            content_type=mime_type or "application/octet-stream",
+            data_content=data_content,
         )
+        upload_response = self._http_client.request_single_retries(request)
+        upload_response.get_success_or_raise(request)
 
     def set_pending_ids(self, items: Sequence[PendingInstanceId]) -> builtins.list[FileMetadataResponse]:
         """Set pending instance IDs for one or more file metadata entries.

cognite_toolkit/_cdf_tk/client/request_classes/filters.py

@@ -5,6 +5,7 @@
 from pydantic_core.core_schema import ValidationInfo
 
 from cognite_toolkit._cdf_tk.client.identifiers import ExternalId, InternalId, ViewId
+from cognite_toolkit._cdf_tk.client.resource_classes import app as app_resources
 from cognite_toolkit._cdf_tk.client.resource_classes import streamlit_
 from cognite_toolkit._cdf_tk.client.resource_classes.annotation import AnnotationStatus, AnnotationType
 from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeId
@@ -56,6 +57,15 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]:
         return body
 
 
+class DuneAppFilter(ClassicFilter):
+    """List filter for Dune app zips stored under ``/dune-apps/``."""
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]:
+        body = self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+        body["directoryPrefix"] = app_resources.DUNE_APPS_DIRECTORY
+        return body
+
+
 class TransformationFilter(Filter):
     data_set_ids: list[ExternalId | InternalId] | None = None