[CDF-27549] Add Dune App resource behind alpha flag apps

Apps live under apps/ with kind App: zip sources like functions, upload via
Files API init to /dune-apps/ and PUT zip (Streamlit-style flow). Gated by
hidden alpha flag apps in cdf.toml.

Author: ronpal

cognite_toolkit/_cdf_tk/builders/__init__.py

@@ -3,6 +3,7 @@
 
 from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning
 
+from ._app import AppBuilder
 from ._base import Builder, DefaultBuilder, get_resource_crud
 from ._datamodels import DataModelBuilder
 from ._file import FileBuilder
@@ -27,6 +28,7 @@ def create_builder(
 
 _BUILDER_BY_RESOURCE_FOLDER = {_builder._resource_folder: _builder for _builder in Builder.__subclasses__()}
 __all__ = [
+    "AppBuilder",
     "Builder",
     "DataModelBuilder",
     "DefaultBuilder",

cognite_toolkit/_cdf_tk/builders/_app.py

@@ -0,0 +1,126 @@
+import shutil
+from collections.abc import Callable, Iterable, Sequence
+from pathlib import Path
+
+from cognite_toolkit._cdf_tk.builders._base import Builder
+from cognite_toolkit._cdf_tk.cruds import AppCRUD
+from cognite_toolkit._cdf_tk.data_classes import (
+    BuildDestinationFile,
+    BuildSourceFile,
+    BuiltResourceList,
+    ModuleLocation,
+)
+from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError
+from cognite_toolkit._cdf_tk.tk_warnings import (
+    FileReadWarning,
+    HighSeverityWarning,
+    LowSeverityWarning,
+    ToolkitWarning,
+    WarningList,
+)
+
+
+class AppBuilder(Builder):
+    _resource_folder = AppCRUD.folder_name
+
+    def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None:
+        super().__init__(build_dir, warn=warn)
+
+    def build(
+        self,
+        source_files: list[BuildSourceFile],
+        module: ModuleLocation,
+        console: Callable[[str], None] | None = None,
+    ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]:
+        for source_file in source_files:
+            if source_file.loaded is None:
+                continue
+            if source_file.source.path.parent.parent != module.dir:
+                continue
+
+            loader, warning = self._get_loader(source_file.source.path)
+            if loader is None:
+                if warning is not None:
+                    yield [warning]
+                continue
+
+            warnings = WarningList[FileReadWarning]()
+            if loader is AppCRUD:
+                warnings = self.copy_app_directory_to_build(source_file)
+
+            destination_path = self._create_destination_path(source_file.source.path, loader.kind)
+
+            yield BuildDestinationFile(
+                path=destination_path,
+                loaded=source_file.loaded,
+                loader=loader,
+                source=source_file.source,
+                extra_sources=None,
+                warnings=warnings,
+            )
+
+    def validate_directory(
+        self,
+        built_resources: BuiltResourceList,
+        module: ModuleLocation,
+    ) -> WarningList[ToolkitWarning]:
+        warnings = WarningList[ToolkitWarning]()
+        has_config_files = any(resource.kind == AppCRUD.kind for resource in built_resources)
+        if has_config_files:
+            return warnings
+        config_files_misplaced = [
+            file
+            for file in module.source_paths_by_resource_folder[AppCRUD.folder_name]
+            if AppCRUD.is_supported_file(file)
+        ]
+        if config_files_misplaced:
+            for yaml_source_path in config_files_misplaced:
+                required_location = module.dir / AppCRUD.folder_name / yaml_source_path.name
+                warning = LowSeverityWarning(
+                    f"The required App resource configuration file "
+                    f"was not found in {required_location.as_posix()!r}. "
+                    f"The file {yaml_source_path.as_posix()!r} is currently "
+                    f"considered part of the App's artifacts and "
+                    f"will not be processed by the Toolkit.",
+                )
+                warnings.append(warning)
+        return warnings
+
+    def copy_app_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]:
+        raw_content = source_file.loaded
+        if raw_content is None:
+            raise ToolkitValueError("App source file should be a YAML file.")
+        raw_apps = raw_content if isinstance(raw_content, list) else [raw_content]
+        warnings = WarningList[FileReadWarning]()
+        for raw_app in raw_apps:
+            app_external_id = raw_app.get("appExternalId")
+            if not app_external_id:
+                warnings.append(
+                    HighSeverityWarning(
+                        f"App in {source_file.source.path.as_posix()!r} has no appExternalId defined. "
+                        f"This is used to match the app to the app directory.",
+                    ),
+                )
+                continue
+            if not raw_app.get("version"):
+                warnings.append(
+                    HighSeverityWarning(
+                        f"App {app_external_id} in {source_file.source.path.as_posix()!r} has no version defined.",
+                    ),
+                )
+
+            app_directory = source_file.source.path.with_name(app_external_id)
+
+            if not app_directory.is_dir():
+                raise ToolkitNotADirectoryError(
+                    f"App directory not found for appExternalId {app_external_id} defined in {source_file.source.path.as_posix()!r}.",
+                )
+
+            destination = self.build_dir / self.resource_folder / app_external_id
+            if destination.exists():
+                raise ToolkitFileExistsError(
+                    f"App {app_external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.",
+                )
+            shutil.copytree(app_directory, destination, ignore=shutil.ignore_patterns("__pycache__"))
+
+        return warnings

cognite_toolkit/_cdf_tk/client/_toolkit_client.py

@@ -9,6 +9,7 @@
 
 from .api.agents import AgentsAPI
 from .api.annotations import AnnotationsAPI
+from .api.apps import AppsAPI
 from .api.assets import AssetsAPI
 from .api.canvas import IndustrialCanvasAPI
 from .api.cognite_files import CogniteFilesAPI
@@ -61,6 +62,7 @@ class ToolAPI:
     def __init__(self, http_client: HTTPClient, console: Console) -> None:
         self.http_client = http_client
         self.agents = AgentsAPI(http_client)
+        self.apps = AppsAPI(http_client)
         self.annotations = AnnotationsAPI(http_client)
         self.assets = AssetsAPI(http_client)
         self.cognite_files = CogniteFilesAPI(http_client)

cognite_toolkit/_cdf_tk/client/api/apps.py

@@ -0,0 +1,96 @@
+"""Apps API: Dune apps as classic files under /dune-apps/ (same pattern as Streamlit + /files)."""
+
+from collections.abc import Iterable, Sequence
+from typing import Literal
+
+from cognite_toolkit._cdf_tk.client.cdf_client import CDFResourceAPI, PagedResponse, ResponseItems
+from cognite_toolkit._cdf_tk.client.cdf_client.api import Endpoint
+from cognite_toolkit._cdf_tk.client.http_client import (
+    HTTPClient,
+    ItemsSuccessResponse,
+    RequestMessage,
+    SuccessResponse,
+)
+from cognite_toolkit._cdf_tk.client.identifiers import ExternalId
+from cognite_toolkit._cdf_tk.client.request_classes.filters import DuneAppFilter
+from cognite_toolkit._cdf_tk.client.resource_classes.app import AppRequest, AppResponse
+
+
+class AppsAPI(CDFResourceAPI[AppResponse]):
+    """Dune apps are file metadata objects under ``/dune-apps/`` with a zip uploaded to ``uploadUrl``."""
+
+    def __init__(self, http_client: HTTPClient) -> None:
+        super().__init__(
+            http_client=http_client,
+            method_endpoint_map={
+                "create": Endpoint(method="POST", path="/files", item_limit=1, concurrency_max_workers=1),
+                "retrieve": Endpoint(method="POST", path="/files/byids", item_limit=1000, concurrency_max_workers=1),
+                "update": Endpoint(method="POST", path="/files/update", item_limit=1000, concurrency_max_workers=1),
+                "delete": Endpoint(method="POST", path="/files/delete", item_limit=1000, concurrency_max_workers=1),
+                "list": Endpoint(method="POST", path="/files/list", item_limit=1000),
+            },
+        )
+
+    def _validate_page_response(self, response: SuccessResponse | ItemsSuccessResponse) -> PagedResponse[AppResponse]:
+        return PagedResponse[AppResponse].model_validate_json(response.body)
+
+    def _reference_response(self, response: SuccessResponse) -> ResponseItems[ExternalId]:
+        return ResponseItems[ExternalId].model_validate_json(response.body)
+
+    def create(self, items: Sequence[AppRequest], overwrite: bool = False) -> list[AppResponse]:
+        endpoint = self._method_endpoint_map["create"]
+        results: list[AppResponse] = []
+        for item in items:
+            request = RequestMessage(
+                endpoint_url=self._make_url(endpoint.path),
+                method=endpoint.method,
+                body_content=item.dump(),
+                parameters={"overwrite": overwrite},
+            )
+            response = self._http_client.request_single_retries(request)
+            result = response.get_success_or_raise(request)
+            results.append(AppResponse.model_validate_json(result.body))
+        return results
+
+    def retrieve(self, items: Sequence[ExternalId], ignore_unknown_ids: bool = False) -> list[AppResponse]:
+        return self._request_item_response(
+            items, method="retrieve", extra_body={"ignoreUnknownIds": ignore_unknown_ids}
+        )
+
+    def update(self, items: Sequence[AppRequest], mode: Literal["patch", "replace"] = "replace") -> list[AppResponse]:
+        return self._update(items, mode=mode)
+
+    def delete(self, items: Sequence[ExternalId], ignore_unknown_ids: bool = False) -> None:
+        self._request_no_response(items, "delete", extra_body={"ignoreUnknownIds": ignore_unknown_ids})
+
+    def paginate(
+        self,
+        filter: DuneAppFilter | None = None,
+        limit: int = 100,
+        cursor: str | None = None,
+    ) -> PagedResponse[AppResponse]:
+        return self._paginate(
+            cursor=cursor,
+            limit=limit,
+            body={"filter": (filter or DuneAppFilter()).dump()},
+        )
+
+    def iterate(
+        self,
+        filter: DuneAppFilter | None = None,
+        limit: int | None = 100,
+    ) -> Iterable[list[AppResponse]]:
+        return self._iterate(
+            limit=limit,
+            body={"filter": (filter or DuneAppFilter()).dump()},
+        )
+
+    def list(
+        self,
+        filter: DuneAppFilter | None = None,
+        limit: int | None = 100,
+    ) -> list[AppResponse]:
+        return self._list(
+            limit=limit,
+            body={"filter": (filter or DuneAppFilter()).dump()},
+        )

cognite_toolkit/_cdf_tk/client/request_classes/filters.py

@@ -5,6 +5,7 @@
 from pydantic_core.core_schema import ValidationInfo
 
 from cognite_toolkit._cdf_tk.client.identifiers import ExternalId, InternalId, ViewId
+from cognite_toolkit._cdf_tk.client.resource_classes import app as app_resources
 from cognite_toolkit._cdf_tk.client.resource_classes import streamlit_
 from cognite_toolkit._cdf_tk.client.resource_classes.annotation import AnnotationStatus, AnnotationType
 from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeId
@@ -56,6 +57,15 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]:
         return body
 
 
+class DuneAppFilter(ClassicFilter):
+    """List filter for Dune app zips stored under ``/dune-apps/``."""
+
+    def dump(self, camel_case: bool = True) -> dict[str, Any]:
+        body = self.model_dump(mode="json", by_alias=camel_case, exclude_unset=True)
+        body["directoryPrefix"] = app_resources.DUNE_APPS_DIRECTORY
+        return body
+
+
 class TransformationFilter(Filter):
     data_set_ids: list[ExternalId | InternalId] | None = None
 

cognite_toolkit/_cdf_tk/client/resource_classes/app.py

@@ -0,0 +1,106 @@
+from typing import Any, Literal
+
+from pydantic import Field, model_validator
+
+from cognite_toolkit._cdf_tk.client._resource_base import (
+    BaseModelObject,
+    ResponseResource,
+    UpdatableRequestResource,
+)
+from cognite_toolkit._cdf_tk.client.identifiers import ExternalId
+
+DUNE_APPS_DIRECTORY = "/dune-apps/"
+
+
+class AppShared(BaseModelObject):
+    """Fields shared between app write/read models (under /dune-apps/)."""
+
+    app_external_id: str = Field(description="Logical app id; directory name in the module.")
+    version: str
+    name: str
+    description: str | None = None
+    published: bool | None = True
+    data_set_id: int | None = None
+    cognite_toolkit_app_hash: str | None = Field(None, alias="cdf-toolkit-app-hash")
+
+
+class AppRequest(AppShared, UpdatableRequestResource):
+    """Create/update body for Files API init; zip is uploaded separately to uploadUrl."""
+
+    @property
+    def external_id(self) -> str:
+        return f"{self.app_external_id}-{self.version}"
+
+    def as_id(self) -> ExternalId:
+        return ExternalId(external_id=self.external_id)
+
+    def dump(
+        self, camel_case: bool = True, exclude_extra: bool = False, context: Literal["api", "toolkit"] = "api"
+    ) -> dict[str, Any]:
+        if context == "toolkit":
+            dumped = super().dump(camel_case=camel_case, exclude_extra=exclude_extra)
+            dumped.pop("externalId", None)
+            dumped.pop("external_id", None)
+            return dumped
+        feid = self.external_id
+        return {
+            "externalId" if camel_case else "external_id": feid,
+            "name": f"{feid}.zip",
+            "dataSetId" if camel_case else "data_set_id": self.data_set_id,
+            "directory": DUNE_APPS_DIRECTORY,
+            "metadata": self._as_metadata(),
+        }
+
+    def _as_metadata(self) -> dict[str, str]:
+        metadata: dict[str, str] = {
+            "published": str(bool(self.published)).lower(),
+            "name": self.name,
+            "description": self.description or "",
+            "externalId": self.external_id,
+            "version": self.version,
+            "appExternalId": self.app_external_id,
+        }
+        if self.cognite_toolkit_app_hash:
+            metadata["cdf-toolkit-app-hash"] = self.cognite_toolkit_app_hash
+        return metadata
+
+    def as_update(self, mode: Literal["patch", "replace"]) -> dict[str, Any]:
+        update_data: dict[str, Any] = {
+            "metadata": {"set": self._as_metadata()},
+            "directory": {"set": DUNE_APPS_DIRECTORY},
+        }
+        if self.data_set_id is not None:
+            update_data["dataSetId"] = {"set": self.data_set_id}
+        elif mode == "replace":
+            update_data["dataSetId"] = {"setNull": True}
+
+        return {
+            "externalId": self.external_id,
+            "update": update_data,
+        }
+
+
+class AppResponse(AppShared, ResponseResource[AppRequest]):
+    """File metadata for a Dune app after create/retrieve (metadata merged to top level)."""
+
+    external_id: str
+    id: int
+    created_time: int
+    last_updated_time: int
+    uploaded_time: int | None = None
+    uploaded: bool
+    upload_url: str | None = None
+
+    @classmethod
+    def request_cls(cls) -> type[AppRequest]:
+        return AppRequest
+
+    @model_validator(mode="before")
+    @classmethod
+    def move_metadata(cls, values: dict[str, Any]) -> dict[str, Any]:
+        if "metadata" not in values:
+            return values
+        values_copy = values.copy()
+        metadata = values_copy.pop("metadata")
+        values_copy.update(metadata)
+        return values_copy