4545: Create skeleton of dataset endpoints (#4651)

warrkan · web-flow · commit c5b12cd33e0b · 2025-09-05T13:45:49.000+02:00
diff --git a/backend/app/api/dependencies.py b/backend/app/api/dependencies.py
@@ -8,7 +8,14 @@
 from fastapi import Depends, HTTPException, Request, status
 
 from app.core import Scheduler
-from app.services import ActivePipelineService, ConfigurationService, ModelService, PipelineService, SystemService
+from app.services import (
+    ActivePipelineService,
+    ConfigurationService,
+    DatasetService,
+    ModelService,
+    PipelineService,
+    SystemService,
+)
 from app.services.metrics_service import MetricsService
 from app.webrtc.manager import WebRTCManager
 
@@ -34,6 +41,13 @@ def get_source_id(source_id: str) -> UUID:
     return UUID(source_id)
 
 
+def get_project_id(project_id: str) -> UUID:
+    """Initializes and validates a project ID"""
+    if not is_valid_uuid(project_id):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid project ID")
+    return UUID(project_id)
+
+
 def get_sink_id(sink_id: str) -> UUID:
     """Initializes and validates a sink ID"""
     if not is_valid_uuid(sink_id):
@@ -48,6 +62,13 @@ def get_model_id(model_id: str) -> UUID:
     return UUID(model_id)
 
 
+def get_dataset_item_id(dataset_item_id: str) -> UUID:
+    """Initializes and validates a dataset item ID"""
+    if not is_valid_uuid(dataset_item_id):
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid dataset item ID")
+    return UUID(dataset_item_id)
+
+
 def get_pipeline_id(pipeline_id: str) -> UUID:
     """Initializes and validates a pipeline ID"""
     if not is_valid_uuid(pipeline_id):
@@ -114,6 +135,12 @@ def get_model_service(
     )
 
 
+@lru_cache
+def get_dataset_service() -> DatasetService:
+    """Provides a DatasetService instance."""
+    return DatasetService()
+
+
 def get_webrtc_manager(request: Request) -> WebRTCManager:
     """Provides the global WebRTCManager instance from FastAPI application's state."""
     return request.app.state.webrtc_manager
diff --git a/backend/app/api/endpoints/datasets.py b/backend/app/api/endpoints/datasets.py
@@ -0,0 +1,152 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+from datetime import datetime
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status
+from starlette.responses import FileResponse
+
+from app.api.dependencies import get_dataset_item_id, get_dataset_service, get_project_id
+from app.schemas import DatasetItem, DatasetItemsWithPagination
+from app.schemas.base import Pagination
+from app.services import DatasetService, ResourceNotFoundError
+
+router = APIRouter(prefix="/api/projects/{project_id}/dataset/items", tags=["Datasets"])
+
+DEFAULT_DATASET_ITEMS_NUMBER_RETURNED = 10
+MAX_DATASET_ITEMS_NUMBER_RETURNED = 100
+
+
+@router.post(
+    "",
+    status_code=status.HTTP_201_CREATED,
+    responses={status.HTTP_201_CREATED: {"description": "Dataset item created", "model": DatasetItem}},
+)
+def add_dataset_item(
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    dataset_service: Annotated[DatasetService, Depends(get_dataset_service)],
+    file: Annotated[UploadFile, File()],
+) -> DatasetItem:
+    """Add a new item to the dataset by uploading an image"""
+    return dataset_service.create_dataset_item(project_id=project_id, file=file.file)
+
+
+@router.get(
+    "",
+    responses={
+        status.HTTP_200_OK: {"description": "List of available dataset items", "model": DatasetItemsWithPagination},
+    },
+)
+def list_dataset_items(
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    dataset_service: Annotated[DatasetService, Depends(get_dataset_service)],
+    limit: Annotated[int, Query(ge=1, le=MAX_DATASET_ITEMS_NUMBER_RETURNED)] = DEFAULT_DATASET_ITEMS_NUMBER_RETURNED,
+    offset: Annotated[int, Query(ge=0)] = 0,
+    start_date: Annotated[datetime | None, Query()] = None,
+    end_date: Annotated[datetime | None, Query()] = None,
+) -> DatasetItemsWithPagination:
+    """List the available dataset items and their metadata. This endpoint supports pagination."""
+    if start_date is not None and end_date is not None and start_date > end_date:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Start date must be before end date."
+        )
+    dataset_items = dataset_service.list_dataset_items(
+        project_id=project_id, limit=limit, offset=offset, start_date=start_date, end_date=end_date
+    )
+    return DatasetItemsWithPagination(  # TODO: implement
+        items=dataset_items,
+        pagination=Pagination(
+            limit=limit,
+            offset=offset,
+            total=0,
+            count=len(dataset_items),
+        ),
+    )
+
+
+@router.get(
+    "/{dataset_item_id}",
+    responses={
+        status.HTTP_200_OK: {"description": "Dataset item found", "model": DatasetItem},
+        status.HTTP_400_BAD_REQUEST: {"description": "Invalid dataset item ID"},
+        status.HTTP_404_NOT_FOUND: {"description": "Dataset item not found"},
+    },
+)
+def get_dataset_item(
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    dataset_item_id: Annotated[UUID, Depends(get_dataset_item_id)],
+    dataset_service: Annotated[DatasetService, Depends(get_dataset_service)],
+) -> DatasetItem:
+    """Get information about a specific dataset item"""
+    try:
+        return dataset_service.get_dataset_item_by_id(project_id=project_id, dataset_item_id=dataset_item_id)
+    except ResourceNotFoundError as e:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
+
+
+@router.get(
+    "/{dataset_item_id}/binary",
+    responses={
+        status.HTTP_200_OK: {"description": "Dataset item binary found"},
+        status.HTTP_400_BAD_REQUEST: {"description": "Invalid dataset item ID"},
+        status.HTTP_404_NOT_FOUND: {"description": "Dataset item binary not found"},
+    },
+)
+def get_dataset_item_binary(
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    dataset_item_id: Annotated[UUID, Depends(get_dataset_item_id)],
+    dataset_service: Annotated[DatasetService, Depends(get_dataset_service)],
+) -> FileResponse:
+    """Get dataset item binary content"""
+    try:
+        binary_path = dataset_service.get_dataset_item_binary_path_by_id(
+            project_id=project_id, dataset_item_id=dataset_item_id
+        )
+        return FileResponse(path=binary_path)
+    except ResourceNotFoundError as e:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
+
+
+@router.get(
+    "/{dataset_item_id}/thumbnail",
+    responses={
+        status.HTTP_200_OK: {"description": "Dataset item thumbnail found"},
+        status.HTTP_400_BAD_REQUEST: {"description": "Invalid dataset item ID"},
+        status.HTTP_404_NOT_FOUND: {"description": "Dataset item thumbnail not found"},
+    },
+)
+def get_dataset_item_thumbnail(
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    dataset_item_id: Annotated[UUID, Depends(get_dataset_item_id)],
+    dataset_service: Annotated[DatasetService, Depends(get_dataset_service)],
+) -> FileResponse:
+    """Get dataset item thumbnail binary content"""
+    try:
+        thumbnail_path = dataset_service.get_dataset_item_thumbnail_path_by_id(
+            project_id=project_id, dataset_item_id=dataset_item_id
+        )
+        return FileResponse(path=thumbnail_path)
+    except ResourceNotFoundError as e:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
+
+
+@router.delete(
+    "/{dataset_item_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    responses={
+        status.HTTP_204_NO_CONTENT: {"description": "Dataset item deleted"},
+        status.HTTP_400_BAD_REQUEST: {"description": "Invalid dataset item ID"},
+        status.HTTP_404_NOT_FOUND: {"description": "Dataset item not found"},
+    },
+)
+def delete_dataset_item(
+    project_id: Annotated[UUID, Depends(get_project_id)],
+    dataset_item_id: Annotated[UUID, Depends(get_dataset_item_id)],
+    dataset_service: Annotated[DatasetService, Depends(get_dataset_service)],
+) -> None:
+    """Delete an item from the dataset"""
+    try:
+        dataset_service.delete_dataset_item(project_id=project_id, dataset_item_id=dataset_item_id)
+    except ResourceNotFoundError as e:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
diff --git a/backend/app/main.py b/backend/app/main.py
@@ -15,7 +15,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 
-from app.api.endpoints import models, pipelines, sinks, sources, system, webrtc
+from app.api.endpoints import datasets, models, pipelines, sinks, sources, system, webrtc
 from app.core import lifespan
 from app.settings import get_settings
 
@@ -50,6 +50,7 @@
 app.include_router(sources.router)
 app.include_router(sinks.router)
 app.include_router(pipelines.router)
+app.include_router(datasets.router)
 app.include_router(models.router)
 app.include_router(system.router)
 app.include_router(webrtc.router)
diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py
@@ -1,13 +1,16 @@
 # Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+from app.schemas.dataset_item import DatasetItem, DatasetItemsWithPagination
 from app.schemas.metrics import InferenceMetrics, LatencyMetrics, PipelineMetrics, TimeWindow
 from app.schemas.model import Model, ModelFormat
 from app.schemas.pipeline import Pipeline, PipelineStatus
 from app.schemas.sink import DisconnectedSinkConfig, OutputFormat, Sink, SinkType
 from app.schemas.source import DisconnectedSourceConfig, Source, SourceType
 
 __all__ = [
+    "DatasetItem",
+    "DatasetItemsWithPagination",
     "DisconnectedSinkConfig",
     "DisconnectedSourceConfig",
     "InferenceMetrics",
diff --git a/backend/app/schemas/base.py b/backend/app/schemas/base.py
@@ -18,3 +18,12 @@ class BaseIDNameModel(ABC, BaseModel):
 
     id: UUID = Field(default_factory=uuid4)
     name: str = "Default Name"
+
+
+class Pagination(ABC, BaseModel):
+    """Pagination model."""
+
+    offset: int  # index of the first item returned (0-based)
+    limit: int  # number of items requested per page
+    count: int  # number of items actually returned (may be less than limit if at the end)
+    total: int  # total number of items available
diff --git a/backend/app/schemas/dataset_item.py b/backend/app/schemas/dataset_item.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import StrEnum
+from uuid import UUID
+
+from pydantic import BaseModel
+
+from app.schemas.base import BaseIDNameModel, Pagination
+
+
+class DatasetItemFormat(StrEnum):
+    JPG = "jpg"
+    PNG = "png"
+
+
+class DatasetItemSubset(StrEnum):
+    UNASSIGNED = "unassigned"
+    TRAINING = "training"
+    VALIDATION = "validation"
+    TESTING = "testing"
+
+
+class DatasetItem(BaseIDNameModel):
+    """
+    Dataset item
+    """
+
+    format: DatasetItemFormat
+    width: int
+    height: int
+    size: int
+    source_id: UUID | None
+    subset: DatasetItemSubset
+
+    model_config = {
+        "json_schema_extra": {
+            "example": {
+                "id": "7b073838-99d3-42ff-9018-4e901eb047fc",
+                "name": "img-010203",
+                "format": "jpg",
+                "width": 1280,
+                "height": 720,
+                "size": 2211840,
+                "source_id": "c1feaabc-da2b-442e-9b3e-55c11c2c2ff3",
+                "subset": "unassigned",
+            }
+        }
+    }
+
+
+class DatasetItemsWithPagination(BaseModel):
+    """
+    Dataset Items list with pagination info
+    """
+
+    items: list[DatasetItem]
+    pagination: Pagination
diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py
@@ -4,6 +4,7 @@
 from .active_pipeline_service import ActivePipelineService
 from .base import ResourceAlreadyExistsError, ResourceInUseError, ResourceNotFoundError, ResourceType
 from .configuration_service import ConfigurationService
+from .dataset_service import DatasetService
 from .dispatch_service import DispatchService
 from .metrics_service import MetricsService
 from .model_service import ModelAlreadyExistsError, ModelService
@@ -14,6 +15,7 @@
 __all__ = [
     "ActivePipelineService",
     "ConfigurationService",
+    "DatasetService",
     "DispatchService",
     "MetricsService",
     "ModelAlreadyExistsError",
diff --git a/backend/app/services/base.py b/backend/app/services/base.py
@@ -24,6 +24,7 @@ class ResourceType(StrEnum):
     SINK = "Sink"
     MODEL = "Model"
     PIPELINE = "Pipeline"
+    DATASET_ITEM = "DatasetItem"
 
 
 class ResourceError(Exception):
diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py
@@ -0,0 +1,46 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import BinaryIO
+from uuid import UUID
+
+from app.schemas.dataset_item import DatasetItem
+from app.services.base import ResourceNotFoundError, ResourceType
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetService:
+    def create_dataset_item(self, project_id: UUID, file: BinaryIO) -> DatasetItem:
+        """Creates a new dataset item"""
+        raise NotImplementedError
+
+    def list_dataset_items(
+        self,
+        project_id: UUID,  # noqa: ARG002
+        limit: int = 20,  # noqa: ARG002
+        offset: int = 0,  # noqa: ARG002
+        start_date: datetime | None = None,  # noqa: ARG002
+        end_date: datetime | None = None,  # noqa: ARG002
+    ) -> list[DatasetItem]:
+        """Get information about available dataset items"""
+        return []
+
+    def get_dataset_item_by_id(self, project_id: UUID, dataset_item_id: UUID) -> DatasetItem:  # noqa: ARG002
+        """Get a dataset item by its ID"""
+        raise ResourceNotFoundError(ResourceType.DATASET_ITEM, str(dataset_item_id))
+
+    def get_dataset_item_binary_path_by_id(self, project_id: UUID, dataset_item_id: UUID) -> Path | str:  # noqa: ARG002
+        """Get a dataset item binary content by its ID"""
+        raise ResourceNotFoundError(ResourceType.DATASET_ITEM, str(dataset_item_id))
+
+    def get_dataset_item_thumbnail_path_by_id(self, project_id: UUID, dataset_item_id: UUID) -> Path | str:  # noqa: ARG002
+        """Get a dataset item thumbnail binary content by its ID"""
+        raise ResourceNotFoundError(ResourceType.DATASET_ITEM, str(dataset_item_id))
+
+    def delete_dataset_item(self, project_id: UUID, dataset_item_id: UUID) -> None:  # noqa: ARG002
+        """Delete a dataset item by its ID"""
+        raise ResourceNotFoundError(ResourceType.DATASET_ITEM, str(dataset_item_id))
diff --git a/backend/tests/unit/endpoints/test_datasets.py b/backend/tests/unit/endpoints/test_datasets.py
