Implement dataset export/import (#1946)

This pull request adds support to import and export datasets. An
exported dataset will include all the images, where references to
callables for images/masks are saved as PNG and image paths will be
copied. The exported dataset also includes metadata, such as the
categories and schema of the dataset and a version number in a JSON
file. The polars dataframe is stored in parquet. Additionally, there is
an option to export everything in a zip. This zip can also be imported
as is without the need to manually extract it first.

**Serialization and Deserialization Enhancements**
* Added `to_dict` and `from_dict` methods to `Categories`,
`LabelCategories`, `HierarchicalLabelCategories`, and `MaskCategories`,
enabling polymorphic serialization and reconstruction of category
objects.
[[1]](diffhunk://#diff-f0858b860d68536c04476766ef0512d873b7f7cc20dde377a42ec13a836d9622L58-R101)
[[2]](diffhunk://#diff-f0858b860d68536c04476766ef0512d873b7f7cc20dde377a42ec13a836d9622R181-R223)
[[3]](diffhunk://#diff-f0858b860d68536c04476766ef0512d873b7f7cc20dde377a42ec13a836d9622R407-R496)
[[4]](diffhunk://#diff-f0858b860d68536c04476766ef0512d873b7f7cc20dde377a42ec13a836d9622R565-R601)
* Implemented `to_dict` and `from_dict` methods for the `Field` base
class, using dataclass introspection to automatically serialize and
reconstruct field attributes, including special handling for semantic
and dtype fields.
* Added serialization and deserialization logic to the `Schema` class,
including storing type information and reconstructing attribute types
via module introspection.

**Dataset Import/Export API**
* Exposed `export_dataset` and `import_dataset` functions at the module
level, and added `export` and `from_file` methods to the `Dataset` class
for saving/loading datasets in a structured format.
[[1]](diffhunk://#diff-6c66a059cb4075cd7afbc29b4f34236f8101a1a47cdcc429c7a2d25868cc74b5R8)
[[2]](diffhunk://#diff-4ac196ddc4dc8e6d33daf684ded18886ff8774fadb8b6cbd4bfa88ca424bb34fR500-R554)

These changes lay the groundwork for robust dataset interchange and
future compatibility across different category and schema types.

<!-- Contributing guide:
https://github.com/open-edge-platform/datumaro/blob/develop/contributing.md
-->

<!--
Please add a summary of changes. You may use Copilot to auto-generate
the PR description but please consider including any other relevant
facts which Copilot may be unaware of (such as design choices and
testing procedure).

Add references to the relevant issues and pull requests if any like so:

Resolves #111 and #222.
Depends on #1000 (for series of dependent commits).
-->

### Checklist
<!-- Put an 'x' in all the boxes that apply -->
- [ ] I have added tests to cover my changes or documented any manual
tests.
- [ ] I have updated the
[documentation](https://github.com/open-edge-platform/datumaro/tree/develop/docs)
accordingly

---------

Signed-off-by: Albert van Houten <[email protected]>
Co-authored-by: Copilot <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

src/datumaro/experimental/__init__.py

@@ -5,6 +5,7 @@
 from . import converters  # Import converters to register them
 from .converter_registry import ConverterRegistry, converter, find_conversion_path
 from .dataset import Dataset, Sample
+from .export_import import export_dataset, import_dataset
 from .fields import (
     BBoxField,
     ImageCallableField,

src/datumaro/experimental/categories.py

@@ -15,7 +15,7 @@
 from dataclasses import dataclass, field
 from enum import IntEnum
 from functools import cache
-from typing import Dict, Iterator, List, NamedTuple, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Tuple, Union
 
 
 class LabelSemantic(IntEnum):
@@ -55,7 +55,50 @@ class Categories:
     label attributes etc.
     """
 
-    pass
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serialize this Categories instance to a JSON-compatible dictionary.
+
+        This default implementation should be overridden by subclasses that have
+        specific serialization needs. Returns just the type by default.
+
+        Returns:
+            Dictionary representation of this Categories instance
+        """
+        return {"type": self.__class__.__name__}
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Categories":
+        """
+        Deserialize a Categories instance from a JSON dictionary.
+
+        This method uses polymorphic dispatch to create the correct subclass
+        based on the "type" field in the dictionary.
+
+        Args:
+            data: Dictionary containing serialized Categories data
+
+        Returns:
+            Reconstructed Categories instance of the appropriate subclass
+        """
+        cat_type = data.get("type")
+
+        if not cat_type:
+            raise ValueError("Categories dictionary must have a 'type' field")
+
+        # Import subclasses to make them available
+        subclass_map = {
+            "LabelCategories": LabelCategories,
+            "HierarchicalLabelCategories": HierarchicalLabelCategories,
+            "MaskCategories": MaskCategories,
+        }
+
+        if cat_type in subclass_map:
+            return subclass_map[cat_type].from_dict(data)
+        else:
+            # Unknown type - return base Categories with just the type info
+            # This allows forward compatibility with new category types
+            raise ValueError(f"Unknown categories type: {cat_type}")
 
 
 @dataclass(frozen=True)
@@ -135,6 +178,49 @@ def __hash__(self):
         # Include label_semantics in the hash
         return hash((self.labels, self.group_type, frozenset(self.label_semantics.items())))
 
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serialize to a JSON-compatible dictionary.
+
+        Returns:
+            Dictionary representation of this LabelCategories instance
+        """
+        return {
+            "type": "LabelCategories",
+            "labels": list(self.labels),
+            "group_type": self.group_type.name,
+            "label_semantics": {
+                k.name if isinstance(k, LabelSemantic) else str(k): v
+                for k, v in self.label_semantics.items()
+            },
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "LabelCategories":
+        """
+        Deserialize from a JSON dictionary.
+
+        Args:
+            data: Dictionary containing serialized LabelCategories data
+
+        Returns:
+            Reconstructed LabelCategories instance
+        """
+        # Reconstruct label_semantics with proper LabelSemantic keys
+        label_semantics = {}
+        for k, v in data.get("label_semantics", {}).items():
+            try:
+                key = LabelSemantic[k]
+            except KeyError:
+                key = k
+            label_semantics[key] = v
+
+        return cls(
+            labels=tuple(data["labels"]),
+            group_type=GroupType[data["group_type"]],
+            label_semantics=label_semantics,
+        )
+
 
 @dataclass(frozen=True)
 class HierarchicalLabelCategory:
@@ -318,6 +404,96 @@ def __hash__(self):
         lg_repr = tuple((lg.name, tuple(lg.labels), lg.group_type) for lg in self.label_groups)
         return hash((self.items, lg_repr, frozenset(self.label_semantics.items())))
 
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serialize to a JSON-compatible dictionary.
+
+        Returns:
+            Dictionary representation of this HierarchicalLabelCategories instance
+        """
+        return {
+            "type": "HierarchicalLabelCategories",
+            "items": [
+                {
+                    "name": item.name,
+                    "parent": item.parent,
+                    "label_semantics": {
+                        k.name if isinstance(k, LabelSemantic) else str(k): v
+                        for k, v in item.label_semantics.items()
+                    },
+                }
+                for item in self.items
+            ],
+            "label_groups": [
+                {
+                    "name": group.name,
+                    "labels": list(group.labels),
+                    "group_type": group.group_type.name,
+                }
+                for group in self.label_groups
+            ],
+            "label_semantics": {
+                k.name if isinstance(k, LabelSemantic) else str(k): v
+                for k, v in self.label_semantics.items()
+            },
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "HierarchicalLabelCategories":
+        """
+        Deserialize from a JSON dictionary.
+
+        Args:
+            data: Dictionary containing serialized HierarchicalLabelCategories data
+
+        Returns:
+            Reconstructed HierarchicalLabelCategories instance
+        """
+        # Reconstruct items
+        items = []
+        for item_dict in data["items"]:
+            item_label_semantics = {}
+            for k, v in item_dict.get("label_semantics", {}).items():
+                try:
+                    key = LabelSemantic[k]
+                except KeyError:
+                    key = k
+                item_label_semantics[key] = v
+
+            items.append(
+                HierarchicalLabelCategory(
+                    name=item_dict["name"],
+                    parent=item_dict.get("parent", ""),
+                    label_semantics=item_label_semantics,
+                )
+            )
+
+        # Reconstruct label groups
+        label_groups = []
+        for group_dict in data.get("label_groups", []):
+            label_groups.append(
+                LabelGroup(
+                    name=group_dict["name"],
+                    labels=tuple(group_dict["labels"]),
+                    group_type=GroupType[group_dict["group_type"]],
+                )
+            )
+
+        # Reconstruct label_semantics
+        label_semantics = {}
+        for k, v in data.get("label_semantics", {}).items():
+            try:
+                key = LabelSemantic[k]
+            except KeyError:
+                key = k
+            label_semantics[key] = v
+
+        return cls(
+            items=tuple(items),
+            label_groups=tuple(label_groups),
+            label_semantics=label_semantics,
+        )
+
 
 class RgbColor(NamedTuple):
     """RGB color representation with named fields."""
@@ -386,6 +562,43 @@ class MaskCategories(Categories):
     def __hash__(self):
         return hash((tuple(self.labels), frozenset(self.colormap.items())))
 
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serialize to a JSON-compatible dictionary.
+
+        Returns:
+            Dictionary representation of this MaskCategories instance
+        """
+        return {
+            "type": "MaskCategories",
+            "labels": list(self.labels),
+            "colormap": {
+                str(idx): [color.r, color.g, color.b] for idx, color in self.colormap.data.items()
+            },
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "MaskCategories":
+        """
+        Deserialize from a JSON dictionary.
+
+        Args:
+            data: Dictionary containing serialized MaskCategories data
+
+        Returns:
+            Reconstructed MaskCategories instance
+        """
+        labels = data.get("labels", [])
+
+        # Reconstruct colormap
+        colormap_data = {}
+        for idx_str, color_list in data.get("colormap", {}).items():
+            idx = int(idx_str)
+            colormap_data[idx] = RgbColor(*color_list)
+
+        colormap = Colormap(data=colormap_data)
+        return cls(labels=labels, colormap=colormap)
+
     @classmethod
     def generate(cls, size: int = 255, include_background: bool = True) -> "MaskCategories":
         """