Bump SDK (#783)

* update

* update

Author: tchaton

.pre-commit-config.yaml

@@ -85,7 +85,7 @@ repos:
         args: ["--print-width=120"]
 
   - repo: https://github.com/tox-dev/pyproject-fmt
-    rev: v2.11.1
+    rev: v2.8.0
     hooks:
       - id: pyproject-fmt
         additional_dependencies: [tox]

requirements/extras.txt

@@ -2,7 +2,7 @@ pillow
 viztracer
 pyarrow
 tqdm
-lightning-sdk==2025.9.30 # Must be pinned to ensure compatibility
+lightning-sdk==2025.12.17 # Must be pinned to ensure compatibility
 google-cloud-storage
 polars
 fsspec

src/litdata/processing/data_processor.py

@@ -1440,12 +1440,12 @@ def run(self, data_recipe: DataRecipe) -> None:
                 storage_dir=self.output_dir.path,
                 dataset_type=data_type,
                 empty=False,
-                size=result.size,
+                size=str(result.size),
                 num_bytes=result.num_bytes,
                 data_format=result.data_format,
                 compression=result.compression,
-                num_chunks=result.num_chunks,
-                num_bytes_per_chunk=result.num_bytes_per_chunk,
+                num_chunks=str(result.num_chunks),
+                num_bytes_per_chunk=[str(v) for v in (result.num_bytes_per_chunk or [])],
             )
         if self.verbose:
             print("Finished data processing!")

src/litdata/processing/utilities.py

@@ -33,14 +33,14 @@ def _create_dataset(
     storage_dir: str,
     dataset_type: Any,
     empty: bool | None = None,
-    size: int | None = None,
+    size: str | None = None,
     num_bytes: str | None = None,
     data_format: str | tuple[str] | None = None,
     compression: str | None = None,
-    num_chunks: int | None = None,
-    num_bytes_per_chunk: list[int] | None = None,
+    num_chunks: str | None = None,
+    num_bytes_per_chunk: list[str] | None = None,
     name: str | None = None,
-    version: int | None = None,
+    version: str | None = None,
 ) -> None:
     """Create a dataset with metadata information about its source and destination using the Lightning SDK.
 
@@ -61,31 +61,31 @@ def _create_dataset(
     if not storage_dir:
         raise ValueError("The storage_dir should be defined.")
 
-    from lightning_sdk.lightning_cloud.openapi import ProjectIdDatasetsBody
+    from lightning_sdk.lightning_cloud.openapi.models import DatasetServiceCreateDatasetBody
     from lightning_sdk.lightning_cloud.openapi.rest import ApiException
     from lightning_sdk.lightning_cloud.rest_client import LightningClient
 
     client = LightningClient(retry=False)
 
     try:
         client.dataset_service_create_dataset(
-            body=ProjectIdDatasetsBody(
-                cloud_space_id=studio_id if lightning_app_id is None else None,
-                cluster_id=cluster_id,
-                creator_id=user_id,
-                empty=empty,
-                input_dir=input_dir,
-                lightning_app_id=lightning_app_id,
-                name=name,
-                size=size,
-                num_bytes=num_bytes,
-                data_format=str(data_format) if data_format else data_format,
-                compression=compression,
-                num_chunks=num_chunks,
-                num_bytes_per_chunk=num_bytes_per_chunk,
+            body=DatasetServiceCreateDatasetBody(
+                cloud_space_id=(studio_id if lightning_app_id is None else None) or "",
+                cluster_id=cluster_id or "",
+                creator_id=user_id or "",
+                empty=empty or True,
+                input_dir=input_dir or "",
+                lightning_app_id=lightning_app_id or "",
+                name=name or "",
+                size=size or "",
+                num_bytes=num_bytes or "",
+                data_format=(str(data_format) if data_format else data_format) or "",
+                compression=compression or "",
+                num_chunks=num_chunks or "",
+                num_bytes_per_chunk=num_bytes_per_chunk or [],
                 storage_dir=storage_dir,
                 type=dataset_type,
-                version=version,
+                version=version or "",
             ),
             project_id=project_id,
         )