chore: always store the cdc parameters as metadata

Author: kszucs

src/datasets/arrow_writer.py

@@ -690,6 +690,7 @@ def _build_writer(self, inferred_schema: pa.Schema):
         self.pa_writer = pq.ParquetWriter(
             self.stream, self._schema, use_content_defined_chunking=self.use_content_defined_chunking
         )
-        self.pa_writer.add_key_value_metadata(
-            {"content_defined_chunking": json.dumps(self.use_content_defined_chunking)}
-        )
+        if self.use_content_defined_chunking is not False:
+            self.pa_writer.add_key_value_metadata(
+                {"content_defined_chunking": json.dumps(self.use_content_defined_chunking)}
+            )

src/datasets/io/parquet.py

@@ -91,7 +91,9 @@ def __init__(
     def write(self) -> int:
         batch_size = self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE
         use_content_defined_chunking = (
-            self.use_content_defined_chunking if self.use_content_defined_chunking else config.DEFAULT_CDC_OPTIONS
+            config.DEFAULT_CDC_OPTIONS
+            if self.use_content_defined_chunking is None
+            else self.use_content_defined_chunking
         )
 
         if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
@@ -140,6 +142,8 @@ def _write(
             written += batch.nbytes
 
         # TODO(kszucs): we may want to persist multiple parameters
-        writer.add_key_value_metadata({"content_defined_chunking": json.dumps(use_content_defined_chunking)})
+        if use_content_defined_chunking is not False:
+            writer.add_key_value_metadata({"content_defined_chunking": json.dumps(use_content_defined_chunking)})
+
         writer.close()
         return written