Automatically parse record set ids when they're not given.

PiperOrigin-RevId: 651096174

Author: fineguy

tensorflow_datasets/core/proto/dataset_info.proto

@@ -111,10 +111,6 @@ message DataSourceAccess {
   // URL referring to the data being used.
   // If provided together with a source, the url should correspond to the source
   // or part of the source.
-  // copybara:strip_begin
-  // If referring to a DataHub dataset, use the following format:
-  // http://data/details/mldataset.tfds.mnist
-  // copybara:strip_end
   Url url = 5;
 }
 

tensorflow_datasets/core/utils/croissant_utils.py

@@ -41,7 +41,7 @@ def get_tfds_dataset_name(dataset: mlc.Dataset) -> str:
   return huggingface_utils.convert_hf_name(dataset_name)
 
 
-def get_record_set_ids(metadata: mlc.Metadata) -> typing.Sequence[str]:
+def get_record_set_ids(metadata: mlc.Metadata) -> list[str]:
   """Returns record set ids of the given MLcroissant metadata.
 
   Record sets which have the attribute `cr:Data` are excluded (e.g. splits that

tensorflow_datasets/scripts/cli/croissant.py

@@ -106,6 +106,12 @@ def dataset(self) -> mlc.Dataset:
   def dataset_name(self) -> str:
     return croissant_utils.get_dataset_name(self.dataset)
 
+  @functools.cached_property
+  def record_set_ids(self) -> list[str]:
+    return self.record_sets or croissant_utils.get_record_set_ids(
+        self.dataset.metadata
+    )
+
 
 def register_subparser(parsers: argparse._SubParsersAction):
   """Add subparser for `convert_format` command."""
@@ -133,7 +139,7 @@ def prepare_croissant_builder(args: CmdArgs) -> None:
   """
   builder = croissant_builder.CroissantBuilder(
       jsonld=args.jsonld,
-      record_set_ids=args.record_sets or None,
+      record_set_ids=args.record_set_ids,
       file_format=args.file_format,
       data_dir=args.data_dir,
       mapping=args.mapping_json,