+ add trust_remote_code=True to all load_dataset invoking

Author: HYLcool

trinity/buffer/reader/file_reader.py

@@ -66,7 +66,7 @@ def __init__(self, meta: StorageConfig, config: BufferConfig):
         self.response_key = meta.format.response_key
         self.read_batch_size = config.read_batch_size
         self.dataset = _HFBatchReader(
-            load_dataset(meta.path, name=subset_name, split=self.split)
+            load_dataset(meta.path, name=subset_name, split=self.split, trust_remote_code=True)
         )  # TODO: support resume
         self.data_iter = self.dataset.iter(self.read_batch_size, drop_last_batch=True)
         self.tokenizer = transformers.AutoTokenizer.from_pretrained(config.tokenizer_path)
@@ -143,7 +143,7 @@ def __init__(self, meta: StorageConfig, config: BufferConfig):
         self.rejected_key = meta.format.rejected_key
         self.read_batch_size = config.read_batch_size
         self.dataset = _HFBatchReader(
-            load_dataset(meta.path, name=subset_name, split=self.split)
+            load_dataset(meta.path, name=subset_name, split=self.split, trust_remote_code=True)
         )  # TODO: support resume
         self.data_iter = self.dataset.iter(self.read_batch_size, drop_last_batch=True)
         self.tokenizer = transformers.AutoTokenizer.from_pretrained(config.tokenizer_path)
@@ -215,7 +215,7 @@ def __init__(self, meta: StorageConfig, config: BufferConfig):
         self.epoch = 0
         datasets.disable_caching()
         self.dataset = _HFBatchReader(
-            load_dataset(meta.path, name=subset_name, split=self.split),
+            load_dataset(meta.path, name=subset_name, split=self.split, trust_remote_code=True),
             max_epoch=self.meta.total_epochs if meta.task_type == TaskType.EXPLORE else 1,
             offset=self.meta.index,
         )
@@ -266,7 +266,7 @@ def read(
 class RawDataReader(BufferReader):
     def __init__(self, meta: StorageConfig, config: Optional[BufferConfig]):
         self.returned = False
-        self.dataset = load_dataset(meta.path, name=meta.subset_name, split=meta.split)
+        self.dataset = load_dataset(meta.path, name=meta.subset_name, split=meta.split, trust_remote_code=True)
 
     def __len__(self):
         return len(self.dataset)