added changes to support mmapped file.

Author: hariharan-devarajan

dlio_benchmark/configs/workload/megatron_deepspeed.yaml

@@ -12,7 +12,7 @@ dataset:
   format: mmap_indexed_binary
   num_files_train: 1
   num_samples_per_file: 277203535
-  record_length: 4096
+  record_length: 1024
 
 reader: 
   data_loader: pytorch

dlio_benchmark/data_generator/indexed_binary_generator.py

@@ -55,7 +55,7 @@ def generate(self):
             sample_size = dim1 * dim2
             total_size = sample_size * self.num_samples
             write_size = total_size
-            MEMORY_SIZE = 2*GB
+            MEMORY_SIZE = 16*GB
             if total_size > MEMORY_SIZE:
                 write_size = MEMORY_SIZE - (MEMORY_SIZE % sample_size)
             out_path_spec = self.storage.get_uri(self._file_list[i])