update ddp (#195)

Author: helloyongyang

llmc/data/dataset/base_dataset.py

@@ -255,7 +255,6 @@ def get_calib_dataset(self):
         samples = self.get_calib_samples()
         if self.calib_dataset_type in ['txt', 'img', 'img_txt']:
             logger.info(f'len(samples) all : {len(samples)}')
-            assert len(samples) % int(os.environ['WORLD_SIZE']) == 0
             samples = samples[int(os.environ['RANK'])::int(os.environ['WORLD_SIZE'])]
             logger.info(f'len(samples) rank : {len(samples)}')
         calib_samples = []