diff --git a/llmc/data/dataset/base_dataset.py b/llmc/data/dataset/base_dataset.py index 306f373aa..f4c79e69f 100644 --- a/llmc/data/dataset/base_dataset.py +++ b/llmc/data/dataset/base_dataset.py @@ -255,7 +255,6 @@ def get_calib_dataset(self): samples = self.get_calib_samples() if self.calib_dataset_type in ['txt', 'img', 'img_txt']: logger.info(f'len(samples) all : {len(samples)}') - assert len(samples) % int(os.environ['WORLD_SIZE']) == 0 samples = samples[int(os.environ['RANK'])::int(os.environ['WORLD_SIZE'])] logger.info(f'len(samples) rank : {len(samples)}') calib_samples = []