Merge pull request #701 from wangzhen38/del_batch_error

update error info when batchSize > datasetSize

Author: frankwhzhang

tools/infer.py

@@ -116,6 +116,10 @@ def main(args):
         infer_run_cost = 0.0
         reader_start = time.time()
 
+        #we will drop the last incomplete batch when dataset size is not divisible by the batch size
+        assert any(test_dataloader(
+        )), "test_dataloader is null, please ensure batch size < dataset size!"
+
         for batch_id, batch in enumerate(test_dataloader()):
             infer_reader_cost += time.time() - reader_start
             infer_start = time.time()

tools/static_infer.py

@@ -128,6 +128,11 @@ def main(args):
 
         if use_auc:
             reset_auc(use_fleet, auc_num)
+
+        #we will drop the last incomplete batch when dataset size is not divisible by the batch size
+        assert any(test_dataloader(
+        )), "test_dataloader's size is null, please ensure batch size < dataset size!"
+
         for batch_id, batch_data in enumerate(test_dataloader()):
             infer_reader_cost += time.time() - reader_start
             infer_start = time.time()

tools/static_trainer.py

@@ -239,6 +239,11 @@ def dataloader_train(epoch_id, train_dataloader, input_data_names, fetch_vars,
     train_run_cost = 0.0
     total_samples = 0
     reader_start = time.time()
+
+    #we will drop the last incomplete batch when dataset size is not divisible by the batch size
+    assert any(train_dataloader(
+    )), "train_dataloader's size is null, please ensure batch size < dataset size!"
+
     for batch_id, batch_data in enumerate(train_dataloader()):
         train_reader_cost += time.time() - reader_start
         train_start = time.time()

tools/trainer.py

@@ -127,6 +127,10 @@ def main(args):
         total_samples = 0
         reader_start = time.time()
 
+        #we will drop the last incomplete batch when dataset size is not divisible by the batch size
+        assert any(train_dataloader(
+        )), "train_dataloader is null, please ensure batch size < dataset size!"
+
         for batch_id, batch in enumerate(train_dataloader()):
             train_reader_cost += time.time() - reader_start
             optimizer.clear_grad()