test=release/1.4, fix test_impertative_transformer (#16708)

Author: JiabinYang

python/paddle/fluid/tests/unittests/test_imperative_transformer.py

@@ -302,11 +302,7 @@ def make_all_inputs(input_fields):
 # if we run sync mode
 sync = False
 
-if not core.is_compiled_with_cuda():
-    # how many batches we use
-    batch_num = 50
-else:
-    batch_num = 5
+batch_num = 5
 
 np.random.seed = 1
 src_word_np = np.random.randint(