test_py_reader_using_executor support test use_decorate_paddle_reader

Author: jacquesqiao

python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py

@@ -58,19 +58,19 @@ def simple_fc_net(in_size,
     if use_feed_list:
         data = fluid.layers.data(name="data", dtype='float32', shape=[in_size])
         label = fluid.layers.data(name='label', dtype='int64', shape=[1])
-        reader = fluid.layers.create_py_reader_by_data(
+        py_reader = fluid.layers.create_py_reader_by_data(
             capacity=queue_capacity,
             use_double_buffer=False,
             feed_list=[data, label])
     else:
-        reader = fluid.layers.py_reader(
+        py_reader = fluid.layers.py_reader(
             capacity=queue_capacity,
             shapes=[[-1, in_size], [-1, 1]],
             lod_levels=[0, 0],
             dtypes=['float32', 'int64'],
             use_double_buffer=False)
-    feed_queue = reader.queue
-    reader = fluid.layers.batch(reader, batch_size=batch_size)
+    feed_queue = py_reader.queue
+    reader = fluid.layers.batch(py_reader, batch_size=batch_size)
     if use_double_buffer:
         reader = fluid.layers.double_buffer(reader)
 
@@ -92,7 +92,7 @@ def simple_fc_net(in_size,
 
     optimizer = fluid.optimizer.Adam()
     optimizer.minimize(loss)
-    return in_data, label, loss, optimizer, feed_queue
+    return in_data, label, loss, optimizer, feed_queue, py_reader
 
 
 class TestPyReaderUsingExecutor(unittest.TestCase):
@@ -110,17 +110,21 @@ def test(self):
             for use_parallel_executor in [False, True]:
                 for use_double_buffer in [False, True]:
                     for use_feed_list in [False, True]:
-                        print('Test Parameters:'),
-                        print({
-                            'use_cuda': use_cuda,
-                            'use_parallel_executor': use_parallel_executor,
-                            'use_double_buffer': use_double_buffer,
-                            'use_feed_list': use_feed_list
-                        })
-                        self.main(use_cuda, use_parallel_executor,
-                                  use_double_buffer, use_feed_list)
-
-    def random_reader(self):
+                        for use_decorate_paddle_reader in [False, True]:
+                            print('Test Parameters:'),
+                            print({
+                                'use_cuda': use_cuda,
+                                'use_parallel_executor': use_parallel_executor,
+                                'use_double_buffer': use_double_buffer,
+                                'use_feed_list': use_feed_list,
+                                'use_decorate_paddle_reader':
+                                use_decorate_paddle_reader
+                            })
+                            self.main(use_cuda, use_parallel_executor,
+                                      use_double_buffer, use_feed_list,
+                                      use_decorate_paddle_reader)
+
+    def tensor_reader(self, use_decorate_paddle_reader):
         def reader():
             self.inputs = []
             cnt = 0
@@ -144,30 +148,36 @@ def reader():
                 elif not self.use_double_buffer:
                     break
 
-                yield tensors
+                if use_decorate_paddle_reader:
+                    yield [(in_data, label)]
+                else:
+                    yield tensors
                 cnt += 1
 
-            yield None
+            if not use_decorate_paddle_reader:
+                yield None
 
         return reader
 
     def main(self,
              use_cuda=True,
              use_parallel_executor=False,
              use_double_buffer=False,
-             use_feed_list=False):
+             use_feed_list=False,
+             use_decorate_paddle_reader=False):
         assert not use_cuda or use_cuda and core.is_compiled_with_cuda()
 
         self.use_cuda = use_cuda
         self.use_parallel_executor = use_parallel_executor
         self.use_double_buffer = use_double_buffer
         self.use_feed_list = use_feed_list
+        self.use_decorate_paddle_reader = use_decorate_paddle_reader
 
         startup_program = fluid.Program()
         main_program = fluid.Program()
 
         with fluid.program_guard(main_program, startup_program):
-            in_data, label, loss, optimizer, feed_queue = simple_fc_net(
+            in_data, label, loss, optimizer, feed_queue, py_reader = simple_fc_net(
                 in_size=self.in_size,
                 class_num=self.class_num,
                 hidden_sizes=self.hidden_sizes,
@@ -192,10 +202,14 @@ def main(self,
                 main_exe = startup_exe
                 self.batch_size_times = 1
 
-            reader = self.random_reader()
-            thread = threading.Thread(
-                target=feed_data, args=(feed_queue, reader))
-            thread.start()
+            reader = self.tensor_reader(use_decorate_paddle_reader)
+            if use_decorate_paddle_reader:
+                py_reader.decorate_paddle_reader(reader)
+                py_reader.start()
+            else:
+                thread = threading.Thread(
+                    target=feed_data, args=(feed_queue, reader))
+                thread.start()
 
             self.outputs = []
             for _ in range(self.iterations):