Run Python OP tests in a single Python process to improve test time. (#8362)

Currently, our tests run with 2 GPUs, the init time is absurdly long:
about 4s for each process.  Currently, we run each OP test on
different processes. This PR:

1. create cmake function py_test_modules which will generate the
Makefile that runs a list of Python unittest module in a single Python
process.

2. move all "python unittest compatible" (e.g., used the unittest
package, not just a regular python file). from fluid/tests to
fluid/tests/unittests.

3. cmake now will run all OP tests in fluid/tests/unittests in a
single process, except the time-consuming tests, they are separated
into different processes to utilize parallelism. Please make sure to
use the unittest package if you put the python test file in
fluid/tests/unittests

4. remove all exit(0) from fluid/tests/unittests/*.py, exit(0) is used
to disable unittest, we can not do it when running all tests in a
single process since it will terminate the process without running the
other tests. Instead, the test is disabled in
fluid/tests/unittests/CMakeLists.txt. FIXME is added for each disabled
item. Please disable the unittest from
fluid/tests/unittests/CMakeLists.txt, instead of adding exit(0) to the
Python file, for all Python file in fluid/tests/unittests/.

5. add an option WITH_FAST_BUNDLE_TEST. When OFF, will run the unit
tests in separate process so that they can be tested individually.

Author: panyx0718

CMakeLists.txt

@@ -60,6 +60,7 @@ option(USE_NNPACK       "Compile PaddlePaddle with NNPACK library"      OFF)
 option(WITH_DISTRIBUTE  "Compile with grpc distributed support"         OFF)
 option(USE_EIGEN_FOR_BLAS   "Use matrix multiplication in Eigen"        OFF)
 option(WITH_ARM_FP16    "Use half precision support on armv8.2-a cpu"   OFF)
+option(WITH_FAST_BUNDLE_TEST    "Bundle tests that can be run in a single process together to reduce launch overhead"   ON)
 
 # CMAKE_BUILD_TYPE
 if(NOT CMAKE_BUILD_TYPE)

python/paddle/v2/fluid/tests/CMakeLists.txt

@@ -1,16 +1,11 @@
 file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
 string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
 
-if(NOT WITH_DISTRIBUTE)
-    list(REMOVE_ITEM TEST_OPS test_recv_op)
-endif(NOT WITH_DISTRIBUTE)
-
-list(REMOVE_ITEM TEST_OPS test_warpctc_op)
 foreach(src ${TEST_OPS})
-    py_test(${src} SRCS ${src}.py)
+  py_test(${src} SRCS ${src}.py)
 endforeach()
-py_test(test_warpctc_op SRCS test_warpctc_op.py ENVS FLAGS_warpctc_dir=${WARPCTC_LIB_DIR})
 
+add_subdirectory(unittests)
 add_subdirectory(book)
 add_subdirectory(book_distribute)
 add_subdirectory(book_memory_optimization)

python/paddle/v2/fluid/tests/unittests/CMakeLists.txt

@@ -0,0 +1,89 @@
+file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
+string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
+
+if(NOT WITH_DISTRIBUTE)
+    list(REMOVE_ITEM TEST_OPS test_recv_op)
+endif(NOT WITH_DISTRIBUTE)
+
+list(REMOVE_ITEM TEST_OPS test_seq_concat_op) # FIXME(helin): https://github.com/PaddlePaddle/Paddle/issues/8290
+list(REMOVE_ITEM TEST_OPS test_modified_huber_loss_op) # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5184 
+list(REMOVE_ITEM TEST_OPS test_lstm_unit_op) # # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5185
+list(REMOVE_ITEM TEST_OPS test_nce) # IXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/7778
+list(REMOVE_ITEM TEST_OPS test_recurrent_op) # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/6152
+list(REMOVE_ITEM TEST_OPS test_cond_op) # FIXME(qijun): https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957
+list(REMOVE_ITEM TEST_OPS test_detection_output_op) # FIXME: detection_output_op will be rewritten. This unittest should be
+
+list(REMOVE_ITEM TEST_OPS op_test) # op_test is a helper python file, not a test
+list(REMOVE_ITEM TEST_OPS decorators) # decorators is a helper python file, not a test
+
+function(py_test_modules TARGET_NAME)
+  if(WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs MODULES DEPS ARGS ENVS)
+    cmake_parse_arguments(py_test_modules "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    add_test(NAME ${TARGET_NAME}
+             COMMAND env PYTHONPATH=${PADDLE_PYTHON_BUILD_DIR}/lib-python ${py_test_modules_ENVS}
+             ${PYTHON_EXECUTABLE} -u -m unittest --verbose ${py_test_modules_MODULES} ${py_test_modules_ARGS}
+             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+endfunction()
+
+# test time consuming OPs in a separate process for expliot parallism
+list(REMOVE_ITEM TEST_OPS test_warpctc_op)
+list(REMOVE_ITEM TEST_OPS test_dyn_rnn)
+list(REMOVE_ITEM TEST_OPS test_mul_op)
+
+# tests that need to be run in separate process.
+list(REMOVE_ITEM TEST_OPS test_multihead_attention)
+list(REMOVE_ITEM TEST_OPS test_calc_gradient)
+list(REMOVE_ITEM TEST_OPS test_while_op)
+list(REMOVE_ITEM TEST_OPS test_lod_array_length_op)
+list(REMOVE_ITEM TEST_OPS test_reorder_lod_tensor)
+list(REMOVE_ITEM TEST_OPS test_profiler)
+list(REMOVE_ITEM TEST_OPS test_normalization_wrapper)
+list(REMOVE_ITEM TEST_OPS test_executor_and_mul)
+list(REMOVE_ITEM TEST_OPS test_assign_value_op)
+list(REMOVE_ITEM TEST_OPS test_array_read_write_op)
+list(REMOVE_ITEM TEST_OPS test_lod_rank_table)
+list(REMOVE_ITEM TEST_OPS test_weight_normalization)
+list(REMOVE_ITEM TEST_OPS test_conditional_block)
+list(REMOVE_ITEM TEST_OPS test_parameter)
+list(REMOVE_ITEM TEST_OPS test_registry)
+list(REMOVE_ITEM TEST_OPS test_fetch_var)
+list(REMOVE_ITEM TEST_OPS test_parallel_op)
+list(REMOVE_ITEM TEST_OPS test_dynrnn_static_input)
+
+# tests that can be bundled together in one python process for speed.
+if(WITH_FAST_BUNDLE_TEST)
+    py_test_modules("test_all_ops" MODULES ${TEST_OPS})
+else()
+    foreach(TEST_OP ${TEST_OPS})
+        py_test_modules(${TEST_OP} MODULES ${TEST_OP})
+    endforeach(TEST_OP)
+endif(WITH_FAST_BUNDLE_TEST)
+
+# tests with high overhead
+py_test_modules(test_warpctc_op MODULES test_warpctc_op ENVS FLAGS_warpctc_dir=${WARPCTC_LIB_DIR})
+py_test_modules(test_train_dyn_rnn MODULES test_dyn_rnn)
+py_test_modules(test_mul_op MODULES test_mul_op)
+
+# tests that need to be run in separate process.
+py_test_modules(test_multihead_attention MODULES test_multihead_attention)
+py_test_modules(test_calc_gradient MODULES test_calc_gradient)
+py_test_modules(test_while_op MODULES test_while_op)
+py_test_modules(test_lod_array_length_op MODULES test_lod_array_length_op)
+py_test_modules(test_reorder_lod_tensor MODULES test_reorder_lod_tensor)
+py_test_modules(test_profiler MODULES test_profiler)
+py_test_modules(test_normalization_wrapper MODULES test_normalization_wrapper)
+py_test_modules(test_executor_and_mul MODULES test_executor_and_mul)
+py_test_modules(test_assign_value_op MODULES test_assign_value_op)
+py_test_modules(test_array_read_write_op MODULES test_array_read_write_op)
+py_test_modules(test_lod_rank_table MODULES test_lod_rank_table)
+py_test_modules(test_weight_normalization MODULES test_weight_normalization)
+py_test_modules(test_conditional_block MODULES test_conditional_block)
+py_test_modules(test_parameter MODULES test_parameter)
+py_test_modules(test_registry MODULES test_registry)
+py_test_modules(test_fetch_var MODULES test_fetch_var)
+py_test_modules(test_dynrnn_static_input MODULES test_dynrnn_static_input)
+py_test_modules(test_parallel_op MODULES test_parallel_op)

python/paddle/v2/fluid/tests/unittests/__init__.py

@@ -0,0 +1,13 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.