Skip to content

Commit 875b468

Browse files
ligurioBuristan
ligurio
authored and
Buristan
committed
cmake: support TAP tests by CTest
CTest support has been added in commit a9b9f2e ("test: execute tests using CTest"), however, `test-run.py` was used for running TAP tests. The patch adds support of running TAP tests (suites app-tap, box-tap, engine-tap and sql-tap) by CTest directly, without using test-run.py. Note, that the following TAP tests were rewritten because these tests were used .result files for execution, see [1]: - box-tap/trigger_yield.test.lua -> box-luatest/trigger_yield_test.lua - app-tap/tap.test.lua -> app-luatest/tap_test.lua - app-tap/logger_pipe.test.lua -> app-luatest/logger_pipe_test.lua - app-tap/init_script.test.lua -> app-luatest/init_script_test.lua - app-tap/pcall.test.lua -> app-luatest/pcall_test.lua - app-tap/debug.test.lua -> app-luatest/debug_test.lua - box-tap/trigger_atexit.test.lua -> box-luatest/trigger_atexit_test.lua - box-tap/tarantoolgh-5602-environment-vars-cfg.test.lua -> box-luatest/tarantoolgh-5602-environment-vars-cfg_test.lua And the following TAP tests were rewritten because these tests required inspector functionality builtin into test-run.py, see [2]: - box-tap/session.storage.test.lua -> box-luatest/session_storage_test.lua - box-tap/net.box.test.lua -> box-luatest/net_box_test.lua - box-tap/session.test.lua -> test/box-luatest/session_test.lua Note, that TAP test suites use environment variable `MEMTX_ALLOCATOR`. Support from the test executor is not required for using this parameter in tests. Note, that `tarantool` was not added to "*-deps", because running target `box_generate_lua_sources`, that builds Lua source files in parallel corrupts the generated Lua files. Follows up tarantool#10216 1. tarantool#10725 2. tarantool#11021 NO_CHANGELOG=testing NO_DOC=testing NO_TEST=testing
1 parent c79715a commit 875b468

File tree

9 files changed

+340
-2
lines changed

9 files changed

+340
-2
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ perf/output/
4747
test/Makefile
4848
test/*/Makefile
4949
test/*/*/Makefile
50+
test/*/*-workdir-*
5051
Doxyfile.API
5152
RPM
5253
*.src.rpm

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ find_program(LD ld)
2929
find_program(CTAGS ctags)
3030
find_program(LUACHECK luacheck ENV PATH)
3131
find_program(PYTHON python)
32+
find_program(RM rm)
3233

3334
include (CTest)
3435
enable_testing()

‎test/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ add_subdirectory(box-tap)
8080
add_subdirectory(box-luatest)
8181
add_subdirectory(config-luatest)
8282
add_subdirectory(engine-luatest)
83+
add_subdirectory(engine-tap)
8384
add_subdirectory(metrics-luatest)
8485
add_subdirectory(replication-luatest)
8586
add_subdirectory(sql-tap)
@@ -167,12 +168,16 @@ add_custom_target(test-force-ctest
167168

168169
list(APPEND EXECUTED_WITHOUT_TEST_RUN
169170
app-luatest
171+
app-tap
170172
box-luatest
173+
box-tap
171174
config-luatest
172175
engine-luatest
176+
engine-tap
173177
metrics-luatest
174178
replication-luatest
175179
sql-luatest
180+
sql-tap
176181
vinyl-luatest
177182
)
178183

‎test/app-tap/CMakeLists.txt‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,86 @@ target_link_libraries(module_api msgpuck)
66
set(CMAKE_C_FLAGS "-Wall -Wextra -std=c99")
77
set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Werror")
88
set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2")
9+
10+
tarantool_make_lua_path(LUA_PATH
11+
PATHS
12+
${CMAKE_CURRENT_SOURCE_DIR}/?.lua
13+
${CMAKE_CURRENT_SOURCE_DIR}/lua/?.lua
14+
# Needed for test-run/test_run.lua.
15+
${PROJECT_SOURCE_DIR}/test-run/?.lua
16+
)
17+
18+
tarantool_make_lua_path(LUA_CPATH
19+
PATHS
20+
${CMAKE_CURRENT_BINARY_DIR}/?${CMAKE_SHARED_LIBRARY_SUFFIX}
21+
)
22+
23+
set(TEST_SUITE_NAME "app-tap")
24+
25+
message(STATUS "Add test suite ${TEST_SUITE_NAME}")
26+
27+
# XXX: The call produces both test and target <app-tap-deps>
28+
# as a side effect.
29+
_add_test_suite_target(${TEST_SUITE_NAME}
30+
LABELS ${TEST_SUITE_NAME}
31+
DEPENDS module_api
32+
)
33+
34+
list(APPEND TEST_ENV
35+
"LUA_PATH=${LUA_PATH}"
36+
"LUA_CPATH=${LUA_CPATH}"
37+
)
38+
39+
file(GLOB tests ${CMAKE_CURRENT_SOURCE_DIR} *.test.lua)
40+
foreach(test_path ${tests})
41+
get_filename_component(test_name ${test_path} NAME)
42+
# FIXME: By default, GLOB lists directories.
43+
# Directories are omitted in the result if LIST_DIRECTORIES
44+
# is set to false. New in version CMake 3.3.
45+
if(${test_name} STREQUAL ${TEST_SUITE_NAME})
46+
continue()
47+
endif()
48+
set(TEST_TITLE "test/${TEST_SUITE_NAME}/${test_name}")
49+
set(WORK_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test_name}-workdir")
50+
file(MAKE_DIRECTORY ${WORK_DIR})
51+
set(TEST_CMD "${RM} -rf ${WORK_DIR}/* && ${TARANTOOL_BIN} ${test_path}")
52+
add_test(NAME ${TEST_TITLE}
53+
COMMAND ${BASH} -c "${TEST_CMD}"
54+
WORKING_DIRECTORY ${WORK_DIR}
55+
)
56+
set(TEST_ENV_VARS "")
57+
if(NOT ${TEST_TITLE} STREQUAL test/app-tap/logger.test.lua)
58+
list(APPEND TEST_ENV_VARS TT_WORK_DIR=${WORK_DIR})
59+
endif()
60+
set_tests_properties(${TEST_TITLE} PROPERTIES
61+
ENVIRONMENT "${TEST_ENV};${TEST_ENV_VARS}"
62+
LABELS "${TEST_SUITE_NAME};regression"
63+
DEPENDS ${TEST_SUITE_NAME}-deps
64+
FAIL_REGULAR_EXPRESSION "${CTEST_FAIL_REGEXP}"
65+
)
66+
endforeach()
67+
68+
# See test/app-tap/suite.ini.
69+
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
70+
set_tests_properties(
71+
test/app-tap/gh-2717-no-quit-sigint.test.lua
72+
PROPERTIES DISABLED TRUE
73+
)
74+
endif()
75+
76+
# See console.skipcond, disabled on FreeBSD due to fail #4271.
77+
if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
78+
set_tests_properties(
79+
test/app-tap/console.test.lua
80+
PROPERTIES DISABLED TRUE
81+
)
82+
endif()
83+
84+
# See fail_main.skipcond and popen.skipcond, disabled on OpenBSD.
85+
if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
86+
set_tests_properties(
87+
test/app-tap/fail_main.test.lua
88+
test/app-tap/popen.test.lua
89+
PROPERTIES DISABLED TRUE
90+
)
91+
endif()

‎test/box-tap/CMakeLists.txt‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,74 @@
11
build_module(check_merge_source check_merge_source.c)
2+
3+
set(TEST_SUITE_NAME "box-tap")
4+
5+
message(STATUS "Add test suite ${TEST_SUITE_NAME}")
6+
7+
tarantool_make_lua_path(LUA_PATH
8+
PATHS
9+
${PROJECT_SOURCE_DIR}/test-run/?.lua
10+
)
11+
12+
# XXX: The call produces both test and target <box-tap-deps>
13+
# as a side effect.
14+
_add_test_suite_target(${TEST_SUITE_NAME}
15+
LABELS ${TEST_SUITE_NAME}
16+
DEPENDS check_merge_source
17+
)
18+
19+
list(APPEND TEST_ENV
20+
"BUILDDIR=${PROJECT_BINARY_DIR}"
21+
"LUA_PATH=${LUA_PATH}"
22+
)
23+
24+
file(GLOB tests ${CMAKE_CURRENT_SOURCE_DIR} *.test.lua)
25+
foreach(test_path ${tests})
26+
get_filename_component(test_name ${test_path} NAME)
27+
# FIXME: By default, GLOB lists directories.
28+
# Directories are omitted in the result if LIST_DIRECTORIES
29+
# is set to false. New in version CMake 3.3.
30+
if(${test_name} STREQUAL ${TEST_SUITE_NAME})
31+
continue()
32+
endif()
33+
set(TEST_TITLE "test/${TEST_SUITE_NAME}/${test_name}")
34+
set(WORK_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test_name}-workdir")
35+
file(MAKE_DIRECTORY ${WORK_DIR})
36+
set(TEST_CMD "${RM} -rf ${WORK_DIR}/* && ${TARANTOOL_BIN} ${test_path}")
37+
add_test(NAME ${TEST_TITLE}
38+
COMMAND ${BASH} -c "${TEST_CMD}"
39+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
40+
)
41+
# `LISTEN` is required by the following tests:
42+
# - test/box-tap/auth.test.lua
43+
# - test/box-tap/extended_error.test.lua
44+
set(TEST_ENV_VARS "${TEST_ENV};LISTEN=${WORK_DIR}/listen.sock;")
45+
# The tests below could be broken by setting environment
46+
# variable `TT_WORK_DIR`, so `TT_WORK_DIR` is not set.
47+
if(NOT ${TEST_TITLE} STREQUAL test/box-tap/cfg.test.lua AND
48+
NOT ${TEST_TITLE} STREQUAL test/box-tap/gh-4562-errno-at-xdir_scan.test.lua)
49+
set(TEST_ENV_VARS ${TEST_ENV_VARS};TT_WORK_DIR=${WORK_DIR};)
50+
endif()
51+
set_tests_properties(${TEST_TITLE} PROPERTIES
52+
ENVIRONMENT "${TEST_ENV_VARS}"
53+
LABELS "${TEST_SUITE_NAME};regression"
54+
DEPENDS ${TEST_SUITE_NAME}-deps
55+
FAIL_REGULAR_EXPRESSION "${CTEST_FAIL_REGEXP}"
56+
)
57+
endforeach()
58+
59+
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
60+
set_tests_properties(
61+
# See test/box-tap/suite.ini.
62+
test/box-tap/errinj_set_with_enviroment_vars.test.lua
63+
test/box-tap/gh-6198-max-cnt-of-tuple-fields-insert-overflow-err-msg.test.lua
64+
PROPERTIES DISABLED TRUE
65+
)
66+
endif()
67+
68+
# See cfg.skipcond, disabled on OpenBSD.
69+
if(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
70+
set_tests_properties(
71+
test/box-tap/cfg.test.lua
72+
PROPERTIES DISABLED TRUE
73+
)
74+
endif()

‎test/engine-tap/CMakeLists.txt‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
tarantool_make_lua_path(LUA_PATH
2+
PATHS
3+
${CMAKE_CURRENT_SOURCE_DIR}/lib/?.lua
4+
)
5+
6+
set(TEST_SUITE_NAME "engine-tap")
7+
8+
message(STATUS "Add test suite ${TEST_SUITE_NAME}")
9+
10+
# XXX: The call produces both test and target <engine-tap-deps>
11+
# as a side effect.
12+
_add_test_suite_target(${TEST_SUITE_NAME}
13+
LABELS ${TEST_SUITE_NAME}
14+
)
15+
16+
list(APPEND TEST_ENV
17+
"LUA_PATH=${LUA_PATH}"
18+
)
19+
20+
file(GLOB tests ${CMAKE_CURRENT_SOURCE_DIR} *.test.lua)
21+
foreach(test_path ${tests})
22+
get_filename_component(test_name ${test_path} NAME)
23+
# FIXME: By default, GLOB lists directories.
24+
# Directories are omitted in the result if LIST_DIRECTORIES
25+
# is set to false. New in version CMake 3.3.
26+
if(${test_name} STREQUAL ${TEST_SUITE_NAME})
27+
continue()
28+
endif()
29+
set(TEST_TITLE "test/${TEST_SUITE_NAME}/${test_name}${TEST_PARAM_DELIM}memtx")
30+
set(WORK_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test_name}-workdir-memtx")
31+
file(MAKE_DIRECTORY ${WORK_DIR})
32+
set(TEST_CMD "${RM} -rf ${WORK_DIR}/* && ${TARANTOOL_BIN} ${test_path}")
33+
add_test(NAME ${TEST_TITLE}
34+
COMMAND ${BASH} -c "${TEST_CMD}"
35+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
36+
)
37+
set_tests_properties(${TEST_TITLE} PROPERTIES
38+
ENVIRONMENT "${TEST_ENV};TT_WORK_DIR=${WORK_DIR};TEST_ENGINE=memtx"
39+
LABELS "${TEST_SUITE_NAME};regression"
40+
DEPENDS ${TEST_SUITE_NAME}-deps
41+
FAIL_REGULAR_EXPRESSION "${CTEST_FAIL_REGEXP}"
42+
)
43+
44+
set(TEST_TITLE "test/${TEST_SUITE_NAME}/${test_name}${TEST_PARAM_DELIM}vinyl")
45+
set(WORK_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test_name}-workdir-vinyl")
46+
file(MAKE_DIRECTORY ${WORK_DIR})
47+
set(TEST_CMD "${RM} -rf ${WORK_DIR}/* && ${TARANTOOL_BIN} ${TAP_TEST_FLAGS} ${test_path}")
48+
add_test(NAME ${TEST_TITLE}
49+
COMMAND ${BASH} -c "${TEST_CMD}"
50+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
51+
)
52+
set_tests_properties(${TEST_TITLE} PROPERTIES
53+
ENVIRONMENT "${TEST_ENV};TT_WORK_DIR=${WORK_DIR};TEST_ENGINE=vinyl;"
54+
LABELS "${TEST_SUITE_NAME};regression"
55+
DEPENDS ${TEST_SUITE_NAME}-deps
56+
FAIL_REGULAR_EXPRESSION "${CTEST_FAIL_REGEXP}"
57+
)
58+
endforeach()

‎test/engine-tap/lib/tester.lua‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ local test = tap.test("errno")
1010

1111
local ok, test_run = pcall(require, 'test_run')
1212
test_run = ok and test_run.new() or nil
13-
local engine = test_run ~= nil and test_run:get_cfg('engine') or 'memtx'
13+
-- Default value `memtx` was added to have ability to run a
14+
-- `engine-tap` test w/o `test-run.py`, see commit
15+
-- commit 3bd870261c462416c29226414fe0a2d79aba0c74
16+
-- ('box, datetime: datetime comparison for indices').
17+
local engine = test_run and test_run:get_cfg('engine') or
18+
(os.getenv('TEST_ENGINE') or 'memtx')
1419

1520
function test.engine(self)
1621
return engine

‎test/sql-tap/CMakeLists.txt‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,110 @@ build_module(sql_uuid sql_uuid.c)
99
target_link_libraries(sql_uuid msgpuck core)
1010
build_module(decimal decimal.c)
1111
target_link_libraries(decimal msgpuck)
12+
13+
tarantool_make_lua_path(LUA_PATH
14+
PATHS
15+
${CMAKE_CURRENT_SOURCE_DIR}/lua/?.lua
16+
${PROJECT_SOURCE_DIR}/test/sql/lua/?.lua
17+
# Needed for identifier.lua.
18+
${PROJECT_SOURCE_DIR}/test/box/lua/?.lua
19+
)
20+
21+
tarantool_make_lua_path(LUA_CPATH
22+
PATHS
23+
${CMAKE_CURRENT_BINARY_DIR}/?${CMAKE_SHARED_LIBRARY_SUFFIX}
24+
)
25+
26+
set(TEST_SUITE_NAME "sql-tap")
27+
28+
message(STATUS "Add test suite ${TEST_SUITE_NAME}")
29+
30+
# XXX: The call produces both test and target <sql-tap-deps>
31+
# as a side effect.
32+
_add_test_suite_target(${TEST_SUITE_NAME}
33+
LABELS ${TEST_SUITE_NAME}
34+
DEPENDS gh-5938-wrong-string-length
35+
gh-6024-funcs-return-bin
36+
gh-2579-custom-aggregate
37+
sql_uuid decimal
38+
)
39+
40+
list(APPEND TEST_ENV
41+
"LUA_PATH=${LUA_PATH}"
42+
"LUA_CPATH=${LUA_CPATH}"
43+
"BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}"
44+
)
45+
46+
file(GLOB tests ${CMAKE_CURRENT_SOURCE_DIR} *.test.lua)
47+
foreach(test_path ${tests})
48+
get_filename_component(test_name ${test_path} NAME)
49+
# FIXME: By default, GLOB lists directories.
50+
# Directories are omitted in the result if LIST_DIRECTORIES
51+
# is set to false. New in version CMake 3.3.
52+
if(${test_name} STREQUAL ${TEST_SUITE_NAME})
53+
continue()
54+
endif()
55+
set(TEST_TITLE "test/${TEST_SUITE_NAME}/${test_name}${TEST_PARAM_DELIM}memtx")
56+
set(WORK_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test_name}-workdir-memtx")
57+
file(MAKE_DIRECTORY ${WORK_DIR})
58+
set(TEST_CMD "${RM} -rf ${WORK_DIR}/* && ${TARANTOOL_BIN} ${test_path}")
59+
add_test(NAME ${TEST_TITLE}
60+
COMMAND ${BASH} -c "${TEST_CMD}"
61+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
62+
)
63+
set_tests_properties(${TEST_TITLE} PROPERTIES
64+
ENVIRONMENT "${TEST_ENV};TT_WORK_DIR=${WORK_DIR};TEST_ENGINE=memtx;"
65+
LABELS "${TEST_SUITE_NAME};regression"
66+
DEPENDS ${TEST_SUITE_NAME}-deps
67+
FAIL_REGULAR_EXPRESSION "${CTEST_FAIL_REGEXP}"
68+
)
69+
70+
set(TEST_TITLE "test/${TEST_SUITE_NAME}/${test_name}${TEST_PARAM_DELIM}vinyl")
71+
set(WORK_DIR "${CMAKE_CURRENT_BINARY_DIR}/${test_name}-workdir-vinyl")
72+
file(MAKE_DIRECTORY ${WORK_DIR})
73+
set(TEST_CMD "${RM} -rf ${WORK_DIR}/* && ${TARANTOOL_BIN} ${TAP_TEST_FLAGS} ${test_path}")
74+
add_test(NAME ${TEST_TITLE}
75+
COMMAND ${BASH} -c "${TEST_CMD}"
76+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
77+
)
78+
set_tests_properties(${TEST_TITLE} PROPERTIES
79+
ENVIRONMENT "${TEST_ENV};TT_WORK_DIR=${WORK_DIR};TEST_ENGINE=vinyl;"
80+
LABELS "${TEST_SUITE_NAME};regression"
81+
DEPENDS ${TEST_SUITE_NAME}-deps
82+
FAIL_REGULAR_EXPRESSION "${CTEST_FAIL_REGEXP}"
83+
)
84+
endforeach()
85+
86+
# See test/sql-tap/suite.ini.
87+
list(APPEND DISABLED_TESTS
88+
test/sql-tap/analyze1.test.lua
89+
test/sql-tap/analyze3.test.lua
90+
test/sql-tap/analyze4.test.lua
91+
test/sql-tap/analyze5.test.lua
92+
test/sql-tap/analyze6.test.lua
93+
test/sql-tap/analyze7.test.lua
94+
test/sql-tap/analyze8.test.lua
95+
test/sql-tap/analyze9.test.lua
96+
test/sql-tap/analyzeC.test.lua
97+
test/sql-tap/analyzeD.test.lua
98+
test/sql-tap/analyzeE.test.lua
99+
test/sql-tap/analyzeF.test.lua
100+
test/sql-tap/date.test.lua
101+
# See gh-3694.
102+
test/sql-tap/debug_mode_only.test.lua
103+
test/sql-tap/e_expr.test.lua
104+
test/sql-tap/gh-3350-skip-scan.test.lua
105+
test/sql-tap/like2.test.lua
106+
test/sql-tap/randexpr1.test.lua
107+
test/sql-tap/selectA.test.lua
108+
test/sql-tap/tkt3791.test.lua
109+
test/sql-tap/tkt-bd484a090c.test.lua
110+
test/sql-tap/types2.test.lua
111+
)
112+
foreach(test_name IN LISTS DISABLED_TESTS)
113+
set_tests_properties(
114+
${test_name}${TEST_PARAM_DELIM}memtx
115+
${test_name}${TEST_PARAM_DELIM}vinyl
116+
PROPERTIES DISABLED TRUE
117+
)
118+
endforeach()

0 commit comments

Comments
 (0)