diff --git a/Makefile b/Makefile index 8485ff54..bd4ed1b0 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,9 @@ eunit: compile-tests erl $(ERL_OPTS) -noinput -pa ebin -pa test -eval \ '$(erl_run_eunit), halt().' +ct: + @rebar3 ct --label "git: $$(git describe --tags --always) $$(git diff --no-ext-diff --quiet --exit-code || echo '(modified)')" + cli-tests: bin/gradualizer test/arg.beam # CLI test cases # 1. When checking a dir with erl files, erl file names are printed diff --git a/rebar.config b/rebar.config index 155bd7cc..0de6c2cb 100644 --- a/rebar.config +++ b/rebar.config @@ -4,10 +4,7 @@ {deps, [ {proper, {git, "https://github.com/proper-testing/proper.git", {branch, "master"}}} - ]}, - %% see the maybe expression fail; - %% the VM also needs to be configured to load the module - {erl_opts, [{feature,maybe_expr,enable}]} + ]} ]} ]}. diff --git a/src/typechecker.erl b/src/typechecker.erl index 9e9337c6..bad64b8a 100644 --- a/src/typechecker.erl +++ b/src/typechecker.erl @@ -5746,8 +5746,10 @@ type_check_forms(Forms, Opts) -> %% a Gradualizer (NOT the checked program!) error. -spec type_check_form_with_timeout(expr(), [any()], boolean(), env(), [any()]) -> [any()]. type_check_form_with_timeout(Function, Errors, StopOnFirstError, Env, Opts) -> - %% TODO: make FormCheckTimeOut configurable - FormCheckTimeOut = ?form_check_timeout_ms, + FormCheckTimeOut = case lists:keyfind(form_check_timeout_ms, 1, Opts) of + false -> ?form_check_timeout_ms; + {form_check_timeout_ms, MS} -> MS + end, ?verbose(Env, "Spawning async task...~n", []), Self = self(), Task = fun () -> diff --git a/test/gradualizer_dynamic_suite.erl b/test/gradualizer_dynamic_suite.erl new file mode 100644 index 00000000..4bae68bb --- /dev/null +++ b/test/gradualizer_dynamic_suite.erl @@ -0,0 +1,60 @@ +-module(gradualizer_dynamic_suite). + +-export([reload/1]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +reload(Config) -> + Module = ?config(dynamic_suite_module, Config), + ?assert(Module /= undefined), + case erlang:function_exported(Module, generated_tests, 0) of + true -> + {ok, Module:generated_tests()}; + false -> + Path = ?config(dynamic_suite_test_path, Config), + ?assert(Path /= undefined), + Forms = get_forms(Module), + FilesForms = map_erl_files(fun (File) -> + make_test_form(Forms, File, Config) + end, Path), + {TestFiles, TestForms} = lists:unzip(FilesForms), + TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], + ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]), + GeneratedTestsForm = make_generated_tests_form(TestNames), + NewForms = Forms ++ TestForms ++ [GeneratedTestsForm, {eof, 0}], + {ok, _} = merl:compile_and_load(NewForms), + {ok, TestNames} + end. + +map_erl_files(Fun, Dir) -> + Files = filelib:wildcard(filename:join(Dir, "*.erl")), + [{filename:basename(File), Fun(File)} || File <- Files]. + +make_test_form(Forms, File, Config) -> + TestTemplateName = ?config(dynamic_test_template, Config), + ?assert(TestTemplateName /= undefined), + TestTemplate = merl:quote("'@Name'(_) -> _@Body."), + {function, _Anno, _Name, 1, Clauses} = lists:keyfind(TestTemplateName, 3, Forms), + [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses, + TestName = filename:basename(File, ".erl"), + ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]), + TestEnv = [ + {'Name', erl_syntax:atom(TestName)}, + {'Body', ClauseBody} + ], + erl_syntax:revert(merl:subst(TestTemplate, TestEnv)). + +make_generated_tests_form(TestNames) -> + Template = merl:quote("generated_tests() -> _@Body."), + erl_syntax:revert(merl:subst(Template, [{'Body', merl:term(TestNames)}])). + +get_forms(Module) -> + ModPath = code:which(Module), + {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]), + {abstract_code, {raw_abstract_v1, Forms}} = Abst, + StripEnd = fun + ({eof, _}) -> false; + (_) -> true + end, + lists:filter(StripEnd, Forms). diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl new file mode 100644 index 00000000..f3ef8811 --- /dev/null +++ b/test/known_problems_should_fail_SUITE.erl @@ -0,0 +1,63 @@ +-module(known_problems_should_fail_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +all() -> + [{group, all_tests}]. + +groups() -> + Config = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_fail")}, + {dynamic_test_template, known_problems_should_fail_template} + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(code:lib_dir(gradualizer)), + Config. + +load_prerequisites(AppBase) -> + %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]), + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +known_problems_should_fail_template(_@File) -> + Result = safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]), + case Result of + crash -> + ok; + Errors -> + ErrorsExceptTimeouts = lists:filter( + fun ({_File, {form_check_timeout, _}}) -> false; (_) -> true end, + Errors), + ?assertEqual(0, length(ErrorsExceptTimeouts)) + end. + +safe_type_check_file(File) -> + safe_type_check_file(File, []). + +safe_type_check_file(File, Opts) -> + try + gradualizer:type_check_file(File, Opts) + catch + _:_ -> crash + end. diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl new file mode 100644 index 00000000..8d912e35 --- /dev/null +++ b/test/known_problems_should_pass_SUITE.erl @@ -0,0 +1,55 @@ +-module(known_problems_should_pass_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +all() -> + [{group, all_tests}]. + +groups() -> + Config = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_pass")}, + {dynamic_test_template, known_problems_should_pass_template} + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(code:lib_dir(gradualizer)), + Config. + +load_prerequisites(_AppBase) -> + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +known_problems_should_pass_template(_@File) -> + {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), + ExpectedErrors = typechecker:number_of_exported_functions(Forms), + ReturnedErrors = length(safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}])), + ?assertEqual(ExpectedErrors, ReturnedErrors). + +safe_type_check_file(File) -> + safe_type_check_file(File, []). + +safe_type_check_file(File, Opts) -> + try + gradualizer:type_check_file(File, Opts) + catch + _:_ -> crash + end. diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl new file mode 100644 index 00000000..9be8d18c --- /dev/null +++ b/test/should_fail_SUITE.erl @@ -0,0 +1,56 @@ +-module(should_fail_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +all() -> + [{group, all_tests}]. + +groups() -> + Config = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_fail")}, + {dynamic_test_template, should_fail_template} + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(code:lib_dir(gradualizer)), + Config. + +load_prerequisites(AppBase) -> + %% user_types.erl is referenced by opaque_fail.erl. + %% It is not in the sourcemap of the DB so let's import it manually + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]), + %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]), + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +should_fail_template(_@File) -> + Errors = gradualizer:type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]), + Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], + ?assertEqual(0, length(Timeouts)), + %% Test that error formatting doesn't crash + Opts = [{fmt_location, brief}, + {fmt_expr_fun, fun erl_prettypr:format/1}], + lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), + {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), + ExpectedErrors = typechecker:number_of_exported_functions(Forms), + ?assertEqual(ExpectedErrors, length(Errors)). diff --git a/test/should_pass/module_info.erl b/test/should_pass/module_info_pass.erl similarity index 92% rename from test/should_pass/module_info.erl rename to test/should_pass/module_info_pass.erl index d4d1c028..8e781741 100644 --- a/test/should_pass/module_info.erl +++ b/test/should_pass/module_info_pass.erl @@ -1,4 +1,4 @@ --module(module_info). +-module(module_info_pass). -compile([export_all, nowarn_export_all]). @@ -18,4 +18,4 @@ unary_direct() -> -spec unary_var() -> atom(). unary_var() -> I = erlang:module_info(module), - I. \ No newline at end of file + I. diff --git a/test/should_pass_SUITE.erl b/test/should_pass_SUITE.erl new file mode 100644 index 00000000..bbd7929a --- /dev/null +++ b/test/should_pass_SUITE.erl @@ -0,0 +1,48 @@ +-module(should_pass_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +all() -> + [{group, all_tests}]. + +groups() -> + Opts = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_pass")}, + {dynamic_test_template, should_pass_template} + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Opts), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(code:lib_dir(gradualizer)), + Config. + +load_prerequisites(AppBase) -> + %% user_types.erl is referenced by remote_types.erl and opaque.erl. + %% It is not in the sourcemap of the DB so let's import it manually + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]), + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/other_module.erl")]), + %% imported.erl references any.erl + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/any.erl")]), + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +should_pass_template(_@File) -> + ?assertEqual(ok, gradualizer:type_check_file(_@File, [{form_check_timeout_ms, 2000}])).