From c66cfb408bd7498d62fafe6db5d421f34fc556d7 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 13:58:02 +0200 Subject: [PATCH 01/20] Sketch a Common Test dynamic suite shape --- test/should_fail_SUITE.erl | 243 +++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 test/should_fail_SUITE.erl diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl new file mode 100644 index 00000000..f249a0b0 --- /dev/null +++ b/test/should_fail_SUITE.erl @@ -0,0 +1,243 @@ +-module(should_fail_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). + +%% Test cases +-export([ + ]). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{minutes,10}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(), + ok = dynamic_suite_reload(?MODULE), + Config. + +load_prerequisites() -> + %% user_types.erl is referenced by opaque_fail.erl. + %% It is not in the sourcemap of the DB so let's import it manually + %gradualizer_db:import_erl_files(["test/should_pass/user_types.erl"]), + %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl + %gradualizer_db:import_erl_files(["test/should_fail/exhaustive_user_type.erl"]), + ok. + +dynamic_suite_reload(Module) -> + Forms = get_forms(Module), + TestTemplate = merl:quote("'@Name'(_) -> _@Body."), + {'fun', _, {clauses, [{clause, _, _, _, Body}]}} = merl:quote(" + fun (_) -> + 1 / 0 + end + "), + TestName = "test1", + TestEnv = [ + {'Name', erl_syntax:atom(TestName)}, + {'Body', Body} + ], + TestForm = erl_syntax:revert(merl:subst(TestTemplate, TestEnv)), + NewForms = Forms ++ [TestForm, {eof, 0}], + merl:compile_and_load(NewForms), + ok. + +%should_fail_template() -> +% Errors = gradualizer:type_check_file(File, [return_errors]), +% Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], +% 0 = length(Timeouts), +% %% Test that error formatting doesn't crash +% Opts = [{fmt_location, brief}, +% {fmt_expr_fun, fun erl_prettypr:format/1}], +% lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), +% {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(File, []), +% ExpectedErrors = typechecker:number_of_exported_functions(Forms), +% ExpectedErrors = length(Errors). + +get_forms(Module) -> + ModPath = code:which(Module), + {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]), + {abstract_code, {raw_abstract_v1, Forms}} = Abst, + StripEnd = fun + ({eof, _}) -> false; + (_) -> true + end, + lists:filter(StripEnd, Forms). + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% GroupName = atom() +%% Name of the test case group that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding configuration data for the group. +%% Reason = term() +%% The reason for skipping all test cases and subgroups in the group. +%% +%% Description: Initialization before each test case group. +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% +%% GroupName = atom() +%% Name of the test case group that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding configuration data for the group. +%% +%% Description: Cleanup after each test case group. +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} | {fail,Reason} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for failing the test case. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% The name of the group. +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% Group properties that may be combined. +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% The name of a test case. +%% Shuffle = shuffle | {shuffle,Seed} +%% To get cases executed in random order. +%% Seed = {integer(),integer(),integer()} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% To get execution of cases repeated. +%% N = integer() | forever +%% +%% Description: Returns a list of test case group definitions. +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% Name of a test case group. +%% TestCase = atom() +%% Name of a test case. +%% Reason = term() +%% The reason for skipping all groups and test cases. +%% +%% Description: Returns the list of groups and test cases that +%% are to be executed. +%%-------------------------------------------------------------------- +all() -> + [test1]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% Comment = term() +%% A comment about the test case that will be printed in the html log. +%% +%% Description: Test case function. (The name of it must be specified in +%% the all/0 list or in a test case group for the test case +%% to be executed). +%%-------------------------------------------------------------------- From 3993820b1c92803bedea85247ba3e6acae5e66fe Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 14:12:57 +0200 Subject: [PATCH 02/20] Add Makefile ct rule which labels a test run with git commit --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 8485ff54..bd4ed1b0 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,9 @@ eunit: compile-tests erl $(ERL_OPTS) -noinput -pa ebin -pa test -eval \ '$(erl_run_eunit), halt().' +ct: + @rebar3 ct --label "git: $$(git describe --tags --always) $$(git diff --no-ext-diff --quiet --exit-code || echo '(modified)')" + cli-tests: bin/gradualizer test/arg.beam # CLI test cases # 1. When checking a dir with erl files, erl file names are printed From d97a9f4f04a4f8da27d33ed5bf3d214cefc8ab91 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 14:40:37 +0200 Subject: [PATCH 03/20] Get should_fail test generation to work --- test/should_fail_SUITE.erl | 39 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index f249a0b0..4696a682 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -66,32 +66,31 @@ load_prerequisites() -> dynamic_suite_reload(Module) -> Forms = get_forms(Module), TestTemplate = merl:quote("'@Name'(_) -> _@Body."), - {'fun', _, {clauses, [{clause, _, _, _, Body}]}} = merl:quote(" - fun (_) -> - 1 / 0 - end - "), - TestName = "test1", + {function, _Anno, _Name, 1, Clauses} = lists:keyfind(should_fail_template, 3, Forms), + [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses, + TestName = "unary_op", + TestFile = "/Users/erszcz/work/erszcz/gradualizer/test/should_fail/unary_op.erl", + ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(TestFile)}]), TestEnv = [ {'Name', erl_syntax:atom(TestName)}, - {'Body', Body} + {'Body', ClauseBody} ], TestForm = erl_syntax:revert(merl:subst(TestTemplate, TestEnv)), NewForms = Forms ++ [TestForm, {eof, 0}], - merl:compile_and_load(NewForms), + {ok, _} = merl:compile_and_load(NewForms), ok. -%should_fail_template() -> -% Errors = gradualizer:type_check_file(File, [return_errors]), -% Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], -% 0 = length(Timeouts), -% %% Test that error formatting doesn't crash -% Opts = [{fmt_location, brief}, -% {fmt_expr_fun, fun erl_prettypr:format/1}], -% lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), -% {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(File, []), -% ExpectedErrors = typechecker:number_of_exported_functions(Forms), -% ExpectedErrors = length(Errors). +should_fail_template(_@File) -> + Errors = gradualizer:type_check_file(_@File, [return_errors]), + Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], + 0 = length(Timeouts), + %% Test that error formatting doesn't crash + Opts = [{fmt_location, brief}, + {fmt_expr_fun, fun erl_prettypr:format/1}], + lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), + {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), + ExpectedErrors = typechecker:number_of_exported_functions(Forms), + ExpectedErrors = length(Errors). get_forms(Module) -> ModPath = code:which(Module), @@ -218,7 +217,7 @@ groups() -> %% are to be executed. %%-------------------------------------------------------------------- all() -> - [test1]. + [unary_op]. %%-------------------------------------------------------------------- From a46d2a094de04a639bfeaed121ba7db483a0cd62 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 14:56:59 +0200 Subject: [PATCH 04/20] Generate all tests for a given path --- test/should_fail_SUITE.erl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 4696a682..a5cc4dbe 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -65,20 +65,29 @@ load_prerequisites() -> dynamic_suite_reload(Module) -> Forms = get_forms(Module), + FilesForms = map_erl_files(fun (File) -> + make_test_form(Forms, File) + end, "/Users/erszcz/work/erszcz/gradualizer/test/should_fail"), + {_TestFiles, TestForms} = lists:unzip(FilesForms), + NewForms = Forms ++ TestForms ++ [{eof, 0}], + {ok, _} = merl:compile_and_load(NewForms), + ok. + +map_erl_files(Fun, Dir) -> + Files = filelib:wildcard(filename:join(Dir, "*.erl")), + [{filename:basename(File), Fun(File)} || File <- Files]. + +make_test_form(Forms, File) -> TestTemplate = merl:quote("'@Name'(_) -> _@Body."), {function, _Anno, _Name, 1, Clauses} = lists:keyfind(should_fail_template, 3, Forms), [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses, - TestName = "unary_op", - TestFile = "/Users/erszcz/work/erszcz/gradualizer/test/should_fail/unary_op.erl", - ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(TestFile)}]), + TestName = filename:basename(File, ".erl"), + ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]), TestEnv = [ {'Name', erl_syntax:atom(TestName)}, {'Body', ClauseBody} ], - TestForm = erl_syntax:revert(merl:subst(TestTemplate, TestEnv)), - NewForms = Forms ++ [TestForm, {eof, 0}], - {ok, _} = merl:compile_and_load(NewForms), - ok. + erl_syntax:revert(merl:subst(TestTemplate, TestEnv)). should_fail_template(_@File) -> Errors = gradualizer:type_check_file(_@File, [return_errors]), From 193dcb07450796bca787b589e02394062a0e63dd Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 15:11:03 +0200 Subject: [PATCH 05/20] Generate SUITE:all/0 dynamically Sadly, this doesn't work, as init_per_suite/1 is called after all/0, so a suite with all/0 not returning any tests is not run :( --- test/should_fail_SUITE.erl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index a5cc4dbe..1a1e1c29 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -68,8 +68,16 @@ dynamic_suite_reload(Module) -> FilesForms = map_erl_files(fun (File) -> make_test_form(Forms, File) end, "/Users/erszcz/work/erszcz/gradualizer/test/should_fail"), - {_TestFiles, TestForms} = lists:unzip(FilesForms), - NewForms = Forms ++ TestForms ++ [{eof, 0}], + {TestFiles, TestForms} = lists:unzip(FilesForms), + TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], + AllTestsTemplate = merl:quote("all() -> _@AllTests."), + AllTestsForm = merl:subst(AllTestsTemplate, [{'AllTests', merl:term(TestNames)}]), + DropAllFunction = fun + ({function, _, all, _, _}) -> false; + (_) -> true + end, + Forms1 = lists:filter(DropAllFunction, Forms), + NewForms = Forms1 ++ [AllTestsForm] ++ TestForms ++ [{eof, 0}], {ok, _} = merl:compile_and_load(NewForms), ok. @@ -226,7 +234,8 @@ groups() -> %% are to be executed. %%-------------------------------------------------------------------- all() -> - [unary_op]. + %% Body of this function is dynamically replaced in init_per_suite/1. + []. %%-------------------------------------------------------------------- From 81e85264b07f8dfefc52f3e5211b22375d70f91f Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 15:25:56 +0200 Subject: [PATCH 06/20] Define all/0 tests manually --- test/should_fail_SUITE.erl | 57 +++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 1a1e1c29..3958e60d 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -52,7 +52,11 @@ suite() -> init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), ok = load_prerequisites(), - ok = dynamic_suite_reload(?MODULE), + {ok, TestNames} = dynamic_suite_reload(?MODULE), + case all() of + TestNames -> ok; + _ -> ct:fail("Please update all/0 to list all should_fail tests") + end, Config. load_prerequisites() -> @@ -65,21 +69,16 @@ load_prerequisites() -> dynamic_suite_reload(Module) -> Forms = get_forms(Module), + Path = "/Users/erszcz/work/erszcz/gradualizer/test/should_fail", FilesForms = map_erl_files(fun (File) -> make_test_form(Forms, File) - end, "/Users/erszcz/work/erszcz/gradualizer/test/should_fail"), + end, Path), {TestFiles, TestForms} = lists:unzip(FilesForms), TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], - AllTestsTemplate = merl:quote("all() -> _@AllTests."), - AllTestsForm = merl:subst(AllTestsTemplate, [{'AllTests', merl:term(TestNames)}]), - DropAllFunction = fun - ({function, _, all, _, _}) -> false; - (_) -> true - end, - Forms1 = lists:filter(DropAllFunction, Forms), - NewForms = Forms1 ++ [AllTestsForm] ++ TestForms ++ [{eof, 0}], + ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]), + NewForms = Forms ++ TestForms ++ [{eof, 0}], {ok, _} = merl:compile_and_load(NewForms), - ok. + {ok, TestNames}. map_erl_files(Fun, Dir) -> Files = filelib:wildcard(filename:join(Dir, "*.erl")), @@ -234,8 +233,40 @@ groups() -> %% are to be executed. %%-------------------------------------------------------------------- all() -> - %% Body of this function is dynamically replaced in init_per_suite/1. - []. + [annotated_types_fail,arg,arith_op_fail,arity_mismatch, + bc_fail,bin_expression,bin_type_error,branch,branch2,call, + call_intersection_function_with_union_arg_fail,case_pattern, + case_pattern2,catch_expr_fail,cons,covariant_map_keys_fail, + cyclic_type_vars,depth,exhaustive,exhaustive_float, + exhaustive_list_variants,exhaustive_refinable_map_variants, + exhaustive_remote_user_type,exhaustive_string_variants, + exhaustive_type,exhaustive_user_type, + exhaustiveness_check_toggling,generator,guard_fail, + imported_undef,infer_enabled,intersection_check, + intersection_fail,intersection_infer, + intersection_with_any_fail,iodata_fail,lambda_not_fun, + lc_generator_not_none_fail,lc_not_list,list_infer_fail, + list_op,list_op_should_fail,list_union_fail, + lists_map_nonempty_fail,literal_char,literal_patterns, + logic_op,map_entry,map_fail,map_failing_expr, + map_failing_subtyping,map_field_invalid_update,map_literal, + map_pattern_fail,map_refinement_fail,map_type_error,match, + messaging_fail,module_info_fail,named_fun_fail, + named_fun_infer_fail,nil,no_idempotent_xor, + non_neg_plus_pos_is_pos_fail, + nonempty_list_match_in_head_nonexhaustive, + nonempty_string_fail,opaque_fail,operator_pattern_fail, + pattern,pattern_record_fail,poly_fail,poly_lists_map_fail, + poly_union_lower_bound_fail,pp_intersection,record, + record_exhaustive,record_field,record_index, + record_info_fail,record_refinement_fail,record_update, + record_wildcard_fail,recursive_type_fail, + recursive_types_failing,rel_op,return_fun_fail, + rigid_type_variables_fail,send_fail,shortcut_ops_fail, + spec_and_fun_clause_intersection_fail,string_literal, + tuple_union_arg_fail,tuple_union_fail,tuple_union_pattern, + tuple_union_refinement,type_refinement_fail,unary_op, + unary_plus_fail,union_with_any,unreachable_after_refinement]. %%-------------------------------------------------------------------- From 421368768caec3ece88f908222c08ab2e52ef771 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 15:35:05 +0200 Subject: [PATCH 07/20] Make paths relative to app location --- test/should_fail_SUITE.erl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 3958e60d..819a827c 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -51,25 +51,26 @@ suite() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), - ok = load_prerequisites(), - {ok, TestNames} = dynamic_suite_reload(?MODULE), + AppBase = code:lib_dir(gradualizer), + ok = load_prerequisites(AppBase), + {ok, TestNames} = dynamic_suite_reload(?MODULE, AppBase), case all() of TestNames -> ok; _ -> ct:fail("Please update all/0 to list all should_fail tests") end, Config. -load_prerequisites() -> +load_prerequisites(AppBase) -> %% user_types.erl is referenced by opaque_fail.erl. %% It is not in the sourcemap of the DB so let's import it manually - %gradualizer_db:import_erl_files(["test/should_pass/user_types.erl"]), + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]), %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl - %gradualizer_db:import_erl_files(["test/should_fail/exhaustive_user_type.erl"]), + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]), ok. -dynamic_suite_reload(Module) -> +dynamic_suite_reload(Module, AppBase) -> Forms = get_forms(Module), - Path = "/Users/erszcz/work/erszcz/gradualizer/test/should_fail", + Path = filename:join(AppBase, "test/should_fail"), FilesForms = map_erl_files(fun (File) -> make_test_form(Forms, File) end, Path), From 57b9ee2a1d7003dfd491ac492c280b1e73cefa51 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 15:38:59 +0200 Subject: [PATCH 08/20] Use EUnit macros for CLI readability --- test/should_fail_SUITE.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 819a827c..786f455f 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -4,6 +4,9 @@ -include_lib("common_test/include/ct.hrl"). +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + %% Test server callbacks -export([suite/0, all/0, @@ -100,14 +103,14 @@ make_test_form(Forms, File) -> should_fail_template(_@File) -> Errors = gradualizer:type_check_file(_@File, [return_errors]), Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], - 0 = length(Timeouts), + ?assertEqual(0, length(Timeouts)), %% Test that error formatting doesn't crash Opts = [{fmt_location, brief}, {fmt_expr_fun, fun erl_prettypr:format/1}], lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), ExpectedErrors = typechecker:number_of_exported_functions(Forms), - ExpectedErrors = length(Errors). + ?assertEqual(ExpectedErrors, length(Errors)). get_forms(Module) -> ModPath = code:which(Module), From fcb443daec05ea7f5015cb77fe0121e81ac80fea Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 15:50:42 +0200 Subject: [PATCH 09/20] Clean up --- test/should_fail_SUITE.erl | 154 +------------------------------------ 1 file changed, 1 insertion(+), 153 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 786f455f..010367cb 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -1,6 +1,6 @@ -module(should_fail_SUITE). --compile(export_all). +-compile([export_all, nowarn_export_all]). -include_lib("common_test/include/ct.hrl"). @@ -15,43 +15,9 @@ init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2]). -%% Test cases --export([ - ]). - -%%-------------------------------------------------------------------- -%% COMMON TEST CALLBACK FUNCTIONS -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% Function: suite() -> Info -%% -%% Info = [tuple()] -%% List of key/value pairs. -%% -%% Description: Returns list of tuples to set default properties -%% for the suite. -%% -%% Note: The suite/0 function is only meant to be used to return -%% default data values, not perform any other operations. -%%-------------------------------------------------------------------- suite() -> [{timetrap,{minutes,10}}]. -%%-------------------------------------------------------------------- -%% Function: init_per_suite(Config0) -> -%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} -%% -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Reason = term() -%% The reason for skipping the suite. -%% -%% Description: Initialization before the suite. -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), AppBase = code:lib_dir(gradualizer), @@ -122,120 +88,24 @@ get_forms(Module) -> end, lists:filter(StripEnd, Forms). -%%-------------------------------------------------------------------- -%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} -%% -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% -%% Description: Cleanup after the suite. -%%-------------------------------------------------------------------- end_per_suite(_Config) -> ok. -%%-------------------------------------------------------------------- -%% Function: init_per_group(GroupName, Config0) -> -%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} -%% -%% GroupName = atom() -%% Name of the test case group that is about to run. -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding configuration data for the group. -%% Reason = term() -%% The reason for skipping all test cases and subgroups in the group. -%% -%% Description: Initialization before each test case group. -%%-------------------------------------------------------------------- init_per_group(_GroupName, Config) -> Config. -%%-------------------------------------------------------------------- -%% Function: end_per_group(GroupName, Config0) -> -%% void() | {save_config,Config1} -%% -%% GroupName = atom() -%% Name of the test case group that is finished. -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding configuration data for the group. -%% -%% Description: Cleanup after each test case group. -%%-------------------------------------------------------------------- end_per_group(_GroupName, _Config) -> ok. -%%-------------------------------------------------------------------- -%% Function: init_per_testcase(TestCase, Config0) -> -%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} -%% -%% TestCase = atom() -%% Name of the test case that is about to run. -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Reason = term() -%% The reason for skipping the test case. -%% -%% Description: Initialization before each test case. -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- init_per_testcase(_TestCase, Config) -> Config. -%%-------------------------------------------------------------------- -%% Function: end_per_testcase(TestCase, Config0) -> -%% void() | {save_config,Config1} | {fail,Reason} -%% -%% TestCase = atom() -%% Name of the test case that is finished. -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Reason = term() -%% The reason for failing the test case. -%% -%% Description: Cleanup after each test case. -%%-------------------------------------------------------------------- end_per_testcase(_TestCase, _Config) -> ok. -%%-------------------------------------------------------------------- -%% Function: groups() -> [Group] -%% -%% Group = {GroupName,Properties,GroupsAndTestCases} -%% GroupName = atom() -%% The name of the group. -%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] -%% Group properties that may be combined. -%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] -%% TestCase = atom() -%% The name of a test case. -%% Shuffle = shuffle | {shuffle,Seed} -%% To get cases executed in random order. -%% Seed = {integer(),integer(),integer()} -%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | -%% repeat_until_any_ok | repeat_until_any_fail -%% To get execution of cases repeated. -%% N = integer() | forever -%% -%% Description: Returns a list of test case group definitions. -%%-------------------------------------------------------------------- groups() -> []. -%%-------------------------------------------------------------------- -%% Function: all() -> GroupsAndTestCases | {skip,Reason} -%% -%% GroupsAndTestCases = [{group,GroupName} | TestCase] -%% GroupName = atom() -%% Name of a test case group. -%% TestCase = atom() -%% Name of a test case. -%% Reason = term() -%% The reason for skipping all groups and test cases. -%% -%% Description: Returns the list of groups and test cases that -%% are to be executed. -%%-------------------------------------------------------------------- all() -> [annotated_types_fail,arg,arith_op_fail,arity_mismatch, bc_fail,bin_expression,bin_type_error,branch,branch2,call, @@ -271,25 +141,3 @@ all() -> tuple_union_arg_fail,tuple_union_fail,tuple_union_pattern, tuple_union_refinement,type_refinement_fail,unary_op, unary_plus_fail,union_with_any,unreachable_after_refinement]. - - -%%-------------------------------------------------------------------- -%% TEST CASES -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% Function: TestCase(Config0) -> -%% ok | exit() | {skip,Reason} | {comment,Comment} | -%% {save_config,Config1} | {skip_and_save,Reason,Config1} -%% -%% Config0 = Config1 = [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Reason = term() -%% The reason for skipping the test case. -%% Comment = term() -%% A comment about the test case that will be printed in the html log. -%% -%% Description: Test case function. (The name of it must be specified in -%% the all/0 list or in a test case group for the test case -%% to be executed). -%%-------------------------------------------------------------------- From f150d53c2ee37670b7c49f576a45a6815e2789ce Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 15:51:21 +0200 Subject: [PATCH 10/20] Make test template function configurable, pass params from the top --- test/should_fail_SUITE.erl | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 010367cb..a859edf5 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -16,20 +16,25 @@ init_per_testcase/2, end_per_testcase/2]). suite() -> - [{timetrap,{minutes,10}}]. - -init_per_suite(Config) -> + [{timetrap, {minutes, 10}}]. + +init_per_suite(Config0) -> + Config = [ + {app_base, code:lib_dir(gradualizer)}, + {dynamic_suite_module, ?MODULE}, + {dynamic_test_template, should_fail_template} + ] ++ Config0, {ok, _} = application:ensure_all_started(gradualizer), - AppBase = code:lib_dir(gradualizer), - ok = load_prerequisites(AppBase), - {ok, TestNames} = dynamic_suite_reload(?MODULE, AppBase), + ok = load_prerequisites(Config), + {ok, TestNames} = dynamic_suite_reload(Config), case all() of TestNames -> ok; _ -> ct:fail("Please update all/0 to list all should_fail tests") end, Config. -load_prerequisites(AppBase) -> +load_prerequisites(Config) -> + AppBase = ?config(app_base, Config), %% user_types.erl is referenced by opaque_fail.erl. %% It is not in the sourcemap of the DB so let's import it manually gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]), @@ -37,11 +42,13 @@ load_prerequisites(AppBase) -> gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]), ok. -dynamic_suite_reload(Module, AppBase) -> +dynamic_suite_reload(Config) -> + Module = ?config(dynamic_suite_module, Config), + AppBase = ?config(app_base, Config), Forms = get_forms(Module), Path = filename:join(AppBase, "test/should_fail"), FilesForms = map_erl_files(fun (File) -> - make_test_form(Forms, File) + make_test_form(Forms, File, Config) end, Path), {TestFiles, TestForms} = lists:unzip(FilesForms), TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], @@ -54,9 +61,10 @@ map_erl_files(Fun, Dir) -> Files = filelib:wildcard(filename:join(Dir, "*.erl")), [{filename:basename(File), Fun(File)} || File <- Files]. -make_test_form(Forms, File) -> +make_test_form(Forms, File, Config) -> + TestTemplateName = ?config(dynamic_test_template, Config), TestTemplate = merl:quote("'@Name'(_) -> _@Body."), - {function, _Anno, _Name, 1, Clauses} = lists:keyfind(should_fail_template, 3, Forms), + {function, _Anno, _Name, 1, Clauses} = lists:keyfind(TestTemplateName, 3, Forms), [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses, TestName = filename:basename(File, ".erl"), ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]), From 34aa4a490b61d33d7150f336153d5da701166342 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 16:15:05 +0200 Subject: [PATCH 11/20] Extract gradualizer_dynamic_suite --- test/gradualizer_dynamic_suite.erl | 50 ++++++++++++++++++ test/should_fail_SUITE.erl | 81 +++++++----------------------- 2 files changed, 69 insertions(+), 62 deletions(-) create mode 100644 test/gradualizer_dynamic_suite.erl diff --git a/test/gradualizer_dynamic_suite.erl b/test/gradualizer_dynamic_suite.erl new file mode 100644 index 00000000..b002d613 --- /dev/null +++ b/test/gradualizer_dynamic_suite.erl @@ -0,0 +1,50 @@ +-module(gradualizer_dynamic_suite). + +-export([reload/1]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +reload(Config) -> + Module = ?config(dynamic_suite_module, Config), + Path = ?config(dynamic_suite_test_path, Config), + ?assert(Module /= undefined), + ?assert(Path /= undefined), + Forms = get_forms(Module), + FilesForms = map_erl_files(fun (File) -> + make_test_form(Forms, File, Config) + end, Path), + {TestFiles, TestForms} = lists:unzip(FilesForms), + TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], + ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]), + NewForms = Forms ++ TestForms ++ [{eof, 0}], + {ok, _} = merl:compile_and_load(NewForms), + {ok, TestNames}. + +map_erl_files(Fun, Dir) -> + Files = filelib:wildcard(filename:join(Dir, "*.erl")), + [{filename:basename(File), Fun(File)} || File <- Files]. + +make_test_form(Forms, File, Config) -> + TestTemplateName = ?config(dynamic_test_template, Config), + ?assert(TestTemplateName /= undefined), + TestTemplate = merl:quote("'@Name'(_) -> _@Body."), + {function, _Anno, _Name, 1, Clauses} = lists:keyfind(TestTemplateName, 3, Forms), + [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses, + TestName = filename:basename(File, ".erl"), + ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]), + TestEnv = [ + {'Name', erl_syntax:atom(TestName)}, + {'Body', ClauseBody} + ], + erl_syntax:revert(merl:subst(TestTemplate, TestEnv)). + +get_forms(Module) -> + ModPath = code:which(Module), + {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]), + {abstract_code, {raw_abstract_v1, Forms}} = Abst, + StripEnd = fun + ({eof, _}) -> false; + (_) -> true + end, + lists:filter(StripEnd, Forms). diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index a859edf5..40c5b2d4 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -2,8 +2,6 @@ -compile([export_all, nowarn_export_all]). --include_lib("common_test/include/ct.hrl"). - %% EUnit has some handy macros, so let's use it, too -include_lib("eunit/include/eunit.hrl"). @@ -19,22 +17,22 @@ suite() -> [{timetrap, {minutes, 10}}]. init_per_suite(Config0) -> + AppBase = code:lib_dir(gradualizer), Config = [ - {app_base, code:lib_dir(gradualizer)}, {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(AppBase, "test/should_fail")}, {dynamic_test_template, should_fail_template} ] ++ Config0, {ok, _} = application:ensure_all_started(gradualizer), - ok = load_prerequisites(Config), - {ok, TestNames} = dynamic_suite_reload(Config), + ok = load_prerequisites(AppBase), + {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), case all() of TestNames -> ok; - _ -> ct:fail("Please update all/0 to list all should_fail tests") + _ -> ct:fail("Please update all/0 to list all tests") end, Config. -load_prerequisites(Config) -> - AppBase = ?config(app_base, Config), +load_prerequisites(AppBase) -> %% user_types.erl is referenced by opaque_fail.erl. %% It is not in the sourcemap of the DB so let's import it manually gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]), @@ -42,61 +40,8 @@ load_prerequisites(Config) -> gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]), ok. -dynamic_suite_reload(Config) -> - Module = ?config(dynamic_suite_module, Config), - AppBase = ?config(app_base, Config), - Forms = get_forms(Module), - Path = filename:join(AppBase, "test/should_fail"), - FilesForms = map_erl_files(fun (File) -> - make_test_form(Forms, File, Config) - end, Path), - {TestFiles, TestForms} = lists:unzip(FilesForms), - TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], - ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]), - NewForms = Forms ++ TestForms ++ [{eof, 0}], - {ok, _} = merl:compile_and_load(NewForms), - {ok, TestNames}. - -map_erl_files(Fun, Dir) -> - Files = filelib:wildcard(filename:join(Dir, "*.erl")), - [{filename:basename(File), Fun(File)} || File <- Files]. - -make_test_form(Forms, File, Config) -> - TestTemplateName = ?config(dynamic_test_template, Config), - TestTemplate = merl:quote("'@Name'(_) -> _@Body."), - {function, _Anno, _Name, 1, Clauses} = lists:keyfind(TestTemplateName, 3, Forms), - [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses, - TestName = filename:basename(File, ".erl"), - ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]), - TestEnv = [ - {'Name', erl_syntax:atom(TestName)}, - {'Body', ClauseBody} - ], - erl_syntax:revert(merl:subst(TestTemplate, TestEnv)). - -should_fail_template(_@File) -> - Errors = gradualizer:type_check_file(_@File, [return_errors]), - Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], - ?assertEqual(0, length(Timeouts)), - %% Test that error formatting doesn't crash - Opts = [{fmt_location, brief}, - {fmt_expr_fun, fun erl_prettypr:format/1}], - lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), - {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), - ExpectedErrors = typechecker:number_of_exported_functions(Forms), - ?assertEqual(ExpectedErrors, length(Errors)). - -get_forms(Module) -> - ModPath = code:which(Module), - {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]), - {abstract_code, {raw_abstract_v1, Forms}} = Abst, - StripEnd = fun - ({eof, _}) -> false; - (_) -> true - end, - lists:filter(StripEnd, Forms). - end_per_suite(_Config) -> + ok = application:stop(gradualizer), ok. init_per_group(_GroupName, Config) -> @@ -149,3 +94,15 @@ all() -> tuple_union_arg_fail,tuple_union_fail,tuple_union_pattern, tuple_union_refinement,type_refinement_fail,unary_op, unary_plus_fail,union_with_any,unreachable_after_refinement]. + +should_fail_template(_@File) -> + Errors = gradualizer:type_check_file(_@File, [return_errors]), + Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], + ?assertEqual(0, length(Timeouts)), + %% Test that error formatting doesn't crash + Opts = [{fmt_location, brief}, + {fmt_expr_fun, fun erl_prettypr:format/1}], + lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors), + {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), + ExpectedErrors = typechecker:number_of_exported_functions(Forms), + ?assertEqual(ExpectedErrors, length(Errors)). From 2c4df557d79089950235a413708a7a8fcd562036 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 16:15:23 +0200 Subject: [PATCH 12/20] Add test/should_pass_SUITE.erl --- test/should_pass_SUITE.erl | 104 +++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 test/should_pass_SUITE.erl diff --git a/test/should_pass_SUITE.erl b/test/should_pass_SUITE.erl new file mode 100644 index 00000000..b89277f8 --- /dev/null +++ b/test/should_pass_SUITE.erl @@ -0,0 +1,104 @@ +-module(should_pass_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +init_per_suite(Config0) -> + AppBase = code:lib_dir(gradualizer), + Config = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(AppBase, "test/should_pass")}, + {dynamic_test_template, should_pass_template} + ] ++ Config0, + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(AppBase), + {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), + case all() of + TestNames -> ok; + _ -> ct:fail("Please update all/0 to list all tests") + end, + Config. + +load_prerequisites(AppBase) -> + %% user_types.erl is referenced by remote_types.erl and opaque.erl. + %% It is not in the sourcemap of the DB so let's import it manually + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]), + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/other_module.erl")]), + %% imported.erl references any.erl + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/any.erl")]), + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +groups() -> + []. + +all() -> + [alias_in_pattern,andalso_any,ann_types,annotated_types,any, + any_doesnt_have_type_none_pass,any_pattern,bc_pass, + binary_exhaustiveness_checking,binary_in_union,binary_literal_pattern, + bitstring,block_scope,bool,bounded_funs, + call_intersection_function_with_union_arg_pass,'case', + case_of_record_with_user_defined,catch_expr_pass,covariant_map_keys_pass, + cyclic_otp_specs,erlang_error_args_none_pass,exhaustiveness_union_types, + factorial,float,flow,fun_capture,fun_spec,guard,guard_sequences_pass,if_expr, + imported,int,intersection_pass,intersection_with_any_pass,iodata,issue131,lc, + lc_generator_not_none,lc_var_binds_in_filters,list,list_concat_op_pass, + list_exhaustiveness_checking_regressions, + list_exhaustiveness_checking_regressions2, + list_exhaustiveness_checking_unreachable_clause_regression,list_infer_pass, + list_op_pass,listsspecs,map,map_as_argument_update,map_creation, + map_field_valid_update,map_infer_pass,map_passing_expr,map_passing_subtyping, + map_pattern,map_refinement,map_update,map_update_with_record_field, + messaging_pass,minimised_gradualizer_fmt,minus,module_info_higher_arity, + module_info_pass,named_fun_infer_pass,named_fun_pass,negate_none, + nested_pattern_match,non_neg_plus_pos_is_pos_pass,nonempty_cons, + nonempty_list_match_in_head_exhaustive,nonempty_string, + nonexhaustive_record_pattern,opaque,operator_pattern_pass,operator_subtypes, + other_module,pattern_bind_reuse,pattern_record,pattern_with_ty_vars, + poly_lists_map_constraints_pass,poly_lists_map_pass,poly_map_pattern, + poly_pass,poly_pass_infer,poly_pass_no_solve_constraints, + poly_union_lower_bound_pass,preludes,qlc_test,record_info,record_refinement, + record_union_pass,record_union_with_any_should_pass,record_var, + record_wildcard_pass,record_with_user_defined,records, + recursive_call_with_remote_union_return_type_pass,recursive_types_passing, + refine_comparison,refine_mismatch_using_guard_bifs,remote_types, + remote_types_pass,return_fun,rigid_type_variables,rigid_type_variables_pass, + scope,send_pass,sets_set,shortcut_ops_pass, + spec_and_fun_clause_intersection_pass,stuff_as_top,'try',try_expr,tuple, + tuple_union_pass,tuple_union_pat,tuple_union_pattern_pass,type_decl, + type_pattern,type_refinement_pass,type_variable,type_vars_term, + typed_record_field_access,unary_negate_union_with_user_type_pass,unary_plus, + underscore,user_type_in_pattern_body,user_types,var,var_fun,varbind_in_block, + varbind_in_case,varbind_in_function_head,varbind_in_lc,variable_binding, + variable_binding_leaks]. + +should_pass_template(_@File) -> + ?assertEqual(ok, gradualizer:type_check_file(_@File)). From 0983a7aa5910d3fa720621f8650cfb0d548f666f Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 20:52:02 +0200 Subject: [PATCH 13/20] Rename: test/should_pass/module_info.erl -> test/should_pass/module_info_pass.erl This is necessary to avoid a problem with the test module name being used as a generated test name, which is a function name. It lead to a name clash with preexisting module_info/1, which is defined for every module. --- test/should_pass/{module_info.erl => module_info_pass.erl} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename test/should_pass/{module_info.erl => module_info_pass.erl} (92%) diff --git a/test/should_pass/module_info.erl b/test/should_pass/module_info_pass.erl similarity index 92% rename from test/should_pass/module_info.erl rename to test/should_pass/module_info_pass.erl index d4d1c028..8e781741 100644 --- a/test/should_pass/module_info.erl +++ b/test/should_pass/module_info_pass.erl @@ -1,4 +1,4 @@ --module(module_info). +-module(module_info_pass). -compile([export_all, nowarn_export_all]). @@ -18,4 +18,4 @@ unary_direct() -> -spec unary_var() -> atom(). unary_var() -> I = erlang:module_info(module), - I. \ No newline at end of file + I. From b1d8222ed85b71f9d7a240b4613d592c141b038e Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 21:02:30 +0200 Subject: [PATCH 14/20] Don't explicitly enable maybe_expr --- rebar.config | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index 155bd7cc..0de6c2cb 100644 --- a/rebar.config +++ b/rebar.config @@ -4,10 +4,7 @@ {deps, [ {proper, {git, "https://github.com/proper-testing/proper.git", {branch, "master"}}} - ]}, - %% see the maybe expression fail; - %% the VM also needs to be configured to load the module - {erl_opts, [{feature,maybe_expr,enable}]} + ]} ]} ]}. From e345dc10801543beea2c05c5021518ccd0ae8636 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 21:25:36 +0200 Subject: [PATCH 15/20] Add test/known_problems_should_fail_SUITE.erl --- test/known_problems_should_fail_SUITE.erl | 87 +++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 test/known_problems_should_fail_SUITE.erl diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl new file mode 100644 index 00000000..9f1eb42a --- /dev/null +++ b/test/known_problems_should_fail_SUITE.erl @@ -0,0 +1,87 @@ +-module(known_problems_should_fail_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +init_per_suite(Config0) -> + AppBase = code:lib_dir(gradualizer), + Config = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_fail")}, + {dynamic_test_template, known_problems_should_fail_template} + ] ++ Config0, + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(AppBase), + {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), + case all() of + TestNames -> ok; + _ -> ct:fail("Please update all/0 to list all tests") + end, + Config. + +load_prerequisites(AppBase) -> + %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl + gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]), + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +groups() -> + []. + +all() -> + [arith_op,binary_comprehension,case_pattern_should_fail, + exhaustive_argumentwise,exhaustive_expr,exhaustive_map_variants, + exhaustive_remote_map_variants,guard_should_fail,infer_any_pattern, + intersection_with_any_should_fail,intersection_with_unreachable, + lambda_wrong_args,map_refinement_fancy,poly_lists_map_should_fail, + poly_should_fail,recursive_types_should_fail,refine_ty_vars,sample]. + +known_problems_should_fail_template(_@File) -> + Result = safe_type_check_file(_@File, [return_errors]), + case Result of + crash -> + ok; + Errors -> + ErrorsExceptTimeouts = lists:filter( + fun ({_File, {form_check_timeout, _}}) -> false; (_) -> true end, + Errors), + ?assertEqual(0, length(ErrorsExceptTimeouts)) + end. + +safe_type_check_file(File) -> + safe_type_check_file(File, []). + +safe_type_check_file(File, Opts) -> + try + gradualizer:type_check_file(File, Opts) + catch + _:_ -> crash + end. From 314ca8b1d6adacb5116fa08161b0f6d51278a99f Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 21:32:14 +0200 Subject: [PATCH 16/20] Add test/known_problems_should_pass_SUITE.erl --- test/known_problems_should_pass_SUITE.erl | 82 +++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 test/known_problems_should_pass_SUITE.erl diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl new file mode 100644 index 00000000..4a7115b1 --- /dev/null +++ b/test/known_problems_should_pass_SUITE.erl @@ -0,0 +1,82 @@ +-module(known_problems_should_pass_SUITE). + +-compile([export_all, nowarn_export_all]). + +%% EUnit has some handy macros, so let's use it, too +-include_lib("eunit/include/eunit.hrl"). + +%% Test server callbacks +-export([suite/0, + all/0, + groups/0, + init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). + +suite() -> + [{timetrap, {minutes, 10}}]. + +init_per_suite(Config0) -> + AppBase = code:lib_dir(gradualizer), + Config = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_pass")}, + {dynamic_test_template, known_problems_should_pass_template} + ] ++ Config0, + {ok, _} = application:ensure_all_started(gradualizer), + ok = load_prerequisites(AppBase), + {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), + case all() of + TestNames -> ok; + _ -> ct:fail("Please update all/0 to list all tests") + end, + Config. + +load_prerequisites(_AppBase) -> + ok. + +end_per_suite(_Config) -> + ok = application:stop(gradualizer), + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +groups() -> + []. + +all() -> + [arith_op_arg_types,binary_exhaustiveness_checking_should_pass, + call_intersection_function_with_union_arg_should_pass, + different_normalization_levels,elixir_list_first,error_in_guard, + fun_subtyping,generator_var_shadow,inner_union_subtype_of_root_union, + intersection_should_pass,intersection_with_any,list_concat_op_should_pass, + list_tail,map_pattern_duplicate_key,maybe_expr,poly_should_pass, + poly_type_vars,recursive_types,refine_bound_var_on_mismatch, + refine_bound_var_with_guard_should_pass,refine_comparison_should_pass, + refine_list_tail,union_fun]. + +known_problems_should_pass_template(_@File) -> + {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), + ExpectedErrors = typechecker:number_of_exported_functions(Forms), + ReturnedErrors = length(safe_type_check_file(_@File, [return_errors])), + ?assertEqual(ExpectedErrors, ReturnedErrors). + +safe_type_check_file(File) -> + safe_type_check_file(File, []). + +safe_type_check_file(File, Opts) -> + try + gradualizer:type_check_file(File, Opts) + catch + _:_ -> crash + end. From b7d3c32db6a84d4261714a3e02323dec49db0778 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Sun, 2 Jun 2024 21:52:17 +0200 Subject: [PATCH 17/20] Parallelize CT tests --- src/typechecker.erl | 6 ++++-- test/known_problems_should_fail_SUITE.erl | 11 +++++++---- test/known_problems_should_pass_SUITE.erl | 11 +++++++---- test/should_fail_SUITE.erl | 11 +++++++---- test/should_pass_SUITE.erl | 13 ++++++++----- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/typechecker.erl b/src/typechecker.erl index 9e9337c6..bad64b8a 100644 --- a/src/typechecker.erl +++ b/src/typechecker.erl @@ -5746,8 +5746,10 @@ type_check_forms(Forms, Opts) -> %% a Gradualizer (NOT the checked program!) error. -spec type_check_form_with_timeout(expr(), [any()], boolean(), env(), [any()]) -> [any()]. type_check_form_with_timeout(Function, Errors, StopOnFirstError, Env, Opts) -> - %% TODO: make FormCheckTimeOut configurable - FormCheckTimeOut = ?form_check_timeout_ms, + FormCheckTimeOut = case lists:keyfind(form_check_timeout_ms, 1, Opts) of + false -> ?form_check_timeout_ms; + {form_check_timeout_ms, MS} -> MS + end, ?verbose(Env, "Spawning async task...~n", []), Self = self(), Task = fun () -> diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl index 9f1eb42a..c65211b9 100644 --- a/test/known_problems_should_fail_SUITE.erl +++ b/test/known_problems_should_fail_SUITE.erl @@ -26,9 +26,9 @@ init_per_suite(Config0) -> {ok, _} = application:ensure_all_started(gradualizer), ok = load_prerequisites(AppBase), {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all() of + case all_tests() of TestNames -> ok; - _ -> ct:fail("Please update all/0 to list all tests") + _ -> ct:fail("Please update all_tests/0 to list all tests") end, Config. @@ -53,10 +53,13 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. +all() -> + [{group, all_tests}]. + groups() -> - []. + [{all_tests, [parallel], all_tests()}]. -all() -> +all_tests() -> [arith_op,binary_comprehension,case_pattern_should_fail, exhaustive_argumentwise,exhaustive_expr,exhaustive_map_variants, exhaustive_remote_map_variants,guard_should_fail,infer_any_pattern, diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl index 4a7115b1..c97bca8b 100644 --- a/test/known_problems_should_pass_SUITE.erl +++ b/test/known_problems_should_pass_SUITE.erl @@ -26,9 +26,9 @@ init_per_suite(Config0) -> {ok, _} = application:ensure_all_started(gradualizer), ok = load_prerequisites(AppBase), {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all() of + case all_tests() of TestNames -> ok; - _ -> ct:fail("Please update all/0 to list all tests") + _ -> ct:fail("Please update all_tests/0 to list all tests") end, Config. @@ -51,10 +51,13 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. +all() -> + [{group, all_tests}]. + groups() -> - []. + [{all_tests, [parallel], all_tests()}]. -all() -> +all_tests() -> [arith_op_arg_types,binary_exhaustiveness_checking_should_pass, call_intersection_function_with_union_arg_should_pass, different_normalization_levels,elixir_list_first,error_in_guard, diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 40c5b2d4..95011f1b 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -26,9 +26,9 @@ init_per_suite(Config0) -> {ok, _} = application:ensure_all_started(gradualizer), ok = load_prerequisites(AppBase), {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all() of + case all_tests() of TestNames -> ok; - _ -> ct:fail("Please update all/0 to list all tests") + _ -> ct:fail("Please update all_tests/0 to list all tests") end, Config. @@ -56,10 +56,13 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. +all() -> + [{group, all_tests}]. + groups() -> - []. + [{all_tests, [parallel], all_tests()}]. -all() -> +all_tests() -> [annotated_types_fail,arg,arith_op_fail,arity_mismatch, bc_fail,bin_expression,bin_type_error,branch,branch2,call, call_intersection_function_with_union_arg_fail,case_pattern, diff --git a/test/should_pass_SUITE.erl b/test/should_pass_SUITE.erl index b89277f8..b8618ab5 100644 --- a/test/should_pass_SUITE.erl +++ b/test/should_pass_SUITE.erl @@ -26,9 +26,9 @@ init_per_suite(Config0) -> {ok, _} = application:ensure_all_started(gradualizer), ok = load_prerequisites(AppBase), {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all() of + case all_tests() of TestNames -> ok; - _ -> ct:fail("Please update all/0 to list all tests") + _ -> ct:fail("Please update all_tests/0 to list all tests") end, Config. @@ -57,10 +57,13 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. +all() -> + [{group, all_tests}]. + groups() -> - []. + [{all_tests, [parallel], all_tests()}]. -all() -> +all_tests() -> [alias_in_pattern,andalso_any,ann_types,annotated_types,any, any_doesnt_have_type_none_pass,any_pattern,bc_pass, binary_exhaustiveness_checking,binary_in_union,binary_literal_pattern, @@ -101,4 +104,4 @@ all() -> variable_binding_leaks]. should_pass_template(_@File) -> - ?assertEqual(ok, gradualizer:type_check_file(_@File)). + ?assertEqual(ok, gradualizer:type_check_file(_@File, [{form_check_timeout_ms, 2000}])). From 06e50662c8bc2c94354961165426b387fcc3b0ce Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Wed, 5 Jun 2024 00:14:14 +0200 Subject: [PATCH 18/20] Avoid hardcoding names of tests which are to be generated --- test/gradualizer_dynamic_suite.erl | 34 +++++++---- test/known_problems_should_fail_SUITE.erl | 35 ++++------- test/known_problems_should_pass_SUITE.erl | 38 ++++-------- test/should_fail_SUITE.erl | 63 ++++--------------- test/should_pass_SUITE.erl | 73 +++++------------------ 5 files changed, 69 insertions(+), 174 deletions(-) diff --git a/test/gradualizer_dynamic_suite.erl b/test/gradualizer_dynamic_suite.erl index b002d613..4bae68bb 100644 --- a/test/gradualizer_dynamic_suite.erl +++ b/test/gradualizer_dynamic_suite.erl @@ -7,19 +7,25 @@ reload(Config) -> Module = ?config(dynamic_suite_module, Config), - Path = ?config(dynamic_suite_test_path, Config), ?assert(Module /= undefined), - ?assert(Path /= undefined), - Forms = get_forms(Module), - FilesForms = map_erl_files(fun (File) -> - make_test_form(Forms, File, Config) - end, Path), - {TestFiles, TestForms} = lists:unzip(FilesForms), - TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], - ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]), - NewForms = Forms ++ TestForms ++ [{eof, 0}], - {ok, _} = merl:compile_and_load(NewForms), - {ok, TestNames}. + case erlang:function_exported(Module, generated_tests, 0) of + true -> + {ok, Module:generated_tests()}; + false -> + Path = ?config(dynamic_suite_test_path, Config), + ?assert(Path /= undefined), + Forms = get_forms(Module), + FilesForms = map_erl_files(fun (File) -> + make_test_form(Forms, File, Config) + end, Path), + {TestFiles, TestForms} = lists:unzip(FilesForms), + TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ], + ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]), + GeneratedTestsForm = make_generated_tests_form(TestNames), + NewForms = Forms ++ TestForms ++ [GeneratedTestsForm, {eof, 0}], + {ok, _} = merl:compile_and_load(NewForms), + {ok, TestNames} + end. map_erl_files(Fun, Dir) -> Files = filelib:wildcard(filename:join(Dir, "*.erl")), @@ -39,6 +45,10 @@ make_test_form(Forms, File, Config) -> ], erl_syntax:revert(merl:subst(TestTemplate, TestEnv)). +make_generated_tests_form(TestNames) -> + Template = merl:quote("generated_tests() -> _@Body."), + erl_syntax:revert(merl:subst(Template, [{'Body', merl:term(TestNames)}])). + get_forms(Module) -> ModPath = code:which(Module), {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]), diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl index c65211b9..37204f54 100644 --- a/test/known_problems_should_fail_SUITE.erl +++ b/test/known_problems_should_fail_SUITE.erl @@ -16,20 +16,21 @@ suite() -> [{timetrap, {minutes, 10}}]. -init_per_suite(Config0) -> - AppBase = code:lib_dir(gradualizer), +all() -> + [{group, all_tests}]. + +groups() -> Config = [ {dynamic_suite_module, ?MODULE}, - {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_fail")}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_fail")}, {dynamic_test_template, known_problems_should_fail_template} - ] ++ Config0, + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), - ok = load_prerequisites(AppBase), - {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all_tests() of - TestNames -> ok; - _ -> ct:fail("Please update all_tests/0 to list all tests") - end, + ok = load_prerequisites(code:lib_dir(gradualizer)), Config. load_prerequisites(AppBase) -> @@ -53,20 +54,6 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. -all() -> - [{group, all_tests}]. - -groups() -> - [{all_tests, [parallel], all_tests()}]. - -all_tests() -> - [arith_op,binary_comprehension,case_pattern_should_fail, - exhaustive_argumentwise,exhaustive_expr,exhaustive_map_variants, - exhaustive_remote_map_variants,guard_should_fail,infer_any_pattern, - intersection_with_any_should_fail,intersection_with_unreachable, - lambda_wrong_args,map_refinement_fancy,poly_lists_map_should_fail, - poly_should_fail,recursive_types_should_fail,refine_ty_vars,sample]. - known_problems_should_fail_template(_@File) -> Result = safe_type_check_file(_@File, [return_errors]), case Result of diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl index c97bca8b..9d004ab3 100644 --- a/test/known_problems_should_pass_SUITE.erl +++ b/test/known_problems_should_pass_SUITE.erl @@ -16,20 +16,21 @@ suite() -> [{timetrap, {minutes, 10}}]. -init_per_suite(Config0) -> - AppBase = code:lib_dir(gradualizer), +all() -> + [{group, all_tests}]. + +groups() -> Config = [ {dynamic_suite_module, ?MODULE}, - {dynamic_suite_test_path, filename:join(AppBase, "test/known_problems/should_pass")}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_pass")}, {dynamic_test_template, known_problems_should_pass_template} - ] ++ Config0, + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), - ok = load_prerequisites(AppBase), - {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all_tests() of - TestNames -> ok; - _ -> ct:fail("Please update all_tests/0 to list all tests") - end, + ok = load_prerequisites(code:lib_dir(gradualizer)), Config. load_prerequisites(_AppBase) -> @@ -51,23 +52,6 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. -all() -> - [{group, all_tests}]. - -groups() -> - [{all_tests, [parallel], all_tests()}]. - -all_tests() -> - [arith_op_arg_types,binary_exhaustiveness_checking_should_pass, - call_intersection_function_with_union_arg_should_pass, - different_normalization_levels,elixir_list_first,error_in_guard, - fun_subtyping,generator_var_shadow,inner_union_subtype_of_root_union, - intersection_should_pass,intersection_with_any,list_concat_op_should_pass, - list_tail,map_pattern_duplicate_key,maybe_expr,poly_should_pass, - poly_type_vars,recursive_types,refine_bound_var_on_mismatch, - refine_bound_var_with_guard_should_pass,refine_comparison_should_pass, - refine_list_tail,union_fun]. - known_problems_should_pass_template(_@File) -> {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), ExpectedErrors = typechecker:number_of_exported_functions(Forms), diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 95011f1b..cf466653 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -16,20 +16,21 @@ suite() -> [{timetrap, {minutes, 10}}]. -init_per_suite(Config0) -> - AppBase = code:lib_dir(gradualizer), +all() -> + [{group, all_tests}]. + +groups() -> Config = [ {dynamic_suite_module, ?MODULE}, - {dynamic_suite_test_path, filename:join(AppBase, "test/should_fail")}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_fail")}, {dynamic_test_template, should_fail_template} - ] ++ Config0, + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), - ok = load_prerequisites(AppBase), - {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all_tests() of - TestNames -> ok; - _ -> ct:fail("Please update all_tests/0 to list all tests") - end, + ok = load_prerequisites(code:lib_dir(gradualizer)), Config. load_prerequisites(AppBase) -> @@ -56,48 +57,6 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. -all() -> - [{group, all_tests}]. - -groups() -> - [{all_tests, [parallel], all_tests()}]. - -all_tests() -> - [annotated_types_fail,arg,arith_op_fail,arity_mismatch, - bc_fail,bin_expression,bin_type_error,branch,branch2,call, - call_intersection_function_with_union_arg_fail,case_pattern, - case_pattern2,catch_expr_fail,cons,covariant_map_keys_fail, - cyclic_type_vars,depth,exhaustive,exhaustive_float, - exhaustive_list_variants,exhaustive_refinable_map_variants, - exhaustive_remote_user_type,exhaustive_string_variants, - exhaustive_type,exhaustive_user_type, - exhaustiveness_check_toggling,generator,guard_fail, - imported_undef,infer_enabled,intersection_check, - intersection_fail,intersection_infer, - intersection_with_any_fail,iodata_fail,lambda_not_fun, - lc_generator_not_none_fail,lc_not_list,list_infer_fail, - list_op,list_op_should_fail,list_union_fail, - lists_map_nonempty_fail,literal_char,literal_patterns, - logic_op,map_entry,map_fail,map_failing_expr, - map_failing_subtyping,map_field_invalid_update,map_literal, - map_pattern_fail,map_refinement_fail,map_type_error,match, - messaging_fail,module_info_fail,named_fun_fail, - named_fun_infer_fail,nil,no_idempotent_xor, - non_neg_plus_pos_is_pos_fail, - nonempty_list_match_in_head_nonexhaustive, - nonempty_string_fail,opaque_fail,operator_pattern_fail, - pattern,pattern_record_fail,poly_fail,poly_lists_map_fail, - poly_union_lower_bound_fail,pp_intersection,record, - record_exhaustive,record_field,record_index, - record_info_fail,record_refinement_fail,record_update, - record_wildcard_fail,recursive_type_fail, - recursive_types_failing,rel_op,return_fun_fail, - rigid_type_variables_fail,send_fail,shortcut_ops_fail, - spec_and_fun_clause_intersection_fail,string_literal, - tuple_union_arg_fail,tuple_union_fail,tuple_union_pattern, - tuple_union_refinement,type_refinement_fail,unary_op, - unary_plus_fail,union_with_any,unreachable_after_refinement]. - should_fail_template(_@File) -> Errors = gradualizer:type_check_file(_@File, [return_errors]), Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], diff --git a/test/should_pass_SUITE.erl b/test/should_pass_SUITE.erl index b8618ab5..0f2e609a 100644 --- a/test/should_pass_SUITE.erl +++ b/test/should_pass_SUITE.erl @@ -16,20 +16,21 @@ suite() -> [{timetrap, {minutes, 10}}]. -init_per_suite(Config0) -> - AppBase = code:lib_dir(gradualizer), - Config = [ - {dynamic_suite_module, ?MODULE}, - {dynamic_suite_test_path, filename:join(AppBase, "test/should_pass")}, - {dynamic_test_template, should_pass_template} - ] ++ Config0, +all() -> + [{group, all_tests}]. + +groups() -> + Opts = [ + {dynamic_suite_module, ?MODULE}, + {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_pass")}, + {dynamic_test_template, should_pass_template} + ], + {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Opts), + [{all_tests, [parallel], GeneratedTests}]. + +init_per_suite(Config) -> {ok, _} = application:ensure_all_started(gradualizer), - ok = load_prerequisites(AppBase), - {ok, TestNames} = gradualizer_dynamic_suite:reload(Config), - case all_tests() of - TestNames -> ok; - _ -> ct:fail("Please update all_tests/0 to list all tests") - end, + ok = load_prerequisites(code:lib_dir(gradualizer)), Config. load_prerequisites(AppBase) -> @@ -57,51 +58,5 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok. -all() -> - [{group, all_tests}]. - -groups() -> - [{all_tests, [parallel], all_tests()}]. - -all_tests() -> - [alias_in_pattern,andalso_any,ann_types,annotated_types,any, - any_doesnt_have_type_none_pass,any_pattern,bc_pass, - binary_exhaustiveness_checking,binary_in_union,binary_literal_pattern, - bitstring,block_scope,bool,bounded_funs, - call_intersection_function_with_union_arg_pass,'case', - case_of_record_with_user_defined,catch_expr_pass,covariant_map_keys_pass, - cyclic_otp_specs,erlang_error_args_none_pass,exhaustiveness_union_types, - factorial,float,flow,fun_capture,fun_spec,guard,guard_sequences_pass,if_expr, - imported,int,intersection_pass,intersection_with_any_pass,iodata,issue131,lc, - lc_generator_not_none,lc_var_binds_in_filters,list,list_concat_op_pass, - list_exhaustiveness_checking_regressions, - list_exhaustiveness_checking_regressions2, - list_exhaustiveness_checking_unreachable_clause_regression,list_infer_pass, - list_op_pass,listsspecs,map,map_as_argument_update,map_creation, - map_field_valid_update,map_infer_pass,map_passing_expr,map_passing_subtyping, - map_pattern,map_refinement,map_update,map_update_with_record_field, - messaging_pass,minimised_gradualizer_fmt,minus,module_info_higher_arity, - module_info_pass,named_fun_infer_pass,named_fun_pass,negate_none, - nested_pattern_match,non_neg_plus_pos_is_pos_pass,nonempty_cons, - nonempty_list_match_in_head_exhaustive,nonempty_string, - nonexhaustive_record_pattern,opaque,operator_pattern_pass,operator_subtypes, - other_module,pattern_bind_reuse,pattern_record,pattern_with_ty_vars, - poly_lists_map_constraints_pass,poly_lists_map_pass,poly_map_pattern, - poly_pass,poly_pass_infer,poly_pass_no_solve_constraints, - poly_union_lower_bound_pass,preludes,qlc_test,record_info,record_refinement, - record_union_pass,record_union_with_any_should_pass,record_var, - record_wildcard_pass,record_with_user_defined,records, - recursive_call_with_remote_union_return_type_pass,recursive_types_passing, - refine_comparison,refine_mismatch_using_guard_bifs,remote_types, - remote_types_pass,return_fun,rigid_type_variables,rigid_type_variables_pass, - scope,send_pass,sets_set,shortcut_ops_pass, - spec_and_fun_clause_intersection_pass,stuff_as_top,'try',try_expr,tuple, - tuple_union_pass,tuple_union_pat,tuple_union_pattern_pass,type_decl, - type_pattern,type_refinement_pass,type_variable,type_vars_term, - typed_record_field_access,unary_negate_union_with_user_type_pass,unary_plus, - underscore,user_type_in_pattern_body,user_types,var,var_fun,varbind_in_block, - varbind_in_case,varbind_in_function_head,varbind_in_lc,variable_binding, - variable_binding_leaks]. - should_pass_template(_@File) -> ?assertEqual(ok, gradualizer:type_check_file(_@File, [{form_check_timeout_ms, 2000}])). From e6c91a8aea332e381b8627b908168f6928e308e2 Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Wed, 5 Jun 2024 00:16:03 +0200 Subject: [PATCH 19/20] Add a longer timeout in all generated tests --- test/known_problems_should_fail_SUITE.erl | 2 +- test/known_problems_should_pass_SUITE.erl | 2 +- test/should_fail_SUITE.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl index 37204f54..f23124d0 100644 --- a/test/known_problems_should_fail_SUITE.erl +++ b/test/known_problems_should_fail_SUITE.erl @@ -55,7 +55,7 @@ end_per_testcase(_TestCase, _Config) -> ok. known_problems_should_fail_template(_@File) -> - Result = safe_type_check_file(_@File, [return_errors]), + Result = safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]), case Result of crash -> ok; diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl index 9d004ab3..49d17376 100644 --- a/test/known_problems_should_pass_SUITE.erl +++ b/test/known_problems_should_pass_SUITE.erl @@ -55,7 +55,7 @@ end_per_testcase(_TestCase, _Config) -> known_problems_should_pass_template(_@File) -> {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), ExpectedErrors = typechecker:number_of_exported_functions(Forms), - ReturnedErrors = length(safe_type_check_file(_@File, [return_errors])), + ReturnedErrors = length(safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}])), ?assertEqual(ExpectedErrors, ReturnedErrors). safe_type_check_file(File) -> diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index cf466653..8688a725 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -58,7 +58,7 @@ end_per_testcase(_TestCase, _Config) -> ok. should_fail_template(_@File) -> - Errors = gradualizer:type_check_file(_@File, [return_errors]), + Errors = gradualizer:type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]), Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], ?assertEqual(0, length(Timeouts)), %% Test that error formatting doesn't crash From dbd167f6f5ad0b18e8b15bfbd8db483a081d966f Mon Sep 17 00:00:00 2001 From: Radek Szymczyszyn Date: Wed, 5 Jun 2024 00:30:26 +0200 Subject: [PATCH 20/20] Remove unnecessary stubs --- test/known_problems_should_fail_SUITE.erl | 16 +--------------- test/known_problems_should_pass_SUITE.erl | 16 +--------------- test/should_fail_SUITE.erl | 16 +--------------- test/should_pass_SUITE.erl | 16 +--------------- 4 files changed, 4 insertions(+), 60 deletions(-) diff --git a/test/known_problems_should_fail_SUITE.erl b/test/known_problems_should_fail_SUITE.erl index f23124d0..f3ef8811 100644 --- a/test/known_problems_should_fail_SUITE.erl +++ b/test/known_problems_should_fail_SUITE.erl @@ -9,9 +9,7 @@ -export([suite/0, all/0, groups/0, - init_per_suite/1, end_per_suite/1, - init_per_group/2, end_per_group/2, - init_per_testcase/2, end_per_testcase/2]). + init_per_suite/1, end_per_suite/1]). suite() -> [{timetrap, {minutes, 10}}]. @@ -42,18 +40,6 @@ end_per_suite(_Config) -> ok = application:stop(gradualizer), ok. -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, _Config) -> - ok. - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, _Config) -> - ok. - known_problems_should_fail_template(_@File) -> Result = safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]), case Result of diff --git a/test/known_problems_should_pass_SUITE.erl b/test/known_problems_should_pass_SUITE.erl index 49d17376..8d912e35 100644 --- a/test/known_problems_should_pass_SUITE.erl +++ b/test/known_problems_should_pass_SUITE.erl @@ -9,9 +9,7 @@ -export([suite/0, all/0, groups/0, - init_per_suite/1, end_per_suite/1, - init_per_group/2, end_per_group/2, - init_per_testcase/2, end_per_testcase/2]). + init_per_suite/1, end_per_suite/1]). suite() -> [{timetrap, {minutes, 10}}]. @@ -40,18 +38,6 @@ end_per_suite(_Config) -> ok = application:stop(gradualizer), ok. -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, _Config) -> - ok. - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, _Config) -> - ok. - known_problems_should_pass_template(_@File) -> {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []), ExpectedErrors = typechecker:number_of_exported_functions(Forms), diff --git a/test/should_fail_SUITE.erl b/test/should_fail_SUITE.erl index 8688a725..9be8d18c 100644 --- a/test/should_fail_SUITE.erl +++ b/test/should_fail_SUITE.erl @@ -9,9 +9,7 @@ -export([suite/0, all/0, groups/0, - init_per_suite/1, end_per_suite/1, - init_per_group/2, end_per_group/2, - init_per_testcase/2, end_per_testcase/2]). + init_per_suite/1, end_per_suite/1]). suite() -> [{timetrap, {minutes, 10}}]. @@ -45,18 +43,6 @@ end_per_suite(_Config) -> ok = application:stop(gradualizer), ok. -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, _Config) -> - ok. - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, _Config) -> - ok. - should_fail_template(_@File) -> Errors = gradualizer:type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]), Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors], diff --git a/test/should_pass_SUITE.erl b/test/should_pass_SUITE.erl index 0f2e609a..bbd7929a 100644 --- a/test/should_pass_SUITE.erl +++ b/test/should_pass_SUITE.erl @@ -9,9 +9,7 @@ -export([suite/0, all/0, groups/0, - init_per_suite/1, end_per_suite/1, - init_per_group/2, end_per_group/2, - init_per_testcase/2, end_per_testcase/2]). + init_per_suite/1, end_per_suite/1]). suite() -> [{timetrap, {minutes, 10}}]. @@ -46,17 +44,5 @@ end_per_suite(_Config) -> ok = application:stop(gradualizer), ok. -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, _Config) -> - ok. - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, _Config) -> - ok. - should_pass_template(_@File) -> ?assertEqual(ok, gradualizer:type_check_file(_@File, [{form_check_timeout_ms, 2000}])).