diff --git a/results.md b/results.md new file mode 100644 index 00000000..a809c47f --- /dev/null +++ b/results.md @@ -0,0 +1,121 @@ +Julia Version 1.8.0-DEV.1169 +Commit f2f31336a1 (2021-12-22 11:27 UTC) +Platform Info: + OS: Windows (x86_64-w64-mingw32) + CPU: Intel(R) Core(TM) i7-10710U CPU @ 1.10GHz + WORD_SIZE: 64 + LIBM: libopenlibm + LLVM: libLLVM-12.0.1 (ORCJIT, skylake) +Test run at: 2021-12-23T13:30:26.035 + +Maximum number of statements per lowered expression: 10000000 + +| Test file | Passes | Fails | Errors | Broken | Aborted blocks | +| --------- | ------:| -----:| ------:| ------:| --------------:| +| ambiguous | 107 | 0 | 0 | 2 | 0 | +| subarray | 318316 | 0 | 0 | 0 | 0 | +| strings/basic | 87674 | 0 | 0 | 0 | 0 | +| strings/search | 876 | 0 | 0 | 0 | 0 | +| strings/util | 1147 | 0 | 0 | 0 | 0 | +| strings/io | 12764 | 0 | 0 | 0 | 0 | +| strings/types | 2302691 | 0 | 0 | 0 | 0 | +| unicode/utf8 | 19 | 0 | 0 | 0 | 0 | +| core | X | X | X | X | X | +| worlds | ☠️ | ☠️ | ☠️ | ☠️ | ☠️ | +| atomics | 3444 | 0 | 0 | 0 | 0 | +| keywordargs | 151 | 0 | 0 | 0 | 0 | +| numbers | 1578757 | 1 | 0 | 2 | 0 | +| subtype | 337674 | 0 | 0 | 19 | 0 | +| char | 1628 | 0 | 0 | 0 | 0 | +| triplequote | 29 | 0 | 0 | 0 | 0 | +| intrinsics | 301 | 0 | 0 | 0 | 0 | +| dict | 144420 | 0 | 0 | 0 | 0 | +| hashing | 12519 | 0 | 0 | 0 | 0 | +| iobuffer | 209 | 0 | 0 | 0 | 0 | +| staged | 59 | 5 | 0 | 0 | 0 | +| offsetarray | 484 | 3 | 0 | 3 | 0 | +| arrayops | 2031 | 0 | 0 | 2 | 0 | +| tuple | 625 | 0 | 0 | 0 | 0 | +| reduce | 8580 | 2 | 0 | 0 | 0 | +| reducedim | 865 | 0 | 0 | 0 | 0 | +| abstractarray | 55892 | 0 | 0 | 24795 | 0 | +| intfuncs | 227902 | 0 | 0 | 0 | 0 | +| simdloop | 240 | 0 | 0 | 0 | 0 | +| vecelement | 678 | 0 | 0 | 0 | 0 | +| rational | 98639 | 0 | 0 | 1 | 0 | +| bitarray | 914001 | 1 | 0 | 0 | 0 | +| copy | 533 | 0 | 0 | 0 | 0 | +| math | 148967 | 1 | 11 | 0 | 0 | +| fastmath | 946 | 0 | 0 | 0 | 0 | +| functional | 98 | 0 | 0 | 0 | 0 | +| iterators | 10164 | 0 | 0 | 0 | 0 | +| operators | 13039 | 0 | 1 | 0 | 0 | +| ordering | 37 | 0 | 0 | 0 | 0 | +| path | 1051 | 0 | 0 | 12 | 0 | +| ccall | X | X | X | X | X | +| parse | 16098 | 0 | 0 | 0 | 0 | +| loading | 168472 | 2 | 1 | 0 | 0 | +| gmp | 2357 | 0 | 0 | 0 | 0 | +| sorting | 16096 | 0 | 0 | 10 | 0 | +| spawn | 230 | 1 | 1 | 4 | 0 | +| backtrace | 13 | 9 | 16 | 1 | 0 | +| exceptions | 43 | 21 | 6 | 0 | 0 | +| file | X | X | X | X | X | +| read | 3870 | 0 | 0 | 0 | 0 | +| version | 2452 | 0 | 0 | 0 | 0 | +| namedtuple | 215 | 0 | 0 | 0 | 0 | +| mpfr | 1135 | 0 | 0 | 1 | 0 | +| broadcast | 509 | 2 | 0 | 0 | 0 | +| complex | 8432 | 0 | 0 | 5 | 0 | +| floatapprox | 49 | 0 | 0 | 0 | 0 | +| reflection | X | X | X | X | X | +| regex | 130 | 0 | 0 | 0 | 0 | +| float16 | 237 | 0 | 0 | 0 | 0 | +| combinatorics | 170 | 0 | 0 | 0 | 0 | +| sysinfo | 4 | 0 | 0 | 0 | 0 | +| env | 94 | 0 | 0 | 0 | 0 | +| rounding | 112720 | 0 | 0 | 0 | 0 | +| ranges | 12110652 | 2 | 0 | 327682 | 0 | +| mod2pi | 80 | 0 | 0 | 0 | 0 | +| euler | 12 | 0 | 0 | 0 | 0 | +| show | 128879 | 0 | 0 | 8 | 0 | +| client | 2 | 3 | 0 | 0 | 0 | +| errorshow | X | X | X | X | X | +| sets | 3594 | 0 | 0 | 1 | 0 | +| goto | 19 | 0 | 0 | 0 | 0 | +| llvmcall2 | 7 | 0 | 0 | 0 | 0 | +| ryu | 31215 | 0 | 0 | 0 | 0 | +| some | 72 | 0 | 0 | 0 | 0 | +| meta | 69 | 0 | 0 | 0 | 0 | +| stacktraces | X | X | X | X | X | +| docs | 237 | 1 | 0 | 0 | 0 | +| misc | X | X | X | X | X | +| threads | X | X | X | X | X | +| stress | 0 | 0 | 0 | 0 | 0 | +| binaryplatforms | 341 | 0 | 0 | 0 | 0 | +| atexit | 40 | 0 | 0 | 0 | 0 | +| enums | 99 | 0 | 0 | 0 | 0 | +| cmdlineargs | 255 | 0 | 0 | 3 | 0 | +| int | 524698 | 0 | 0 | 0 | 0 | +| interpreter | 3 | 0 | 0 | 0 | 0 | +| checked | 1239 | 0 | 0 | 0 | 0 | +| bitset | 195 | 0 | 0 | 0 | 0 | +| floatfuncs | 215 | 0 | 0 | 0 | 0 | +| precompile | X | X | X | X | X | +| boundscheck | X | X | X | X | X | +| error | 31 | 0 | 0 | 0 | 0 | +| cartesian | 343 | 0 | 0 | 3 | 0 | +| osutils | 57 | 0 | 0 | 0 | 0 | +| channels | 252 | 6 | 0 | 0 | 0 | +| iostream | 50 | 0 | 0 | 0 | 0 | +| secretbuffer | 27 | 0 | 0 | 0 | 0 | +| specificity | X | X | X | X | X | +| reinterpretarray | 232 | 0 | 0 | 0 | 0 | +| syntax | X | X | X | X | X | +| corelogging | 230 | 1 | 0 | 0 | 0 | +| missing | 565 | 0 | 0 | 1 | 0 | +| asyncmap | 304 | 0 | 0 | 0 | 0 | +| smallarrayshrink | 36 | 0 | 0 | 0 | 0 | +| opaque_closure | X | X | X | X | X | +| filesystem | 4 | 0 | 0 | 0 | 0 | +| download | X | X | X | X | X | diff --git a/src/construct.jl b/src/construct.jl index 2b2c3d9b..7ca96bfc 100644 --- a/src/construct.jl +++ b/src/construct.jl @@ -35,6 +35,7 @@ const compiled_modules = Set{Module}() const junk_framedata = FrameData[] # to allow re-use of allocated memory (this is otherwise a bottleneck) const junk_frames = Frame[] +disable_recycle() = true debug_recycle() = false @noinline function _check_frame_not_in_junk(frame) @assert frame.framedata ∉ junk_framedata @@ -43,8 +44,8 @@ end @inline function recycle(frame) debug_recycle() && _check_frame_not_in_junk(frame) - push!(junk_framedata, frame.framedata) - push!(junk_frames, frame) + disable_recycle() || push!(junk_framedata, frame.framedata) + disable_recycle() || push!(junk_frames, frame) end function return_from(frame::Frame) @@ -264,7 +265,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e ng, ns = isa(ssavt, Int) ? ssavt : length(ssavt::Vector{Any}), length(src.slotflags) if length(junk_framedata) > 0 olddata = pop!(junk_framedata) - locals, ssavalues, sparams = olddata.locals, olddata.ssavalues, olddata.sparams + locals, ssavalues, times, sparams = olddata.locals, olddata.ssavalues, olddata.times, olddata.sparams exception_frames, last_reference = olddata.exception_frames, olddata.last_reference last_exception = olddata.last_exception callargs = olddata.callargs @@ -272,6 +273,8 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e fill!(locals, nothing) resize!(ssavalues, 0) resize!(ssavalues, ng) + resize!(times, 0) + resize!(times, ng) # for check_isdefined to work properly, we need sparams to start out unassigned resize!(sparams, 0) empty!(exception_frames) @@ -280,6 +283,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e else locals = Vector{Union{Nothing,Some{Any}}}(nothing, ns) ssavalues = Vector{Any}(undef, ng) + times = Vector{UInt64}(undef, ng) sparams = Vector{Any}(undef, 0) exception_frames = Int[] last_reference = Vector{Int}(undef, ns) @@ -310,7 +314,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e isa(T, TypeVar) && continue # only fill concrete types sparams[i] = T end - FrameData(locals, ssavalues, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs) + FrameData(locals, ssavalues, times, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs) end """ diff --git a/src/interpret.jl b/src/interpret.jl index c366f6a2..3a86a545 100644 --- a/src/interpret.jl +++ b/src/interpret.jl @@ -454,7 +454,8 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev # show_stackloc(frame) # @show node # end - @assert is_leaf(frame) + # The following assertion is hit by Julia subtype tests + # @assert is_leaf(frame) @static VERSION >= v"1.8.0-DEV.370" && coverage_visit_line!(frame) local rhs # For debugging: diff --git a/src/types.jl b/src/types.jl index 510e2112..a05cdb04 100644 --- a/src/types.jl +++ b/src/types.jl @@ -194,6 +194,7 @@ Important fields: struct FrameData locals::Vector{Union{Nothing,Some{Any}}} ssavalues::Vector{Any} + times::Vector{UInt64} sparams::Vector{Any} exception_frames::Vector{Int} last_exception::Base.RefValue{Any} diff --git a/test/juliatests.jl b/test/juliatests.jl index 6b4bd7e3..b35af103 100644 --- a/test/juliatests.jl +++ b/test/juliatests.jl @@ -7,8 +7,8 @@ if !isdefined(Main, :read_and_parse) include("utils.jl") end -const juliadir = dirname(dirname(Sys.BINDIR)) -const testdir = joinpath(juliadir, "test") +const juliadir = dirname(Sys.BINDIR) +const testdir = joinpath(juliadir, "share/julia/test") if isdir(testdir) include(joinpath(testdir, "choosetests.jl")) else @@ -28,7 +28,7 @@ function test_path(test) end end -nstmts = 10^4 # very quick, aborts a lot +nstmts = 10^7 # very quick, aborts a lot outputfile = "results.md" i = 1 while i <= length(ARGS) @@ -45,11 +45,12 @@ while i <= length(ARGS) end end -tests, _, exit_on_error, seed = choosetests(ARGS) +tests, _, exit_on_error, seed = choosetests(["--skip", "llvmcall", "compiler", "stdlib", ARGS...]) function spin_up_worker() p = addprocs(1)[1] - remotecall_wait(include, p, "utils.jl") + remotecall_wait(include, p, "test/utils.jl") + remotecall_wait(JuliaInterpreter.clear_caches, p) remotecall_wait(configure_test, p) return p end @@ -58,29 +59,16 @@ function spin_up_workers(n) procs = addprocs(n) @sync begin @async for p in procs - remotecall_wait(include, p, "utils.jl") + remotecall_wait(include, p, "test/utils.jl") + remotecall_wait(JuliaInterpreter.clear_caches, p) remotecall_wait(configure_test, p) end end return procs end -# Really, we're just going to skip all the tests that run on node1 -const node1_tests = String[] -function move_to_node1(t) - if t in tests - splice!(tests, findfirst(isequal(t), tests)) - push!(node1_tests, t) - end - nothing -end -move_to_node1("precompile") -move_to_node1("SharedArrays") -move_to_node1("stress") -move_to_node1("Distributed") - @testset "Julia tests" begin - nworkers = min(Sys.CPU_THREADS, length(tests)) + nworkers = Threads.nthreads() println("Using $nworkers workers") results = Dict{String,Any}() tests0 = copy(tests) @@ -99,7 +87,8 @@ move_to_node1("Distributed") try resp = disable_sigint() do p = spin_up_worker() - result = remotecall_fetch(run_test_by_eval, p, test, fullpath, nstmts) + # result = remotecall_fetch(run_test_by_limited_eval, p, test, fullpath, nstmts) + result = remotecall_fetch(run_test_by_eval, p, test, fullpath) rmprocs(p; waitfor=5) result end @@ -137,7 +126,7 @@ move_to_node1("Distributed") catch end end - foreach(wait, all_tasks) + foreach(wait, [task for task in all_tasks if isa(task, Task)]) end open(outputfile, "w") do io diff --git a/test/utils.jl b/test/utils.jl index a41e728a..c4b5abe5 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -85,14 +85,14 @@ function evaluate_limited!(@nospecialize(recurse), frame::Frame, nstmts::Int, is nstmts = refnstmts[] elseif istoplevel && stmt.head == :thunk code = stmt.args[1] - if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].args[1], :method) + if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].val, :method) # Julia 1.2+ puts a :thunk before the start of each method new_pc = pc + 1 else refnstmts[] = nstmts - newframe = Frame(moduleof(frame), stmt) + newframe = Frame(moduleof(frame), code) if isa(recurse, Compiled) - finish!(recurse, newframe, true) + JuliaInterpreter.finish!(recurse, newframe, true) else newframe.caller = frame frame.callee = newframe @@ -144,11 +144,108 @@ function limited_exec!(@nospecialize(recurse), newframe, refnstmts, istoplevel) return isa(ret, Aborted) ? ret : something(ret) end +""" + ret = evaluate!(recurse, frame, istoplevel::Bool=true) + +Run `frame` until one of: +- execution terminates normally (`ret = Some{Any}(val)`, where `val` is the returned value of `frame`) +- if `istoplevel` and a `thunk` or `method` expression is encountered (`ret = nothing`) +""" +function evaluate!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + # The following is like finish!, except we intercept :call expressions so that we can run them + # with limexec! rather than the default finish_and_return! + pc = frame.pc + while true + shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc) + stmt = pc_expr(frame, pc) + if isassigned(frame.framedata.times, pc) && frame.framedata.times[pc] > 0 + recurse_pc = Compiled() + else + recurse_pc = recurse + end + time = time_ns() + if isa(stmt, Expr) + if stmt.head == :call && !isa(recurse_pc, Compiled) + try + rhs = evaluate_call!(exec!, frame, stmt) + lhs = SSAValue(pc) + do_assignment!(frame, lhs, rhs) + new_pc = pc + 1 + catch err + new_pc = handle_err(recurse_pc, frame, err) + end + elseif stmt.head == :(=) && isexpr(stmt.args[2], :call) && !isa(recurse_pc, Compiled) + try + rhs = evaluate_call!(exec!, frame, stmt.args[2]) + do_assignment!(frame, stmt.args[1], rhs) + new_pc = pc + 1 + catch err + new_pc = handle_err(recurse_pc, frame, err) + end + elseif istoplevel && stmt.head == :thunk + code = stmt.args[1] + if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].val, :method) + # Julia 1.2+ puts a :thunk before the start of each method + new_pc = pc + 1 + else + newframe = Frame(moduleof(frame), code) + if isa(recurse_pc, Compiled) + JuliaInterpreter.finish!(recurse_pc, newframe, true) + else + newframe.caller = frame + frame.callee = newframe + exec!(recurse_pc, newframe, istoplevel) + frame.callee = nothing + end + JuliaInterpreter.recycle(newframe) + # Because thunks may define new methods, return to toplevel + frame.pc = pc + 1 + return nothing + end + elseif istoplevel && stmt.head == :method && length(stmt.args) == 3 + step_expr!(recurse_pc, frame, stmt, istoplevel) + frame.pc = pc + 1 + return nothing + else + new_pc = step_expr!(recurse_pc, frame, stmt, istoplevel) + end + else + new_pc = step_expr!(recurse_pc, frame, stmt, istoplevel) + end + (new_pc === nothing || isa(new_pc, BreakpointRef)) && break + if !isassigned(frame.framedata.times, pc) + frame.framedata.times[pc] = time_ns() - time + else + frame.framedata.times[pc] += time_ns() - time + end + pc = frame.pc = new_pc + end + # Handle the return + stmt = pc_expr(frame, pc) + ret = get_return(frame) + return Some{Any}(ret) +end + +evaluate!(@nospecialize(recurse), modex::Tuple{Module,Expr,Frame}, istoplevel::Bool=true) = + evaluate!(recurse, modex[end], istoplevel) +evaluate!(@nospecialize(recurse), modex::Tuple{Module,Expr,Expr}, istoplevel::Bool=true) = + Some{Any}(Core.eval(modex[1], modex[3])) + +evaluate!(frame::Union{Frame, Tuple}, istoplevel::Bool=false) = + evaluate!(finish_and_return!, frame, istoplevel) + +function exec!(@nospecialize(recurse), newframe, istoplevel) + ret = evaluate!(recurse, newframe, istoplevel) + return something(ret) +end + ### Functions needed on workers for running tests function configure_test() # To run tests efficiently, certain methods must be run in Compiled mode, # in particular those that are used by the Test infrastructure + cc = JuliaInterpreter.compiled_calls + empty!(cc) cm = JuliaInterpreter.compiled_methods empty!(cm) JuliaInterpreter.set_compiled_methods() @@ -171,7 +268,42 @@ function configure_test() push!(cm, which(SHA.update!, Tuple{SHA.SHA1_CTX,Vector{UInt8}})) end -function run_test_by_eval(test, fullpath, nstmts) +function run_test_by_limited_eval(test, fullpath, nstmts) + Core.eval(Main, Expr(:toplevel, :(module JuliaTests using Test, Random end), quote + # These must be run at top level, so we can't put this in a function + println("Working on ", $test, "...") + ex = read_and_parse($fullpath) + isexpr(ex, :error) && @error "error parsing $($test): $ex" + aborts = Aborted[] + ts = Test.DefaultTestSet($test) + Test.push_testset(ts) + current_task().storage[:SOURCE_PATH] = $fullpath + modexs = collect(ExprSplitter(JuliaTests, ex)) + for (i, modex) in enumerate(modexs) # having the index can be useful for debugging + nstmtsframe = $nstmts + local mod, ex = modex + # @show mod ex + frame = Frame(mod, ex) + while nstmtsframe > 0 + yield() # allow communication between processes + ret, nstmtsleft = evaluate_limited!(frame, nstmtsframe, true) + @assert !isa(ret, Nothing) || nstmtsleft < nstmtsframe + nstmtsframe = nstmtsleft + if isa(ret, Some) + break + elseif isa(ret, Aborted) + push!(aborts, ret) + JuliaInterpreter.finish_stack!(Compiled(), frame, true) + break + end + end + end + println("Finished ", $test) + return ts, aborts + end)) +end + +function run_test_by_eval(test, fullpath) Core.eval(Main, Expr(:toplevel, :(module JuliaTests using Test, Random end), quote # These must be run at top level, so we can't put this in a function println("Working on ", $test, "...") @@ -183,15 +315,15 @@ function run_test_by_eval(test, fullpath, nstmts) current_task().storage[:SOURCE_PATH] = $fullpath modexs = collect(ExprSplitter(JuliaTests, ex)) for (i, modex) in enumerate(modexs) # having the index can be useful for debugging - nstmtsleft = $nstmts - # mod, ex = modex + local mod, ex = modex # @show mod ex - frame = Frame(modex) - yield() # allow communication between processes - ret, nstmtsleft = evaluate_limited!(frame, nstmtsleft, true) - if isa(ret, Aborted) - push!(aborts, ret) - JuliaInterpreter.finish_stack!(Compiled(), frame, true) + frame = Frame(mod, ex) + while true + yield() # allow communication between processes + ret = evaluate!(frame, true) + if isa(ret, Some) + break + end end end println("Finished ", $test)