From a583832f02542fb03453bccc4f0d3404d2b2027d Mon Sep 17 00:00:00 2001 From: goerch Date: Fri, 10 Dec 2021 11:30:12 +0100 Subject: [PATCH 1/7] Fixing Julia tests --- results.md | 222 +++++++++++++++++++++++++++++++++++++++++++++ test/juliatests.jl | 10 +- test/utils.jl | 13 +-- 3 files changed, 234 insertions(+), 11 deletions(-) create mode 100644 results.md diff --git a/results.md b/results.md new file mode 100644 index 00000000..3285e9a3 --- /dev/null +++ b/results.md @@ -0,0 +1,222 @@ +Julia Version 1.7.0 +Commit 3bf9d17731 (2021-11-30 12:12 UTC) +Platform Info: + OS: Windows (x86_64-w64-mingw32) + CPU: Intel(R) Core(TM) i7-10710U CPU @ 1.10GHz + WORD_SIZE: 64 + LIBM: libopenlibm + LLVM: libLLVM-12.0.1 (ORCJIT, skylake) +Environment: + JULIA_EDITOR = code + JULIA_NUM_THREADS = 5 +Test run at: 2021-12-10T11:06:04.727 + +Maximum number of statements per lowered expression: 10000 + +| Test file | Passes | Fails | Errors | Broken | Aborted blocks | +| --------- | ------:| -----:| ------:| ------:| --------------:| +| ambiguous | 34 | 1 | 3 | 0 | 15 | +| compiler/inference | X | X | X | X | X | +| compiler/validation | 17 | 0 | 5 | 0 | 8 | +| compiler/ssair | X | X | X | X | X | +| compiler/irpasses | X | X | X | X | X | +| compiler/codegen | X | X | X | X | X | +| compiler/inline | X | X | X | X | X | +| compiler/contextual | 5 | 1 | 6 | 0 | 8 | +| subarray | 238 | 0 | 21 | 0 | 16 | +| strings/basic | 152 | 0 | 1 | 0 | 17 | +| strings/search | X | X | X | X | X | +| strings/util | 615 | 0 | 0 | 0 | 7 | +| strings/io | 12735 | 0 | 0 | 0 | 4 | +| strings/types | 2302691 | 0 | 0 | 0 | 5 | +| unicode/utf8 | 19 | 0 | 0 | 0 | 1 | +| core | X | X | X | X | X | +| worlds | X | X | X | X | X | +| atomics | X | X | X | X | X | +| keywordargs | 5 | 0 | 11 | 0 | 20 | +| numbers | X | X | X | X | X | +| subtype | X | X | X | X | X | +| char | 1524 | 0 | 2 | 0 | 4 | +| triplequote | 29 | 0 | 0 | 0 | 0 | +| intrinsics | 42 | 0 | 1 | 0 | 1 | +| dict | X | X | X | X | X | +| hashing | X | X | X | X | X | +| iobuffer | 91 | 1 | 1 | 0 | 4 | +| staged | X | X | X | X | X | +| offsetarray | X | X | X | X | X | +| arrayops | X | X | X | X | X | +| tuple | 0 | 0 | 0 | 0 | 3 | +| reduce | 102 | 4 | 12 | 0 | 39 | +| reducedim | X | X | X | X | X | +| abstractarray | X | X | X | X | X | +| intfuncs | X | X | X | X | X | +| simdloop | X | X | X | X | X | +| vecelement | X | X | X | X | X | +| rational | 98633 | 0 | 1 | 1 | 13 | +| bitarray | 2968 | 0 | 1 | 0 | 20 | +| copy | 0 | 0 | 0 | 0 | 4 | +| math | 117538 | 0 | 0 | 0 | 31 | +| fastmath | 928 | 0 | 0 | 0 | 4 | +| functional | 23 | 0 | 0 | 0 | 0 | +| iterators | 5627 | 0 | 4147 | 0 | 12 | +| operators | 45 | 0 | 0 | 0 | 5 | +| ordering | 2 | 0 | 0 | 0 | 0 | +| path | 0 | 0 | 0 | 0 | 0 | +| ccall | X | X | X | X | X | +| parse | X | X | X | X | X | +| loading | X | X | X | X | X | +| gmp | X | X | X | X | X | +| sorting | 97 | 0 | 0 | 0 | 9 | +| spawn | X | X | X | X | X | +| backtrace | X | X | X | X | X | +| exceptions | 5 | 8 | 4 | 0 | 4 | +| file | X | X | X | X | X | +| read | 25 | 1 | 0 | 0 | 3 | +| version | 2452 | 0 | 0 | 0 | 9 | +| namedtuple | 205 | 0 | 2 | 0 | 3 | +| mpfr | 0 | 0 | 0 | 0 | 8 | +| broadcast | 247 | 0 | 2 | 0 | 26 | +| complex | X | X | X | X | X | +| floatapprox | 49 | 0 | 0 | 0 | 2 | +| reflection | X | X | X | X | X | +| regex | 0 | 0 | 0 | 0 | 0 | +| float16 | 762091 | 0 | 0 | 0 | 2 | +| combinatorics | 170 | 0 | 0 | 0 | 3 | +| sysinfo | 4 | 0 | 0 | 0 | 1 | +| env | 1 | 0 | 0 | 0 | 5 | +| rounding | 112552 | 0 | 144 | 0 | 4 | +| mod2pi | X | X | X | X | X | +| euler | X | X | X | X | X | +| show | X | X | X | X | X | +| client | 0 | 4 | 0 | 0 | 2 | +| errorshow | X | X | X | X | X | +| sets | X | X | X | X | X | +| goto | 19 | 0 | 0 | 0 | 0 | +| llvmcall | X | X | X | X | X | +| llvmcall2 | 7 | 0 | 0 | 0 | 1 | +| ryu | 31194 | 0 | 21 | 0 | 1 | +| some | 71 | 0 | 0 | 0 | 0 | +| meta | 57 | 2 | 3 | 0 | 5 | +| stacktraces | X | X | X | X | X | +| docs | X | X | X | X | X | +| misc | X | X | X | X | X | +| threads | X | X | X | X | X | +| binaryplatforms | 4 | 0 | 46 | 0 | 8 | +| atexit | 0 | 0 | 0 | 0 | 0 | +| enums | 99 | 0 | 0 | 0 | 5 | +| cmdlineargs | X | X | X | X | X | +| int | X | X | X | X | X | +| interpreter | 3 | 0 | 0 | 0 | 3 | +| checked | X | X | X | X | X | +| bitset | 195 | 0 | 0 | 0 | 11 | +| floatfuncs | 202 | 0 | 0 | 0 | 2 | +| boundscheck | X | X | X | X | X | +| error | 0 | 0 | 0 | 0 | 0 | +| cartesian | 135 | 0 | 3 | 0 | 5 | +| osutils | 57 | 0 | 0 | 0 | 2 | +| channels | X | X | X | X | X | +| iostream | 0 | 0 | 0 | 0 | 1 | +| secretbuffer | 0 | 0 | 0 | 0 | 0 | +| specificity | X | X | X | X | X | +| reinterpretarray | X | X | X | X | X | +| syntax | X | X | X | X | X | +| corelogging | 0 | 0 | 0 | 0 | 0 | +| missing | 41 | 0 | 0 | 1 | 9 | +| asyncmap | 304 | 2 | 2 | 0 | 9 | +| smallarrayshrink | 36 | 0 | 0 | 0 | 0 | +| opaque_closure | X | X | X | X | X | +| filesystem | 0 | 0 | 0 | 0 | 0 | +| download | X | X | X | X | X | +| SparseArrays/higherorderfns | 11 | 2 | 0 | 0 | 15 | +| SparseArrays/sparse | 502 | 0 | 0 | 0 | 123 | +| SparseArrays/sparsevector | 112 | 0 | 1 | 0 | 33 | +| LinearAlgebra/triangular | X | X | X | X | X | +| LinearAlgebra/qr | 489 | 0 | 0 | 0 | 20 | +| LinearAlgebra/dense | 8571 | 0 | 24 | 0 | 35 | +| LinearAlgebra/matmul | X | X | X | X | X | +| LinearAlgebra/schur | 11 | 0 | 0 | 0 | 4 | +| LinearAlgebra/special | 2514 | 0 | 0 | 0 | 7 | +| LinearAlgebra/eigen | 34 | 0 | 1 | 0 | 8 | +| LinearAlgebra/bunchkaufman | 5688 | 0 | 0 | 0 | 12 | +| LinearAlgebra/svd | 0 | 0 | 0 | 0 | 7 | +| LinearAlgebra/lapack | 798 | 3 | 0 | 0 | 36 | +| LinearAlgebra/tridiag | 21 | 0 | 0 | 0 | 12 | +| LinearAlgebra/bidiag | 695 | 0 | 0 | 0 | 10 | +| LinearAlgebra/diagonal | 2 | 0 | 0 | 0 | 21 | +| LinearAlgebra/cholesky | 143 | 0 | 3 | 0 | 13 | +| LinearAlgebra/lu | 1225 | 3 | 1 | 0 | 15 | +| LinearAlgebra/symmetric | 2605 | 0 | 6 | 0 | 21 | +| LinearAlgebra/generic | 517 | 1 | 2 | 0 | 20 | +| LinearAlgebra/uniformscaling | X | X | X | X | X | +| LinearAlgebra/lq | 2856 | 0 | 0 | 0 | 8 | +| LinearAlgebra/hessenberg | 349 | 0 | 11 | 17 | 2 | +| LinearAlgebra/blas | 30 | 0 | 2 | 0 | 4 | +| LinearAlgebra/adjtrans | 264 | 0 | 0 | 0 | 14 | +| LinearAlgebra/pinv | 292 | 0 | 0 | 0 | 1 | +| LinearAlgebra/givens | 7 | 0 | 0 | 0 | 0 | +| LinearAlgebra/structuredbroadcast | 645 | 0 | 0 | 0 | 6 | +| LinearAlgebra/addmul | X | X | X | X | X | +| LinearAlgebra/ldlt | 7 | 0 | 0 | 0 | 3 | +| LinearAlgebra/factorization | 6 | 0 | 0 | 0 | 1 | +| LibGit2/libgit2 | 13 | 0 | 0 | 0 | 5 | +| Dates/accessors | 7723858 | 0 | 0 | 0 | 6 | +| Dates/adjusters | X | X | X | X | X | +| Dates/query | X | X | X | X | X | +| Dates/periods | 227 | 0 | 0 | 0 | 14 | +| Dates/ranges | 350537 | 0 | 1 | 0 | 10 | +| Dates/rounding | X | X | X | X | X | +| Dates/types | 231 | 0 | 0 | 0 | 3 | +| Dates/io | 222 | 0 | 0 | 0 | 19 | +| Dates/arithmetic | 376 | 0 | 1 | 0 | 4 | +| Dates/conversions | 160 | 0 | 0 | 0 | 2 | +| ArgTools | 2 | 5 | 6 | 0 | 3 | +| Artifacts | 0 | 0 | 0 | 0 | 3 | +| Base64 | 1006 | 0 | 1001 | 0 | 4 | +| CRC32c | X | X | X | X | X | +| CompilerSupportLibraries_jll | 4 | 0 | 0 | 0 | 0 | +| DelimitedFiles | X | X | X | X | X | +| Downloads | 144 | 0 | 15 | 0 | 1 | +| FileWatching | X | X | X | X | X | +| Future | 0 | 0 | 0 | 0 | 0 | +| GMP_jll | 0 | 0 | 1 | 0 | 1 | +| InteractiveUtils | X | X | X | X | X | +| LLVMLibUnwind_jll | 0 | 0 | 0 | 0 | 0 | +| LazyArtifacts | 1 | 0 | 0 | 0 | 1 | +| LibCURL | 0 | 0 | 0 | 0 | 0 | +| LibCURL_jll | 1 | 0 | 0 | 0 | 0 | +| LibGit2_jll | 2 | 0 | 0 | 0 | 0 | +| LibSSH2_jll | 0 | 0 | 0 | 0 | 0 | +| LibUV_jll | 1 | 0 | 0 | 0 | 0 | +| LibUnwind_jll | 0 | 0 | 0 | 0 | 0 | +| Libdl | 69 | 0 | 0 | 0 | 1 | +| Logging | 8 | 0 | 1 | 0 | 1 | +| MPFR_jll | 1 | 0 | 0 | 0 | 0 | +| Markdown | 241 | 1 | 0 | 0 | 82 | +| MbedTLS_jll | 1 | 0 | 0 | 0 | 0 | +| Mmap | 134 | 2 | 0 | 0 | 63 | +| MozillaCACerts_jll | 1 | 0 | 0 | 0 | 0 | +| NetworkOptions | 2575 | 0 | 929 | 0 | 4 | +| OpenBLAS_jll | 1 | 0 | 0 | 0 | 0 | +| OpenLibm_jll | 1 | 0 | 0 | 0 | 0 | +| PCRE2_jll | 2 | 0 | 0 | 0 | 0 | +| Printf | 669 | 0 | 1 | 0 | 1 | +| Profile | X | X | X | X | X | +| REPL | 1250 | 0 | 0 | 4 | 0 | +| Random | 3701 | 0 | 0 | 0 | 44 | +| SHA | X | X | X | X | X | +| Serialization | X | X | X | X | X | +| Sockets | X | X | X | X | X | +| Statistics | 713 | 0 | 6 | 0 | 18 | +| SuiteSparse | 921 | 0 | 0 | 0 | 1 | +| SuiteSparse_jll | 1 | 0 | 0 | 0 | 0 | +| TOML | 372 | 0 | 43 | 8 | 0 | +| Tar | 78 | 0 | 40 | 0 | 13 | +| Test | X | X | X | X | X | +| UUIDs | 1029 | 0 | 0 | 0 | 20 | +| Unicode | 19 | 0 | 0 | 0 | 6 | +| Zlib_jll | 1 | 0 | 0 | 0 | 0 | +| dSFMT_jll | 1 | 0 | 0 | 0 | 0 | +| libLLVM_jll | 1 | 0 | 0 | 0 | 0 | +| libblastrampoline_jll | 1 | 0 | 0 | 0 | 0 | +| nghttp2_jll | 1 | 0 | 0 | 0 | 0 | +| p7zip_jll | 1 | 0 | 0 | 0 | 0 | diff --git a/test/juliatests.jl b/test/juliatests.jl index 6b4bd7e3..dfc08982 100644 --- a/test/juliatests.jl +++ b/test/juliatests.jl @@ -7,8 +7,8 @@ if !isdefined(Main, :read_and_parse) include("utils.jl") end -const juliadir = dirname(dirname(Sys.BINDIR)) -const testdir = joinpath(juliadir, "test") +const juliadir = dirname(Sys.BINDIR) +const testdir = joinpath(juliadir, "share/julia/test") if isdir(testdir) include(joinpath(testdir, "choosetests.jl")) else @@ -49,7 +49,7 @@ tests, _, exit_on_error, seed = choosetests(ARGS) function spin_up_worker() p = addprocs(1)[1] - remotecall_wait(include, p, "utils.jl") + remotecall_wait(include, p, "test/utils.jl") remotecall_wait(configure_test, p) return p end @@ -58,7 +58,7 @@ function spin_up_workers(n) procs = addprocs(n) @sync begin @async for p in procs - remotecall_wait(include, p, "utils.jl") + remotecall_wait(include, p, "test/utils.jl") remotecall_wait(configure_test, p) end end @@ -80,7 +80,7 @@ move_to_node1("stress") move_to_node1("Distributed") @testset "Julia tests" begin - nworkers = min(Sys.CPU_THREADS, length(tests)) + nworkers = Threads.nthreads()-1 println("Using $nworkers workers") results = Dict{String,Any}() tests0 = copy(tests) diff --git a/test/utils.jl b/test/utils.jl index a41e728a..8d6c56e8 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -85,21 +85,22 @@ function evaluate_limited!(@nospecialize(recurse), frame::Frame, nstmts::Int, is nstmts = refnstmts[] elseif istoplevel && stmt.head == :thunk code = stmt.args[1] - if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].args[1], :method) + if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].val, :method) # Julia 1.2+ puts a :thunk before the start of each method new_pc = pc + 1 else refnstmts[] = nstmts - newframe = Frame(moduleof(frame), stmt) if isa(recurse, Compiled) - finish!(recurse, newframe, true) + newframe = Frame(moduleof(frame), stmt) + JuliaInterpreter.finish!(recurse, newframe, true) else + newframe = Frame(moduleof(frame), code) newframe.caller = frame frame.callee = newframe ret = limited_exec!(recurse, newframe, refnstmts, istoplevel) isa(ret, Aborted) && return ret, refnstmts[] frame.callee = nothing - end + end JuliaInterpreter.recycle(newframe) # Because thunks may define new methods, return to toplevel frame.pc = pc + 1 @@ -184,9 +185,9 @@ function run_test_by_eval(test, fullpath, nstmts) modexs = collect(ExprSplitter(JuliaTests, ex)) for (i, modex) in enumerate(modexs) # having the index can be useful for debugging nstmtsleft = $nstmts - # mod, ex = modex + mod, ex = modex # @show mod ex - frame = Frame(modex) + frame = Frame(mod, ex) yield() # allow communication between processes ret, nstmtsleft = evaluate_limited!(frame, nstmtsleft, true) if isa(ret, Aborted) From 7606d4705eedfd2e3392e9176306ce1a91f523b8 Mon Sep 17 00:00:00 2001 From: goerch Date: Fri, 10 Dec 2021 11:57:57 +0100 Subject: [PATCH 2/7] Trying an alternative fix for evaluate_limited! seems to make no significant difference. So is this the correct handling for a thunk? ``` newframe = Frame(moduleof(frame), code) if isa(recurse, Compiled) JuliaInterpreter.finish!(recurse, newframe, true) else newframe.caller = frame frame.callee = newframe ret = limited_exec!(recurse, newframe, refnstmts, istoplevel) isa(ret, Aborted) && return ret, refnstmts[] frame.callee = nothing end ``` --- results.md | 4 ++-- test/utils.jl | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/results.md b/results.md index 3285e9a3..2a92c6d7 100644 --- a/results.md +++ b/results.md @@ -9,7 +9,7 @@ Platform Info: Environment: JULIA_EDITOR = code JULIA_NUM_THREADS = 5 -Test run at: 2021-12-10T11:06:04.727 +Test run at: 2021-12-10T11:54:53.288 Maximum number of statements per lowered expression: 10000 @@ -128,7 +128,7 @@ Maximum number of statements per lowered expression: 10000 | filesystem | 0 | 0 | 0 | 0 | 0 | | download | X | X | X | X | X | | SparseArrays/higherorderfns | 11 | 2 | 0 | 0 | 15 | -| SparseArrays/sparse | 502 | 0 | 0 | 0 | 123 | +| SparseArrays/sparse | 502 | 0 | 0 | 0 | 122 | | SparseArrays/sparsevector | 112 | 0 | 1 | 0 | 33 | | LinearAlgebra/triangular | X | X | X | X | X | | LinearAlgebra/qr | 489 | 0 | 0 | 0 | 20 | diff --git a/test/utils.jl b/test/utils.jl index 8d6c56e8..d9fbc668 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -90,11 +90,10 @@ function evaluate_limited!(@nospecialize(recurse), frame::Frame, nstmts::Int, is new_pc = pc + 1 else refnstmts[] = nstmts + newframe = Frame(moduleof(frame), code) if isa(recurse, Compiled) - newframe = Frame(moduleof(frame), stmt) JuliaInterpreter.finish!(recurse, newframe, true) else - newframe = Frame(moduleof(frame), code) newframe.caller = frame frame.callee = newframe ret = limited_exec!(recurse, newframe, refnstmts, istoplevel) From 064c24018d77d5acb8a97d79f5af78c762abc463 Mon Sep 17 00:00:00 2001 From: goerch Date: Sun, 12 Dec 2021 00:15:01 +0100 Subject: [PATCH 3/7] Fixing run_test_by_eval and increasing test isolation --- results.md | 257 +++++++++++++++------------------------------ test/juliatests.jl | 23 ++-- test/utils.jl | 20 ++-- 3 files changed, 103 insertions(+), 197 deletions(-) diff --git a/results.md b/results.md index 2a92c6d7..f142237b 100644 --- a/results.md +++ b/results.md @@ -9,214 +9,123 @@ Platform Info: Environment: JULIA_EDITOR = code JULIA_NUM_THREADS = 5 -Test run at: 2021-12-10T11:54:53.288 +Test run at: 2021-12-12T00:11:31.538 -Maximum number of statements per lowered expression: 10000 +Maximum number of statements per lowered expression: 100000 | Test file | Passes | Fails | Errors | Broken | Aborted blocks | | --------- | ------:| -----:| ------:| ------:| --------------:| -| ambiguous | 34 | 1 | 3 | 0 | 15 | -| compiler/inference | X | X | X | X | X | -| compiler/validation | 17 | 0 | 5 | 0 | 8 | +| ambiguous | 105 | 0 | 1 | 2 | 14 | +| compiler/inference | 997 | 5 | 2 | 2 | 3 | +| compiler/validation | 24 | 0 | 1 | 0 | 0 | | compiler/ssair | X | X | X | X | X | | compiler/irpasses | X | X | X | X | X | | compiler/codegen | X | X | X | X | X | -| compiler/inline | X | X | X | X | X | -| compiler/contextual | 5 | 1 | 6 | 0 | 8 | -| subarray | 238 | 0 | 21 | 0 | 16 | -| strings/basic | 152 | 0 | 1 | 0 | 17 | -| strings/search | X | X | X | X | X | -| strings/util | 615 | 0 | 0 | 0 | 7 | -| strings/io | 12735 | 0 | 0 | 0 | 4 | -| strings/types | 2302691 | 0 | 0 | 0 | 5 | -| unicode/utf8 | 19 | 0 | 0 | 0 | 1 | +| compiler/inline | 63 | 2 | 0 | 1 | 0 | +| compiler/contextual | 9 | 2 | 1 | 0 | 1 | +| subarray | 318276 | 0 | 1 | 0 | 5 | +| strings/basic | 87674 | 0 | 0 | 0 | 5 | +| strings/search | 692 | 0 | 0 | 0 | 1 | +| strings/util | 619 | 0 | 0 | 0 | 6 | +| strings/io | 12764 | 0 | 0 | 0 | 2 | +| strings/types | 2302691 | 0 | 0 | 0 | 4 | +| unicode/utf8 | 19 | 0 | 0 | 0 | 0 | | core | X | X | X | X | X | -| worlds | X | X | X | X | X | -| atomics | X | X | X | X | X | -| keywordargs | 5 | 0 | 11 | 0 | 20 | -| numbers | X | X | X | X | X | +| atomics | 3444 | 0 | 0 | 0 | 0 | +| keywordargs | 151 | 0 | 0 | 0 | 0 | +| numbers | 1577509 | 0 | 1 | 1 | 16 | | subtype | X | X | X | X | X | -| char | 1524 | 0 | 2 | 0 | 4 | +| char | 1625 | 0 | 0 | 0 | 3 | | triplequote | 29 | 0 | 0 | 0 | 0 | -| intrinsics | 42 | 0 | 1 | 0 | 1 | +| intrinsics | 301 | 0 | 0 | 0 | 0 | | dict | X | X | X | X | X | -| hashing | X | X | X | X | X | -| iobuffer | 91 | 1 | 1 | 0 | 4 | -| staged | X | X | X | X | X | +| hashing | 12521 | 0 | 0 | 0 | 7 | +| iobuffer | 205 | 0 | 0 | 0 | 1 | +| staged | 58 | 6 | 0 | 0 | 0 | | offsetarray | X | X | X | X | X | -| arrayops | X | X | X | X | X | -| tuple | 0 | 0 | 0 | 0 | 3 | -| reduce | 102 | 4 | 12 | 0 | 39 | -| reducedim | X | X | X | X | X | -| abstractarray | X | X | X | X | X | -| intfuncs | X | X | X | X | X | +| arrayops | 1996 | 2 | 5 | 2 | 24 | +| tuple | 606 | 0 | 0 | 0 | 0 | +| reduce | 8578 | 2 | 0 | 0 | 13 | +| reducedim | 865 | 0 | 0 | 0 | 14 | +| abstractarray | 55292 | 0 | 1 | 24795 | 8 | +| intfuncs | 215863 | 0 | 1 | 0 | 6 | | simdloop | X | X | X | X | X | -| vecelement | X | X | X | X | X | -| rational | 98633 | 0 | 1 | 1 | 13 | -| bitarray | 2968 | 0 | 1 | 0 | 20 | -| copy | 0 | 0 | 0 | 0 | 4 | -| math | 117538 | 0 | 0 | 0 | 31 | -| fastmath | 928 | 0 | 0 | 0 | 4 | -| functional | 23 | 0 | 0 | 0 | 0 | -| iterators | 5627 | 0 | 4147 | 0 | 12 | -| operators | 45 | 0 | 0 | 0 | 5 | -| ordering | 2 | 0 | 0 | 0 | 0 | -| path | 0 | 0 | 0 | 0 | 0 | +| vecelement | 678 | 0 | 0 | 0 | 1 | +| rational | 98633 | 0 | 1 | 1 | 7 | +| bitarray | 381657 | 0 | 3661 | 0 | 23 | +| copy | 331 | 0 | 2 | 0 | 2 | +| math | X | X | X | X | X | +| fastmath | 946 | 0 | 0 | 0 | 1 | +| functional | 98 | 0 | 0 | 0 | 0 | +| iterators | 10080 | 0 | 0 | 0 | 4 | +| operators | 13039 | 0 | 0 | 0 | 2 | +| ordering | 35 | 0 | 0 | 0 | 0 | +| path | 1051 | 0 | 0 | 12 | 2 | | ccall | X | X | X | X | X | -| parse | X | X | X | X | X | -| loading | X | X | X | X | X | -| gmp | X | X | X | X | X | -| sorting | 97 | 0 | 0 | 0 | 9 | +| parse | 16098 | 0 | 0 | 0 | 4 | +| loading | 153389 | 2 | 1 | 0 | 15 | +| gmp | 2323 | 1 | 0 | 0 | 6 | +| sorting | 12649 | 0 | 8 | 10 | 8 | | spawn | X | X | X | X | X | -| backtrace | X | X | X | X | X | -| exceptions | 5 | 8 | 4 | 0 | 4 | +| backtrace | 8 | 11 | 17 | 1 | 2 | +| exceptions | 38 | 24 | 7 | 0 | 3 | | file | X | X | X | X | X | -| read | 25 | 1 | 0 | 0 | 3 | -| version | 2452 | 0 | 0 | 0 | 9 | -| namedtuple | 205 | 0 | 2 | 0 | 3 | -| mpfr | 0 | 0 | 0 | 0 | 8 | -| broadcast | 247 | 0 | 2 | 0 | 26 | -| complex | X | X | X | X | X | -| floatapprox | 49 | 0 | 0 | 0 | 2 | +| read | 3733 | 0 | 0 | 0 | 3 | +| version | 2452 | 0 | 0 | 0 | 1 | +| namedtuple | 214 | 0 | 0 | 0 | 1 | +| mpfr | 1099 | 13 | 15 | 1 | 6 | +| broadcast | 507 | 1 | 0 | 0 | 6 | +| complex | 8432 | 0 | 0 | 5 | 7 | +| floatapprox | 49 | 0 | 0 | 0 | 0 | | reflection | X | X | X | X | X | -| regex | 0 | 0 | 0 | 0 | 0 | -| float16 | 762091 | 0 | 0 | 0 | 2 | -| combinatorics | 170 | 0 | 0 | 0 | 3 | -| sysinfo | 4 | 0 | 0 | 0 | 1 | -| env | 1 | 0 | 0 | 0 | 5 | -| rounding | 112552 | 0 | 144 | 0 | 4 | -| mod2pi | X | X | X | X | X | -| euler | X | X | X | X | X | +| regex | 128 | 0 | 2 | 0 | 1 | +| float16 | 762091 | 0 | 0 | 0 | 1 | +| combinatorics | 170 | 0 | 0 | 0 | 2 | +| sysinfo | 4 | 0 | 0 | 0 | 0 | +| env | 96 | 0 | 1 | 0 | 4 | +| rounding | 112720 | 0 | 0 | 0 | 3 | +| ranges | 12110676 | 5 | 26 | 327699 | 19 | +| mod2pi | 80 | 0 | 0 | 0 | 0 | +| euler | 12 | 0 | 0 | 0 | 6 | | show | X | X | X | X | X | -| client | 0 | 4 | 0 | 0 | 2 | +| client | 0 | 4 | 0 | 0 | 1 | | errorshow | X | X | X | X | X | | sets | X | X | X | X | X | | goto | 19 | 0 | 0 | 0 | 0 | | llvmcall | X | X | X | X | X | -| llvmcall2 | 7 | 0 | 0 | 0 | 1 | +| llvmcall2 | 7 | 0 | 0 | 0 | 0 | | ryu | 31194 | 0 | 21 | 0 | 1 | | some | 71 | 0 | 0 | 0 | 0 | -| meta | 57 | 2 | 3 | 0 | 5 | +| meta | 66 | 2 | 1 | 0 | 0 | | stacktraces | X | X | X | X | X | | docs | X | X | X | X | X | | misc | X | X | X | X | X | | threads | X | X | X | X | X | -| binaryplatforms | 4 | 0 | 46 | 0 | 8 | -| atexit | 0 | 0 | 0 | 0 | 0 | -| enums | 99 | 0 | 0 | 0 | 5 | +| stress | 0 | 0 | 0 | 0 | 0 | +| binaryplatforms | 334 | 0 | 7 | 0 | 7 | +| atexit | 40 | 0 | 0 | 0 | 1 | +| enums | 99 | 0 | 0 | 0 | 0 | | cmdlineargs | X | X | X | X | X | | int | X | X | X | X | X | -| interpreter | 3 | 0 | 0 | 0 | 3 | -| checked | X | X | X | X | X | -| bitset | 195 | 0 | 0 | 0 | 11 | -| floatfuncs | 202 | 0 | 0 | 0 | 2 | +| interpreter | 3 | 0 | 0 | 0 | 0 | +| checked | 1239 | 0 | 0 | 0 | 1 | +| bitset | 195 | 0 | 0 | 0 | 3 | +| floatfuncs | 208 | 0 | 13 | 0 | 2 | +| precompile | X | X | X | X | X | | boundscheck | X | X | X | X | X | -| error | 0 | 0 | 0 | 0 | 0 | -| cartesian | 135 | 0 | 3 | 0 | 5 | -| osutils | 57 | 0 | 0 | 0 | 2 | +| error | 31 | 0 | 0 | 0 | 0 | +| cartesian | 236 | 0 | 1 | 3 | 3 | +| osutils | 57 | 0 | 0 | 0 | 0 | | channels | X | X | X | X | X | -| iostream | 0 | 0 | 0 | 0 | 1 | -| secretbuffer | 0 | 0 | 0 | 0 | 0 | +| iostream | 50 | 0 | 0 | 0 | 1 | +| secretbuffer | X | X | X | X | X | | specificity | X | X | X | X | X | -| reinterpretarray | X | X | X | X | X | +| reinterpretarray | 232 | 0 | 0 | 0 | 3 | | syntax | X | X | X | X | X | -| corelogging | 0 | 0 | 0 | 0 | 0 | -| missing | 41 | 0 | 0 | 1 | 9 | -| asyncmap | 304 | 2 | 2 | 0 | 9 | +| corelogging | 55 | 2 | 21 | 0 | 1 | +| missing | 563 | 0 | 2 | 1 | 3 | +| asyncmap | X | X | X | X | X | | smallarrayshrink | 36 | 0 | 0 | 0 | 0 | | opaque_closure | X | X | X | X | X | -| filesystem | 0 | 0 | 0 | 0 | 0 | +| filesystem | 4 | 0 | 0 | 0 | 0 | | download | X | X | X | X | X | -| SparseArrays/higherorderfns | 11 | 2 | 0 | 0 | 15 | -| SparseArrays/sparse | 502 | 0 | 0 | 0 | 122 | -| SparseArrays/sparsevector | 112 | 0 | 1 | 0 | 33 | -| LinearAlgebra/triangular | X | X | X | X | X | -| LinearAlgebra/qr | 489 | 0 | 0 | 0 | 20 | -| LinearAlgebra/dense | 8571 | 0 | 24 | 0 | 35 | -| LinearAlgebra/matmul | X | X | X | X | X | -| LinearAlgebra/schur | 11 | 0 | 0 | 0 | 4 | -| LinearAlgebra/special | 2514 | 0 | 0 | 0 | 7 | -| LinearAlgebra/eigen | 34 | 0 | 1 | 0 | 8 | -| LinearAlgebra/bunchkaufman | 5688 | 0 | 0 | 0 | 12 | -| LinearAlgebra/svd | 0 | 0 | 0 | 0 | 7 | -| LinearAlgebra/lapack | 798 | 3 | 0 | 0 | 36 | -| LinearAlgebra/tridiag | 21 | 0 | 0 | 0 | 12 | -| LinearAlgebra/bidiag | 695 | 0 | 0 | 0 | 10 | -| LinearAlgebra/diagonal | 2 | 0 | 0 | 0 | 21 | -| LinearAlgebra/cholesky | 143 | 0 | 3 | 0 | 13 | -| LinearAlgebra/lu | 1225 | 3 | 1 | 0 | 15 | -| LinearAlgebra/symmetric | 2605 | 0 | 6 | 0 | 21 | -| LinearAlgebra/generic | 517 | 1 | 2 | 0 | 20 | -| LinearAlgebra/uniformscaling | X | X | X | X | X | -| LinearAlgebra/lq | 2856 | 0 | 0 | 0 | 8 | -| LinearAlgebra/hessenberg | 349 | 0 | 11 | 17 | 2 | -| LinearAlgebra/blas | 30 | 0 | 2 | 0 | 4 | -| LinearAlgebra/adjtrans | 264 | 0 | 0 | 0 | 14 | -| LinearAlgebra/pinv | 292 | 0 | 0 | 0 | 1 | -| LinearAlgebra/givens | 7 | 0 | 0 | 0 | 0 | -| LinearAlgebra/structuredbroadcast | 645 | 0 | 0 | 0 | 6 | -| LinearAlgebra/addmul | X | X | X | X | X | -| LinearAlgebra/ldlt | 7 | 0 | 0 | 0 | 3 | -| LinearAlgebra/factorization | 6 | 0 | 0 | 0 | 1 | -| LibGit2/libgit2 | 13 | 0 | 0 | 0 | 5 | -| Dates/accessors | 7723858 | 0 | 0 | 0 | 6 | -| Dates/adjusters | X | X | X | X | X | -| Dates/query | X | X | X | X | X | -| Dates/periods | 227 | 0 | 0 | 0 | 14 | -| Dates/ranges | 350537 | 0 | 1 | 0 | 10 | -| Dates/rounding | X | X | X | X | X | -| Dates/types | 231 | 0 | 0 | 0 | 3 | -| Dates/io | 222 | 0 | 0 | 0 | 19 | -| Dates/arithmetic | 376 | 0 | 1 | 0 | 4 | -| Dates/conversions | 160 | 0 | 0 | 0 | 2 | -| ArgTools | 2 | 5 | 6 | 0 | 3 | -| Artifacts | 0 | 0 | 0 | 0 | 3 | -| Base64 | 1006 | 0 | 1001 | 0 | 4 | -| CRC32c | X | X | X | X | X | -| CompilerSupportLibraries_jll | 4 | 0 | 0 | 0 | 0 | -| DelimitedFiles | X | X | X | X | X | -| Downloads | 144 | 0 | 15 | 0 | 1 | -| FileWatching | X | X | X | X | X | -| Future | 0 | 0 | 0 | 0 | 0 | -| GMP_jll | 0 | 0 | 1 | 0 | 1 | -| InteractiveUtils | X | X | X | X | X | -| LLVMLibUnwind_jll | 0 | 0 | 0 | 0 | 0 | -| LazyArtifacts | 1 | 0 | 0 | 0 | 1 | -| LibCURL | 0 | 0 | 0 | 0 | 0 | -| LibCURL_jll | 1 | 0 | 0 | 0 | 0 | -| LibGit2_jll | 2 | 0 | 0 | 0 | 0 | -| LibSSH2_jll | 0 | 0 | 0 | 0 | 0 | -| LibUV_jll | 1 | 0 | 0 | 0 | 0 | -| LibUnwind_jll | 0 | 0 | 0 | 0 | 0 | -| Libdl | 69 | 0 | 0 | 0 | 1 | -| Logging | 8 | 0 | 1 | 0 | 1 | -| MPFR_jll | 1 | 0 | 0 | 0 | 0 | -| Markdown | 241 | 1 | 0 | 0 | 82 | -| MbedTLS_jll | 1 | 0 | 0 | 0 | 0 | -| Mmap | 134 | 2 | 0 | 0 | 63 | -| MozillaCACerts_jll | 1 | 0 | 0 | 0 | 0 | -| NetworkOptions | 2575 | 0 | 929 | 0 | 4 | -| OpenBLAS_jll | 1 | 0 | 0 | 0 | 0 | -| OpenLibm_jll | 1 | 0 | 0 | 0 | 0 | -| PCRE2_jll | 2 | 0 | 0 | 0 | 0 | -| Printf | 669 | 0 | 1 | 0 | 1 | -| Profile | X | X | X | X | X | -| REPL | 1250 | 0 | 0 | 4 | 0 | -| Random | 3701 | 0 | 0 | 0 | 44 | -| SHA | X | X | X | X | X | -| Serialization | X | X | X | X | X | -| Sockets | X | X | X | X | X | -| Statistics | 713 | 0 | 6 | 0 | 18 | -| SuiteSparse | 921 | 0 | 0 | 0 | 1 | -| SuiteSparse_jll | 1 | 0 | 0 | 0 | 0 | -| TOML | 372 | 0 | 43 | 8 | 0 | -| Tar | 78 | 0 | 40 | 0 | 13 | -| Test | X | X | X | X | X | -| UUIDs | 1029 | 0 | 0 | 0 | 20 | -| Unicode | 19 | 0 | 0 | 0 | 6 | -| Zlib_jll | 1 | 0 | 0 | 0 | 0 | -| dSFMT_jll | 1 | 0 | 0 | 0 | 0 | -| libLLVM_jll | 1 | 0 | 0 | 0 | 0 | -| libblastrampoline_jll | 1 | 0 | 0 | 0 | 0 | -| nghttp2_jll | 1 | 0 | 0 | 0 | 0 | -| p7zip_jll | 1 | 0 | 0 | 0 | 0 | diff --git a/test/juliatests.jl b/test/juliatests.jl index dfc08982..54a89891 100644 --- a/test/juliatests.jl +++ b/test/juliatests.jl @@ -28,7 +28,7 @@ function test_path(test) end end -nstmts = 10^4 # very quick, aborts a lot +nstmts = 10^5 # very quick, aborts a lot outputfile = "results.md" i = 1 while i <= length(ARGS) @@ -50,6 +50,7 @@ tests, _, exit_on_error, seed = choosetests(ARGS) function spin_up_worker() p = addprocs(1)[1] remotecall_wait(include, p, "test/utils.jl") + remotecall_wait(JuliaInterpreter.clear_caches, p) remotecall_wait(configure_test, p) return p end @@ -59,28 +60,15 @@ function spin_up_workers(n) @sync begin @async for p in procs remotecall_wait(include, p, "test/utils.jl") + remotecall_wait(JuliaInterpreter.clear_caches, p) remotecall_wait(configure_test, p) end end return procs end -# Really, we're just going to skip all the tests that run on node1 -const node1_tests = String[] -function move_to_node1(t) - if t in tests - splice!(tests, findfirst(isequal(t), tests)) - push!(node1_tests, t) - end - nothing -end -move_to_node1("precompile") -move_to_node1("SharedArrays") -move_to_node1("stress") -move_to_node1("Distributed") - @testset "Julia tests" begin - nworkers = Threads.nthreads()-1 + nworkers = Threads.nthreads() println("Using $nworkers workers") results = Dict{String,Any}() tests0 = copy(tests) @@ -137,7 +125,7 @@ move_to_node1("Distributed") catch end end - foreach(wait, all_tasks) + foreach(wait, [task for task in all_tasks if isa(task, Task)]) end open(outputfile, "w") do io @@ -163,4 +151,5 @@ move_to_node1("Distributed") end end end + return nothing end diff --git a/test/utils.jl b/test/utils.jl index d9fbc668..56f328eb 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -183,18 +183,26 @@ function run_test_by_eval(test, fullpath, nstmts) current_task().storage[:SOURCE_PATH] = $fullpath modexs = collect(ExprSplitter(JuliaTests, ex)) for (i, modex) in enumerate(modexs) # having the index can be useful for debugging - nstmtsleft = $nstmts + nstmtsframe = $nstmts mod, ex = modex # @show mod ex frame = Frame(mod, ex) - yield() # allow communication between processes - ret, nstmtsleft = evaluate_limited!(frame, nstmtsleft, true) - if isa(ret, Aborted) - push!(aborts, ret) - JuliaInterpreter.finish_stack!(Compiled(), frame, true) + while nstmtsframe > 0 + yield() # allow communication between processes + ret, nstmtsleft = evaluate_limited!(frame, nstmtsframe, true) + @assert !isa(ret, Nothing) || nstmtsleft < nstmtsframe + nstmtsframe = nstmtsleft + if isa(ret, Some) + break + elseif isa(ret, Aborted) + push!(aborts, ret) + JuliaInterpreter.finish_stack!(Compiled(), frame, true) + break + end end end println("Finished ", $test) return ts, aborts end)) end + From 4d5204613e1ae4efdd7cee9036c51a76c6c35b4e Mon Sep 17 00:00:00 2001 From: goerch Date: Sun, 12 Dec 2021 01:42:32 +0100 Subject: [PATCH 4/7] Increasing test isolation --- results.md | 133 +++++++++++++++++++++++---------------------- test/juliatests.jl | 3 +- test/utils.jl | 4 +- 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/results.md b/results.md index f142237b..d83a6176 100644 --- a/results.md +++ b/results.md @@ -9,88 +9,89 @@ Platform Info: Environment: JULIA_EDITOR = code JULIA_NUM_THREADS = 5 -Test run at: 2021-12-12T00:11:31.538 +Test run at: 2021-12-12T01:39:18.353 -Maximum number of statements per lowered expression: 100000 +Maximum number of statements per lowered expression: 10000000 | Test file | Passes | Fails | Errors | Broken | Aborted blocks | | --------- | ------:| -----:| ------:| ------:| --------------:| | ambiguous | 105 | 0 | 1 | 2 | 14 | -| compiler/inference | 997 | 5 | 2 | 2 | 3 | +| compiler/inference | 1003 | 5 | 2 | 2 | 3 | | compiler/validation | 24 | 0 | 1 | 0 | 0 | | compiler/ssair | X | X | X | X | X | -| compiler/irpasses | X | X | X | X | X | +| compiler/irpasses | 25 | 0 | 0 | 0 | 1 | | compiler/codegen | X | X | X | X | X | | compiler/inline | 63 | 2 | 0 | 1 | 0 | -| compiler/contextual | 9 | 2 | 1 | 0 | 1 | -| subarray | 318276 | 0 | 1 | 0 | 5 | -| strings/basic | 87674 | 0 | 0 | 0 | 5 | -| strings/search | 692 | 0 | 0 | 0 | 1 | -| strings/util | 619 | 0 | 0 | 0 | 6 | -| strings/io | 12764 | 0 | 0 | 0 | 2 | -| strings/types | 2302691 | 0 | 0 | 0 | 4 | +| compiler/contextual | 9 | 2 | 1 | 0 | 0 | +| subarray | 326868 | 0 | 0 | 0 | 2 | +| strings/basic | 87674 | 0 | 0 | 0 | 2 | +| strings/search | 692 | 0 | 0 | 0 | 0 | +| strings/util | 619 | 0 | 0 | 0 | 0 | +| strings/io | 12764 | 0 | 0 | 0 | 1 | +| strings/types | 2302691 | 0 | 0 | 0 | 2 | | unicode/utf8 | 19 | 0 | 0 | 0 | 0 | | core | X | X | X | X | X | +| worlds | X | X | X | X | X | | atomics | 3444 | 0 | 0 | 0 | 0 | | keywordargs | 151 | 0 | 0 | 0 | 0 | -| numbers | 1577509 | 0 | 1 | 1 | 16 | +| numbers | 1622626 | 0 | 0 | 1 | 8 | | subtype | X | X | X | X | X | -| char | 1625 | 0 | 0 | 0 | 3 | +| char | 1623 | 0 | 0 | 0 | 0 | | triplequote | 29 | 0 | 0 | 0 | 0 | | intrinsics | 301 | 0 | 0 | 0 | 0 | | dict | X | X | X | X | X | -| hashing | 12521 | 0 | 0 | 0 | 7 | -| iobuffer | 205 | 0 | 0 | 0 | 1 | +| hashing | 12519 | 0 | 0 | 0 | 1 | +| iobuffer | 205 | 0 | 0 | 0 | 0 | | staged | 58 | 6 | 0 | 0 | 0 | -| offsetarray | X | X | X | X | X | -| arrayops | 1996 | 2 | 5 | 2 | 24 | +| offsetarray | 465 | 11 | 6 | 3 | 1 | +| arrayops | 2013 | 0 | 0 | 2 | 3 | | tuple | 606 | 0 | 0 | 0 | 0 | -| reduce | 8578 | 2 | 0 | 0 | 13 | -| reducedim | 865 | 0 | 0 | 0 | 14 | -| abstractarray | 55292 | 0 | 1 | 24795 | 8 | -| intfuncs | 215863 | 0 | 1 | 0 | 6 | -| simdloop | X | X | X | X | X | -| vecelement | 678 | 0 | 0 | 0 | 1 | -| rational | 98633 | 0 | 1 | 1 | 7 | -| bitarray | 381657 | 0 | 3661 | 0 | 23 | -| copy | 331 | 0 | 2 | 0 | 2 | -| math | X | X | X | X | X | -| fastmath | 946 | 0 | 0 | 0 | 1 | +| reduce | X | X | X | X | X | +| reducedim | 865 | 0 | 0 | 0 | 1 | +| abstractarray | 1838 | 0 | 0 | 0 | 3 | +| intfuncs | 215864 | 0 | 0 | 0 | 1 | +| simdloop | 240 | 0 | 0 | 0 | 0 | +| vecelement | 678 | 0 | 0 | 0 | 0 | +| rational | 98634 | 0 | 0 | 1 | 1 | +| bitarray | 652056 | 0 | 3658 | 0 | 14 | +| copy | 533 | 0 | 0 | 0 | 0 | +| math | 148737 | 0 | 1 | 0 | 3 | +| fastmath | 946 | 0 | 0 | 0 | 0 | | functional | 98 | 0 | 0 | 0 | 0 | -| iterators | 10080 | 0 | 0 | 0 | 4 | -| operators | 13039 | 0 | 0 | 0 | 2 | +| iterators | 10080 | 0 | 0 | 0 | 2 | +| operators | 13039 | 0 | 0 | 0 | 0 | | ordering | 35 | 0 | 0 | 0 | 0 | -| path | 1051 | 0 | 0 | 12 | 2 | +| path | 1295 | 0 | 0 | 12 | 1 | | ccall | X | X | X | X | X | -| parse | 16098 | 0 | 0 | 0 | 4 | -| loading | 153389 | 2 | 1 | 0 | 15 | -| gmp | 2323 | 1 | 0 | 0 | 6 | -| sorting | 12649 | 0 | 8 | 10 | 8 | -| spawn | X | X | X | X | X | -| backtrace | 8 | 11 | 17 | 1 | 2 | -| exceptions | 38 | 24 | 7 | 0 | 3 | +| parse | 16098 | 0 | 0 | 0 | 1 | +| loading | X | X | X | X | X | +| gmp | 2323 | 1 | 0 | 0 | 1 | +| sorting | 12705 | 0 | 3 | 10 | 4 | +| spawn | 209 | 1 | 1 | 4 | 0 | +| backtrace | 8 | 11 | 17 | 1 | 0 | +| exceptions | 38 | 25 | 7 | 0 | 0 | | file | X | X | X | X | X | -| read | 3733 | 0 | 0 | 0 | 3 | -| version | 2452 | 0 | 0 | 0 | 1 | -| namedtuple | 214 | 0 | 0 | 0 | 1 | -| mpfr | 1099 | 13 | 15 | 1 | 6 | -| broadcast | 507 | 1 | 0 | 0 | 6 | -| complex | 8432 | 0 | 0 | 5 | 7 | +| read | X | X | X | X | X | +| version | 2452 | 0 | 0 | 0 | 0 | +| namedtuple | 214 | 0 | 0 | 0 | 0 | +| mpfr | 1127 | 0 | 0 | 1 | 0 | +| broadcast | 507 | 1 | 0 | 0 | 2 | +| complex | 8432 | 0 | 0 | 5 | 1 | | floatapprox | 49 | 0 | 0 | 0 | 0 | | reflection | X | X | X | X | X | -| regex | 128 | 0 | 2 | 0 | 1 | +| regex | 130 | 0 | 0 | 0 | 0 | | float16 | 762091 | 0 | 0 | 0 | 1 | -| combinatorics | 170 | 0 | 0 | 0 | 2 | +| combinatorics | 170 | 0 | 0 | 0 | 0 | | sysinfo | 4 | 0 | 0 | 0 | 0 | -| env | 96 | 0 | 1 | 0 | 4 | -| rounding | 112720 | 0 | 0 | 0 | 3 | -| ranges | 12110676 | 5 | 26 | 327699 | 19 | +| env | 105 | 0 | 0 | 0 | 0 | +| rounding | 112720 | 0 | 0 | 0 | 1 | +| ranges | 12123407 | 254 | 0 | 329460 | 4 | | mod2pi | 80 | 0 | 0 | 0 | 0 | -| euler | 12 | 0 | 0 | 0 | 6 | +| euler | 12 | 0 | 0 | 0 | 2 | | show | X | X | X | X | X | -| client | 0 | 4 | 0 | 0 | 1 | +| client | 0 | 4 | 0 | 0 | 0 | | errorshow | X | X | X | X | X | -| sets | X | X | X | X | X | +| sets | 3529 | 0 | 0 | 1 | 1 | | goto | 19 | 0 | 0 | 0 | 0 | | llvmcall | X | X | X | X | X | | llvmcall2 | 7 | 0 | 0 | 0 | 0 | @@ -98,33 +99,33 @@ Maximum number of statements per lowered expression: 100000 | some | 71 | 0 | 0 | 0 | 0 | | meta | 66 | 2 | 1 | 0 | 0 | | stacktraces | X | X | X | X | X | -| docs | X | X | X | X | X | +| docs | 235 | 1 | 0 | 0 | 2 | | misc | X | X | X | X | X | | threads | X | X | X | X | X | | stress | 0 | 0 | 0 | 0 | 0 | -| binaryplatforms | 334 | 0 | 7 | 0 | 7 | -| atexit | 40 | 0 | 0 | 0 | 1 | +| binaryplatforms | 341 | 0 | 0 | 0 | 1 | +| atexit | 40 | 0 | 0 | 0 | 0 | | enums | 99 | 0 | 0 | 0 | 0 | -| cmdlineargs | X | X | X | X | X | -| int | X | X | X | X | X | +| cmdlineargs | 242 | 0 | 0 | 3 | 0 | +| int | 524693 | 0 | 0 | 0 | 1 | | interpreter | 3 | 0 | 0 | 0 | 0 | -| checked | 1239 | 0 | 0 | 0 | 1 | -| bitset | 195 | 0 | 0 | 0 | 3 | +| checked | 1239 | 0 | 0 | 0 | 0 | +| bitset | 195 | 0 | 0 | 0 | 0 | | floatfuncs | 208 | 0 | 13 | 0 | 2 | | precompile | X | X | X | X | X | | boundscheck | X | X | X | X | X | | error | 31 | 0 | 0 | 0 | 0 | -| cartesian | 236 | 0 | 1 | 3 | 3 | +| cartesian | 238 | 0 | 0 | 3 | 0 | | osutils | 57 | 0 | 0 | 0 | 0 | | channels | X | X | X | X | X | -| iostream | 50 | 0 | 0 | 0 | 1 | -| secretbuffer | X | X | X | X | X | +| iostream | 50 | 0 | 0 | 0 | 0 | +| secretbuffer | 27 | 0 | 0 | 0 | 0 | | specificity | X | X | X | X | X | -| reinterpretarray | 232 | 0 | 0 | 0 | 3 | +| reinterpretarray | 232 | 0 | 0 | 0 | 0 | | syntax | X | X | X | X | X | -| corelogging | 55 | 2 | 21 | 0 | 1 | -| missing | 563 | 0 | 2 | 1 | 3 | -| asyncmap | X | X | X | X | X | +| corelogging | 230 | 1 | 0 | 0 | 0 | +| missing | 563 | 0 | 2 | 1 | 1 | +| asyncmap | 304 | 0 | 0 | 0 | 0 | | smallarrayshrink | 36 | 0 | 0 | 0 | 0 | | opaque_closure | X | X | X | X | X | | filesystem | 4 | 0 | 0 | 0 | 0 | diff --git a/test/juliatests.jl b/test/juliatests.jl index 54a89891..0b946598 100644 --- a/test/juliatests.jl +++ b/test/juliatests.jl @@ -28,7 +28,7 @@ function test_path(test) end end -nstmts = 10^5 # very quick, aborts a lot +nstmts = 10^7 # very quick, aborts a lot outputfile = "results.md" i = 1 while i <= length(ARGS) @@ -151,5 +151,4 @@ end end end end - return nothing end diff --git a/test/utils.jl b/test/utils.jl index 56f328eb..cec496fb 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -149,6 +149,8 @@ end function configure_test() # To run tests efficiently, certain methods must be run in Compiled mode, # in particular those that are used by the Test infrastructure + cc = JuliaInterpreter.compiled_calls + empty!(cc) cm = JuliaInterpreter.compiled_methods empty!(cm) JuliaInterpreter.set_compiled_methods() @@ -184,7 +186,7 @@ function run_test_by_eval(test, fullpath, nstmts) modexs = collect(ExprSplitter(JuliaTests, ex)) for (i, modex) in enumerate(modexs) # having the index can be useful for debugging nstmtsframe = $nstmts - mod, ex = modex + local mod, ex = modex # @show mod ex frame = Frame(mod, ex) while nstmtsframe > 0 From 9bc95a4e99611f5890beb7ad3829caf6175779e9 Mon Sep 17 00:00:00 2001 From: goerch Date: Mon, 13 Dec 2021 13:21:54 +0100 Subject: [PATCH 5/7] Assertion hit by subtype tests commented --- src/interpret.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/interpret.jl b/src/interpret.jl index c366f6a2..3a86a545 100644 --- a/src/interpret.jl +++ b/src/interpret.jl @@ -454,7 +454,8 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev # show_stackloc(frame) # @show node # end - @assert is_leaf(frame) + # The following assertion is hit by Julia subtype tests + # @assert is_leaf(frame) @static VERSION >= v"1.8.0-DEV.370" && coverage_visit_line!(frame) local rhs # For debugging: From f1835774f07bc9f3dbd002bf8502c66986192c3c Mon Sep 17 00:00:00 2001 From: goerch Date: Fri, 17 Dec 2021 18:30:32 +0100 Subject: [PATCH 6/7] Optional unlimited evaluation of Julia tests --- test/juliatests.jl | 3 +- test/utils.jl | 115 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/test/juliatests.jl b/test/juliatests.jl index 0b946598..786a1a55 100644 --- a/test/juliatests.jl +++ b/test/juliatests.jl @@ -87,7 +87,8 @@ end try resp = disable_sigint() do p = spin_up_worker() - result = remotecall_fetch(run_test_by_eval, p, test, fullpath, nstmts) + # result = remotecall_fetch(run_test_by_limited_eval, p, test, fullpath, nstmts) + result = remotecall_fetch(run_test_by_eval, p, test, fullpath) rmprocs(p; waitfor=5) result end diff --git a/test/utils.jl b/test/utils.jl index cec496fb..782b2887 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -99,7 +99,7 @@ function evaluate_limited!(@nospecialize(recurse), frame::Frame, nstmts::Int, is ret = limited_exec!(recurse, newframe, refnstmts, istoplevel) isa(ret, Aborted) && return ret, refnstmts[] frame.callee = nothing - end + end JuliaInterpreter.recycle(newframe) # Because thunks may define new methods, return to toplevel frame.pc = pc + 1 @@ -144,6 +144,90 @@ function limited_exec!(@nospecialize(recurse), newframe, refnstmts, istoplevel) return isa(ret, Aborted) ? ret : something(ret) end +""" + ret = evaluate!(recurse, frame, istoplevel::Bool=true) + +Run `frame` until one of: +- execution terminates normally (`ret = Some{Any}(val)`, where `val` is the returned value of `frame`) +- if `istoplevel` and a `thunk` or `method` expression is encountered (`ret = nothing`) +""" +function evaluate!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) + # The following is like finish!, except we intercept :call expressions so that we can run them + # with limexec! rather than the default finish_and_return! + pc = frame.pc + while true + shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc) + stmt = pc_expr(frame, pc) + if isa(stmt, Expr) + if stmt.head == :call && !isa(recurse, Compiled) + try + rhs = evaluate_call!(exec!, frame, stmt) + lhs = SSAValue(pc) + do_assignment!(frame, lhs, rhs) + new_pc = pc + 1 + catch err + new_pc = handle_err(recurse, frame, err) + end + elseif stmt.head == :(=) && isexpr(stmt.args[2], :call) && !isa(recurse, Compiled) + try + rhs = evaluate_call!(exec!, frame, stmt.args[2]) + do_assignment!(frame, stmt.args[1], rhs) + new_pc = pc + 1 + catch err + new_pc = handle_err(recurse, frame, err) + end + elseif istoplevel && stmt.head == :thunk + code = stmt.args[1] + if length(code.code) == 1 && JuliaInterpreter.is_return(code.code[end]) && isexpr(code.code[end].val, :method) + # Julia 1.2+ puts a :thunk before the start of each method + new_pc = pc + 1 + else + newframe = Frame(moduleof(frame), code) + if isa(recurse, Compiled) + JuliaInterpreter.finish!(recurse, newframe, true) + else + newframe.caller = frame + frame.callee = newframe + exec!(recurse, newframe, istoplevel) + frame.callee = nothing + end + JuliaInterpreter.recycle(newframe) + # Because thunks may define new methods, return to toplevel + frame.pc = pc + 1 + return nothing + end + elseif istoplevel && stmt.head == :method && length(stmt.args) == 3 + step_expr!(recurse, frame, stmt, istoplevel) + frame.pc = pc + 1 + return nothing + else + new_pc = step_expr!(recurse, frame, stmt, istoplevel) + end + else + new_pc = step_expr!(recurse, frame, stmt, istoplevel) + end + (new_pc === nothing || isa(new_pc, BreakpointRef)) && break + pc = frame.pc = new_pc + end + # Handle the return + stmt = pc_expr(frame, pc) + ret = get_return(frame) + return Some{Any}(ret) +end + +evaluate!(@nospecialize(recurse), modex::Tuple{Module,Expr,Frame}, istoplevel::Bool=true) = + evaluate!(recurse, modex[end], istoplevel) +evaluate!(@nospecialize(recurse), modex::Tuple{Module,Expr,Expr}, istoplevel::Bool=true) = + Some{Any}(Core.eval(modex[1], modex[3])) + +evaluate!(frame::Union{Frame, Tuple}, istoplevel::Bool=false) = + evaluate!(finish_and_return!, frame, istoplevel) + +function exec!(@nospecialize(recurse), newframe, istoplevel) + ret = evaluate!(recurse, newframe, istoplevel) + return something(ret) +end + ### Functions needed on workers for running tests function configure_test() @@ -173,7 +257,7 @@ function configure_test() push!(cm, which(SHA.update!, Tuple{SHA.SHA1_CTX,Vector{UInt8}})) end -function run_test_by_eval(test, fullpath, nstmts) +function run_test_by_limited_eval(test, fullpath, nstmts) Core.eval(Main, Expr(:toplevel, :(module JuliaTests using Test, Random end), quote # These must be run at top level, so we can't put this in a function println("Working on ", $test, "...") @@ -208,3 +292,30 @@ function run_test_by_eval(test, fullpath, nstmts) end)) end +function run_test_by_eval(test, fullpath) + Core.eval(Main, Expr(:toplevel, :(module JuliaTests using Test, Random end), quote + # These must be run at top level, so we can't put this in a function + println("Working on ", $test, "...") + ex = read_and_parse($fullpath) + isexpr(ex, :error) && @error "error parsing $($test): $ex" + aborts = Aborted[] + ts = Test.DefaultTestSet($test) + Test.push_testset(ts) + current_task().storage[:SOURCE_PATH] = $fullpath + modexs = collect(ExprSplitter(JuliaTests, ex)) + for (i, modex) in enumerate(modexs) # having the index can be useful for debugging + local mod, ex = modex + @show mod ex + frame = Frame(mod, ex) + while true + yield() # allow communication between processes + ret = evaluate!(frame, true) + if isa(ret, Some) + break + end + end + end + println("Finished ", $test) + return ts, aborts + end)) +end From 50f35b64cceb6837343eac696e4d862b8c0ea9e5 Mon Sep 17 00:00:00 2001 From: goerch Date: Thu, 23 Dec 2021 13:33:25 +0100 Subject: [PATCH 7/7] Reasonable performance with time budgeting call sites... ... and reasonable results without aborting --- results.md | 137 +++++++++++++++++++++------------------------ src/construct.jl | 12 ++-- src/types.jl | 1 + test/juliatests.jl | 2 +- test/utils.jl | 35 ++++++++---- 5 files changed, 96 insertions(+), 91 deletions(-) diff --git a/results.md b/results.md index d83a6176..a809c47f 100644 --- a/results.md +++ b/results.md @@ -1,130 +1,119 @@ -Julia Version 1.7.0 -Commit 3bf9d17731 (2021-11-30 12:12 UTC) +Julia Version 1.8.0-DEV.1169 +Commit f2f31336a1 (2021-12-22 11:27 UTC) Platform Info: OS: Windows (x86_64-w64-mingw32) CPU: Intel(R) Core(TM) i7-10710U CPU @ 1.10GHz WORD_SIZE: 64 LIBM: libopenlibm LLVM: libLLVM-12.0.1 (ORCJIT, skylake) -Environment: - JULIA_EDITOR = code - JULIA_NUM_THREADS = 5 -Test run at: 2021-12-12T01:39:18.353 +Test run at: 2021-12-23T13:30:26.035 Maximum number of statements per lowered expression: 10000000 | Test file | Passes | Fails | Errors | Broken | Aborted blocks | | --------- | ------:| -----:| ------:| ------:| --------------:| -| ambiguous | 105 | 0 | 1 | 2 | 14 | -| compiler/inference | 1003 | 5 | 2 | 2 | 3 | -| compiler/validation | 24 | 0 | 1 | 0 | 0 | -| compiler/ssair | X | X | X | X | X | -| compiler/irpasses | 25 | 0 | 0 | 0 | 1 | -| compiler/codegen | X | X | X | X | X | -| compiler/inline | 63 | 2 | 0 | 1 | 0 | -| compiler/contextual | 9 | 2 | 1 | 0 | 0 | -| subarray | 326868 | 0 | 0 | 0 | 2 | -| strings/basic | 87674 | 0 | 0 | 0 | 2 | -| strings/search | 692 | 0 | 0 | 0 | 0 | -| strings/util | 619 | 0 | 0 | 0 | 0 | -| strings/io | 12764 | 0 | 0 | 0 | 1 | -| strings/types | 2302691 | 0 | 0 | 0 | 2 | +| ambiguous | 107 | 0 | 0 | 2 | 0 | +| subarray | 318316 | 0 | 0 | 0 | 0 | +| strings/basic | 87674 | 0 | 0 | 0 | 0 | +| strings/search | 876 | 0 | 0 | 0 | 0 | +| strings/util | 1147 | 0 | 0 | 0 | 0 | +| strings/io | 12764 | 0 | 0 | 0 | 0 | +| strings/types | 2302691 | 0 | 0 | 0 | 0 | | unicode/utf8 | 19 | 0 | 0 | 0 | 0 | | core | X | X | X | X | X | -| worlds | X | X | X | X | X | +| worlds | ☠️ | ☠️ | ☠️ | ☠️ | ☠️ | | atomics | 3444 | 0 | 0 | 0 | 0 | | keywordargs | 151 | 0 | 0 | 0 | 0 | -| numbers | 1622626 | 0 | 0 | 1 | 8 | -| subtype | X | X | X | X | X | -| char | 1623 | 0 | 0 | 0 | 0 | +| numbers | 1578757 | 1 | 0 | 2 | 0 | +| subtype | 337674 | 0 | 0 | 19 | 0 | +| char | 1628 | 0 | 0 | 0 | 0 | | triplequote | 29 | 0 | 0 | 0 | 0 | | intrinsics | 301 | 0 | 0 | 0 | 0 | -| dict | X | X | X | X | X | -| hashing | 12519 | 0 | 0 | 0 | 1 | -| iobuffer | 205 | 0 | 0 | 0 | 0 | -| staged | 58 | 6 | 0 | 0 | 0 | -| offsetarray | 465 | 11 | 6 | 3 | 1 | -| arrayops | 2013 | 0 | 0 | 2 | 3 | -| tuple | 606 | 0 | 0 | 0 | 0 | -| reduce | X | X | X | X | X | -| reducedim | 865 | 0 | 0 | 0 | 1 | -| abstractarray | 1838 | 0 | 0 | 0 | 3 | -| intfuncs | 215864 | 0 | 0 | 0 | 1 | +| dict | 144420 | 0 | 0 | 0 | 0 | +| hashing | 12519 | 0 | 0 | 0 | 0 | +| iobuffer | 209 | 0 | 0 | 0 | 0 | +| staged | 59 | 5 | 0 | 0 | 0 | +| offsetarray | 484 | 3 | 0 | 3 | 0 | +| arrayops | 2031 | 0 | 0 | 2 | 0 | +| tuple | 625 | 0 | 0 | 0 | 0 | +| reduce | 8580 | 2 | 0 | 0 | 0 | +| reducedim | 865 | 0 | 0 | 0 | 0 | +| abstractarray | 55892 | 0 | 0 | 24795 | 0 | +| intfuncs | 227902 | 0 | 0 | 0 | 0 | | simdloop | 240 | 0 | 0 | 0 | 0 | | vecelement | 678 | 0 | 0 | 0 | 0 | -| rational | 98634 | 0 | 0 | 1 | 1 | -| bitarray | 652056 | 0 | 3658 | 0 | 14 | +| rational | 98639 | 0 | 0 | 1 | 0 | +| bitarray | 914001 | 1 | 0 | 0 | 0 | | copy | 533 | 0 | 0 | 0 | 0 | -| math | 148737 | 0 | 1 | 0 | 3 | +| math | 148967 | 1 | 11 | 0 | 0 | | fastmath | 946 | 0 | 0 | 0 | 0 | | functional | 98 | 0 | 0 | 0 | 0 | -| iterators | 10080 | 0 | 0 | 0 | 2 | -| operators | 13039 | 0 | 0 | 0 | 0 | -| ordering | 35 | 0 | 0 | 0 | 0 | -| path | 1295 | 0 | 0 | 12 | 1 | +| iterators | 10164 | 0 | 0 | 0 | 0 | +| operators | 13039 | 0 | 1 | 0 | 0 | +| ordering | 37 | 0 | 0 | 0 | 0 | +| path | 1051 | 0 | 0 | 12 | 0 | | ccall | X | X | X | X | X | -| parse | 16098 | 0 | 0 | 0 | 1 | -| loading | X | X | X | X | X | -| gmp | 2323 | 1 | 0 | 0 | 1 | -| sorting | 12705 | 0 | 3 | 10 | 4 | -| spawn | 209 | 1 | 1 | 4 | 0 | -| backtrace | 8 | 11 | 17 | 1 | 0 | -| exceptions | 38 | 25 | 7 | 0 | 0 | +| parse | 16098 | 0 | 0 | 0 | 0 | +| loading | 168472 | 2 | 1 | 0 | 0 | +| gmp | 2357 | 0 | 0 | 0 | 0 | +| sorting | 16096 | 0 | 0 | 10 | 0 | +| spawn | 230 | 1 | 1 | 4 | 0 | +| backtrace | 13 | 9 | 16 | 1 | 0 | +| exceptions | 43 | 21 | 6 | 0 | 0 | | file | X | X | X | X | X | -| read | X | X | X | X | X | +| read | 3870 | 0 | 0 | 0 | 0 | | version | 2452 | 0 | 0 | 0 | 0 | -| namedtuple | 214 | 0 | 0 | 0 | 0 | -| mpfr | 1127 | 0 | 0 | 1 | 0 | -| broadcast | 507 | 1 | 0 | 0 | 2 | -| complex | 8432 | 0 | 0 | 5 | 1 | +| namedtuple | 215 | 0 | 0 | 0 | 0 | +| mpfr | 1135 | 0 | 0 | 1 | 0 | +| broadcast | 509 | 2 | 0 | 0 | 0 | +| complex | 8432 | 0 | 0 | 5 | 0 | | floatapprox | 49 | 0 | 0 | 0 | 0 | | reflection | X | X | X | X | X | | regex | 130 | 0 | 0 | 0 | 0 | -| float16 | 762091 | 0 | 0 | 0 | 1 | +| float16 | 237 | 0 | 0 | 0 | 0 | | combinatorics | 170 | 0 | 0 | 0 | 0 | | sysinfo | 4 | 0 | 0 | 0 | 0 | -| env | 105 | 0 | 0 | 0 | 0 | -| rounding | 112720 | 0 | 0 | 0 | 1 | -| ranges | 12123407 | 254 | 0 | 329460 | 4 | +| env | 94 | 0 | 0 | 0 | 0 | +| rounding | 112720 | 0 | 0 | 0 | 0 | +| ranges | 12110652 | 2 | 0 | 327682 | 0 | | mod2pi | 80 | 0 | 0 | 0 | 0 | -| euler | 12 | 0 | 0 | 0 | 2 | -| show | X | X | X | X | X | -| client | 0 | 4 | 0 | 0 | 0 | +| euler | 12 | 0 | 0 | 0 | 0 | +| show | 128879 | 0 | 0 | 8 | 0 | +| client | 2 | 3 | 0 | 0 | 0 | | errorshow | X | X | X | X | X | -| sets | 3529 | 0 | 0 | 1 | 1 | +| sets | 3594 | 0 | 0 | 1 | 0 | | goto | 19 | 0 | 0 | 0 | 0 | -| llvmcall | X | X | X | X | X | | llvmcall2 | 7 | 0 | 0 | 0 | 0 | -| ryu | 31194 | 0 | 21 | 0 | 1 | -| some | 71 | 0 | 0 | 0 | 0 | -| meta | 66 | 2 | 1 | 0 | 0 | +| ryu | 31215 | 0 | 0 | 0 | 0 | +| some | 72 | 0 | 0 | 0 | 0 | +| meta | 69 | 0 | 0 | 0 | 0 | | stacktraces | X | X | X | X | X | -| docs | 235 | 1 | 0 | 0 | 2 | +| docs | 237 | 1 | 0 | 0 | 0 | | misc | X | X | X | X | X | | threads | X | X | X | X | X | | stress | 0 | 0 | 0 | 0 | 0 | -| binaryplatforms | 341 | 0 | 0 | 0 | 1 | +| binaryplatforms | 341 | 0 | 0 | 0 | 0 | | atexit | 40 | 0 | 0 | 0 | 0 | | enums | 99 | 0 | 0 | 0 | 0 | -| cmdlineargs | 242 | 0 | 0 | 3 | 0 | -| int | 524693 | 0 | 0 | 0 | 1 | +| cmdlineargs | 255 | 0 | 0 | 3 | 0 | +| int | 524698 | 0 | 0 | 0 | 0 | | interpreter | 3 | 0 | 0 | 0 | 0 | | checked | 1239 | 0 | 0 | 0 | 0 | | bitset | 195 | 0 | 0 | 0 | 0 | -| floatfuncs | 208 | 0 | 13 | 0 | 2 | +| floatfuncs | 215 | 0 | 0 | 0 | 0 | | precompile | X | X | X | X | X | | boundscheck | X | X | X | X | X | | error | 31 | 0 | 0 | 0 | 0 | -| cartesian | 238 | 0 | 0 | 3 | 0 | +| cartesian | 343 | 0 | 0 | 3 | 0 | | osutils | 57 | 0 | 0 | 0 | 0 | -| channels | X | X | X | X | X | +| channels | 252 | 6 | 0 | 0 | 0 | | iostream | 50 | 0 | 0 | 0 | 0 | | secretbuffer | 27 | 0 | 0 | 0 | 0 | | specificity | X | X | X | X | X | | reinterpretarray | 232 | 0 | 0 | 0 | 0 | | syntax | X | X | X | X | X | | corelogging | 230 | 1 | 0 | 0 | 0 | -| missing | 563 | 0 | 2 | 1 | 1 | +| missing | 565 | 0 | 0 | 1 | 0 | | asyncmap | 304 | 0 | 0 | 0 | 0 | | smallarrayshrink | 36 | 0 | 0 | 0 | 0 | | opaque_closure | X | X | X | X | X | diff --git a/src/construct.jl b/src/construct.jl index 2b2c3d9b..7ca96bfc 100644 --- a/src/construct.jl +++ b/src/construct.jl @@ -35,6 +35,7 @@ const compiled_modules = Set{Module}() const junk_framedata = FrameData[] # to allow re-use of allocated memory (this is otherwise a bottleneck) const junk_frames = Frame[] +disable_recycle() = true debug_recycle() = false @noinline function _check_frame_not_in_junk(frame) @assert frame.framedata ∉ junk_framedata @@ -43,8 +44,8 @@ end @inline function recycle(frame) debug_recycle() && _check_frame_not_in_junk(frame) - push!(junk_framedata, frame.framedata) - push!(junk_frames, frame) + disable_recycle() || push!(junk_framedata, frame.framedata) + disable_recycle() || push!(junk_frames, frame) end function return_from(frame::Frame) @@ -264,7 +265,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e ng, ns = isa(ssavt, Int) ? ssavt : length(ssavt::Vector{Any}), length(src.slotflags) if length(junk_framedata) > 0 olddata = pop!(junk_framedata) - locals, ssavalues, sparams = olddata.locals, olddata.ssavalues, olddata.sparams + locals, ssavalues, times, sparams = olddata.locals, olddata.ssavalues, olddata.times, olddata.sparams exception_frames, last_reference = olddata.exception_frames, olddata.last_reference last_exception = olddata.last_exception callargs = olddata.callargs @@ -272,6 +273,8 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e fill!(locals, nothing) resize!(ssavalues, 0) resize!(ssavalues, ng) + resize!(times, 0) + resize!(times, ng) # for check_isdefined to work properly, we need sparams to start out unassigned resize!(sparams, 0) empty!(exception_frames) @@ -280,6 +283,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e else locals = Vector{Union{Nothing,Some{Any}}}(nothing, ns) ssavalues = Vector{Any}(undef, ng) + times = Vector{UInt64}(undef, ng) sparams = Vector{Any}(undef, 0) exception_frames = Int[] last_reference = Vector{Int}(undef, ns) @@ -310,7 +314,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e isa(T, TypeVar) && continue # only fill concrete types sparams[i] = T end - FrameData(locals, ssavalues, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs) + FrameData(locals, ssavalues, times, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs) end """ diff --git a/src/types.jl b/src/types.jl index 510e2112..a05cdb04 100644 --- a/src/types.jl +++ b/src/types.jl @@ -194,6 +194,7 @@ Important fields: struct FrameData locals::Vector{Union{Nothing,Some{Any}}} ssavalues::Vector{Any} + times::Vector{UInt64} sparams::Vector{Any} exception_frames::Vector{Int} last_exception::Base.RefValue{Any} diff --git a/test/juliatests.jl b/test/juliatests.jl index 786a1a55..b35af103 100644 --- a/test/juliatests.jl +++ b/test/juliatests.jl @@ -45,7 +45,7 @@ while i <= length(ARGS) end end -tests, _, exit_on_error, seed = choosetests(ARGS) +tests, _, exit_on_error, seed = choosetests(["--skip", "llvmcall", "compiler", "stdlib", ARGS...]) function spin_up_worker() p = addprocs(1)[1] diff --git a/test/utils.jl b/test/utils.jl index 782b2887..c4b5abe5 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -157,24 +157,30 @@ function evaluate!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) pc = frame.pc while true shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc) - stmt = pc_expr(frame, pc) + stmt = pc_expr(frame, pc) + if isassigned(frame.framedata.times, pc) && frame.framedata.times[pc] > 0 + recurse_pc = Compiled() + else + recurse_pc = recurse + end + time = time_ns() if isa(stmt, Expr) - if stmt.head == :call && !isa(recurse, Compiled) + if stmt.head == :call && !isa(recurse_pc, Compiled) try rhs = evaluate_call!(exec!, frame, stmt) lhs = SSAValue(pc) do_assignment!(frame, lhs, rhs) new_pc = pc + 1 catch err - new_pc = handle_err(recurse, frame, err) + new_pc = handle_err(recurse_pc, frame, err) end - elseif stmt.head == :(=) && isexpr(stmt.args[2], :call) && !isa(recurse, Compiled) + elseif stmt.head == :(=) && isexpr(stmt.args[2], :call) && !isa(recurse_pc, Compiled) try rhs = evaluate_call!(exec!, frame, stmt.args[2]) do_assignment!(frame, stmt.args[1], rhs) new_pc = pc + 1 catch err - new_pc = handle_err(recurse, frame, err) + new_pc = handle_err(recurse_pc, frame, err) end elseif istoplevel && stmt.head == :thunk code = stmt.args[1] @@ -183,12 +189,12 @@ function evaluate!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) new_pc = pc + 1 else newframe = Frame(moduleof(frame), code) - if isa(recurse, Compiled) - JuliaInterpreter.finish!(recurse, newframe, true) + if isa(recurse_pc, Compiled) + JuliaInterpreter.finish!(recurse_pc, newframe, true) else newframe.caller = frame frame.callee = newframe - exec!(recurse, newframe, istoplevel) + exec!(recurse_pc, newframe, istoplevel) frame.callee = nothing end JuliaInterpreter.recycle(newframe) @@ -197,16 +203,21 @@ function evaluate!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) return nothing end elseif istoplevel && stmt.head == :method && length(stmt.args) == 3 - step_expr!(recurse, frame, stmt, istoplevel) + step_expr!(recurse_pc, frame, stmt, istoplevel) frame.pc = pc + 1 return nothing else - new_pc = step_expr!(recurse, frame, stmt, istoplevel) + new_pc = step_expr!(recurse_pc, frame, stmt, istoplevel) end else - new_pc = step_expr!(recurse, frame, stmt, istoplevel) + new_pc = step_expr!(recurse_pc, frame, stmt, istoplevel) end (new_pc === nothing || isa(new_pc, BreakpointRef)) && break + if !isassigned(frame.framedata.times, pc) + frame.framedata.times[pc] = time_ns() - time + else + frame.framedata.times[pc] += time_ns() - time + end pc = frame.pc = new_pc end # Handle the return @@ -305,7 +316,7 @@ function run_test_by_eval(test, fullpath) modexs = collect(ExprSplitter(JuliaTests, ex)) for (i, modex) in enumerate(modexs) # having the index can be useful for debugging local mod, ex = modex - @show mod ex + # @show mod ex frame = Frame(mod, ex) while true yield() # allow communication between processes