diff --git a/CondaPkg.toml b/CondaPkg.toml new file mode 100644 index 00000000..9108042d --- /dev/null +++ b/CondaPkg.toml @@ -0,0 +1,3 @@ +[deps] +jupyter_client = "" +jupyter_kernel_test = "" diff --git a/Project.toml b/Project.toml index bc085fa3..87ba2c4f 100644 --- a/Project.toml +++ b/Project.toml @@ -11,10 +11,12 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" SoftGlobalScope = "b85f4697-e234-5449-a836-ec8e2f98b302" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" @@ -28,10 +30,12 @@ JSON = "0.18,0.19,0.20,0.21,1" Logging = "1" Markdown = "1" Pkg = "1" +PrecompileTools = "1.2.1" Printf = "1" REPL = "1" Random = "1" SHA = "0.7, 1" +Sockets = "1" SoftGlobalScope = "1" UUIDs = "1" ZMQ = "1.3" diff --git a/deps/kspec.jl b/deps/kspec.jl index 0ad44117..cb5f7e69 100644 --- a/deps/kspec.jl +++ b/deps/kspec.jl @@ -3,7 +3,7 @@ import JSON ####################################################################### # Install Jupyter kernel-spec files. -copy_config(src, dest) = cp(src, joinpath(dest, basename(src)), force=true) +copy_config(src::AbstractString, dest::AbstractString) = cp(src, joinpath(dest, basename(src)), force=true) # return the user kernelspec directory, according to # https://jupyter-client.readthedocs.io/en/latest/kernels.html#kernelspecs diff --git a/docs/src/_changelog.md b/docs/src/_changelog.md index 3111d29a..2ce174d8 100644 --- a/docs/src/_changelog.md +++ b/docs/src/_changelog.md @@ -7,6 +7,20 @@ CurrentModule = IJulia This documents notable changes in IJulia.jl. The format is based on [Keep a Changelog](https://keepachangelog.com). +## Unreleased + +### Added +- Implemented [`reset_stdio_count()`](@ref) to provide a public API for + resetting the stdio count ([#1145]). + +### Changed +- IJulia was completely refactored to minimize global state ([#1145]). This + allows for better testing (for the first time we can test kernel execution) + and for executing precompilation workloads. We've tried to avoid any breaking + changes but it's possible that some packages may be relying on internals that + have changed. If you have any problems with this release please open an issue + on Github so we can help. + ## [v1.29.2] - 2025-07-29 ### Fixed diff --git a/docs/src/library/internals.md b/docs/src/library/internals.md index f6f5bad5..206171c7 100644 --- a/docs/src/library/internals.md +++ b/docs/src/library/internals.md @@ -59,8 +59,6 @@ IJulia.waitloop ```@docs IJulia.IJuliaStdio -IJulia.capture_stdout -IJulia.capture_stderr IJulia.watch_stream ``` diff --git a/docs/src/library/public.md b/docs/src/library/public.md index 5f5290d6..f05ab8f1 100644 --- a/docs/src/library/public.md +++ b/docs/src/library/public.md @@ -45,4 +45,5 @@ IJulia.load_string ```@docs IJulia.readprompt IJulia.set_max_stdio +IJulia.reset_stdio_count ``` diff --git a/src/IJulia.jl b/src/IJulia.jl index 76de0559..23d932e6 100644 --- a/src/IJulia.jl +++ b/src/IJulia.jl @@ -35,12 +35,14 @@ export notebook, jupyterlab, installkernel import SHA using ZMQ, JSON, SoftGlobalScope -import Base.invokelatest +import Base: invokelatest, RefValue import Dates using Dates: now, format, UTC, ISODateTimeFormat import Random +import Random: seed! using Base64: Base64EncodePipe import REPL +import Logging # InteractiveUtils is not used inside IJulia, but loaded in src/kernel.jl # and this import makes it possible to load InteractiveUtils from the IJulia namespace @@ -50,11 +52,187 @@ const depfile = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl") isfile(depfile) || error("IJulia not properly installed. Please run Pkg.build(\"IJulia\")") include(depfile) # generated by Pkg.build("IJulia") -####################################################################### -# Debugging IJulia +# use our own random seed for msg_id so that we +# don't alter the user-visible random state (issue #336) +const IJulia_RNG = seed!(Random.MersenneTwister(0)) +import UUIDs +uuid4() = string(UUIDs.uuid4(IJulia_RNG)) + +""" +IPython message struct. +""" +mutable struct Msg + idents::Vector{String} + header::Dict + content::Dict + parent_header::Dict + metadata::Dict + function Msg(idents, header::Dict, content::Dict, + parent_header=Dict{String,Any}(), metadata=Dict{String,Any}()) + new(idents,header,content,parent_header,metadata) + end +end + +mutable struct Comm{target} + id::String + primary::Bool + on_msg::Function + on_close::Function + function (::Type{Comm{target}})(id, primary, on_msg, on_close, kernel) where {target} + comm = new{target}(id, primary, on_msg, on_close) + kernel.comms[id] = comm + return comm + end +end + +@kwdef mutable struct Kernel + verbose::Bool = IJULIA_DEBUG + inited::Bool = false + current_module::Module = Main + + # These fields are special and are mirrored to their corresponding global + # variables. + In::Dict{Int, String} = Dict{Int, String}() + Out::Dict{Int, Any} = Dict{Int, Any}() + ans::Any = nothing + n::Int = 0 + + capture_stdout::Bool = true + capture_stderr::Bool = !IJULIA_DEBUG + capture_stdin::Bool = true + + # This dict holds a map from CommID to Comm so that we can + # pick out the right Comm object when messages arrive + # from the front-end. + comms = Dict{String, CommManager.Comm}() + + postexecute_hooks::Vector{Function} = Function[] + preexecute_hooks::Vector{Function} = Function[] + posterror_hooks::Vector{Function} = Function[] + shutdown::Function = exit + + # the following constants need to be initialized in init(). + publish::RefValue{Socket} = Ref{Socket}() + raw_input::RefValue{Socket} = Ref{Socket}() + requests::RefValue{Socket} = Ref{Socket}() + control::RefValue{Socket} = Ref{Socket}() + heartbeat::RefValue{Socket} = Ref{Socket}() + heartbeat_context::RefValue{Context} = Ref{Context}() + profile::Dict{String, Any} = Dict{String, Any}() + connection_file::Union{String, Nothing} = nothing + read_stdout::RefValue{Base.PipeEndpoint} = Ref{Base.PipeEndpoint}() + read_stderr::RefValue{Base.PipeEndpoint} = Ref{Base.PipeEndpoint}() + socket_locks = Dict{Socket, ReentrantLock}() + sha_ctx::RefValue{SHA.SHA_CTX} = Ref{SHA.SHA_CTX}() + hmac_key::Vector{UInt8} = UInt8[] + + stop_event::Base.Event = Base.Event() + waitloop_task::RefValue{Task} = Ref{Task}() + + requests_task::RefValue{Task} = Ref{Task}() + watch_stdout_task::RefValue{Task} = Ref{Task}() + watch_stderr_task::RefValue{Task} = Ref{Task}() + watch_stdout_timer::RefValue{Timer} = Ref{Timer}() + watch_stderr_timer::RefValue{Timer} = Ref{Timer}() + + # name=>iobuffer for each stream ("stdout","stderr") so they can be sent in flush + bufs::Dict{String, IOBuffer} = Dict{String, IOBuffer}() + bufs_locks::Dict{String, ReentrantLock} = Dict{String, ReentrantLock}() + # max output per code cell is 512 kb by default + max_output_per_request::RefValue{Int} = Ref(1 << 19) + + # Variable so that display can be done in the correct Msg context + execute_msg::Msg = Msg(["julia"], Dict("username"=>"jlkernel", "session"=>uuid4()), Dict()) + # Variable tracking the number of bytes written in the current execution request + stdio_bytes::Int = 0 + # Use an array to accumulate "payloads" for the execute_reply message + execute_payloads::Vector{Dict} = Dict[] + + heartbeat_threadid::Vector{Int} = zeros(Int, 128) # sizeof(uv_thread_t) <= 8 on Linux, OSX, Win + + # queue of objects to display at end of cell execution + displayqueue::Vector{Any} = Any[] +end + +function Base.setproperty!(kernel::Kernel, name::Symbol, x) + # These fields need to be assigned explicitly to their global counterparts + if name ∈ (:ans, :n, :In, :Out, :inited) + setproperty!(IJulia, name, x) + + if name ∈ (:ans, :In, :Out) + if hasproperty(kernel.current_module, name) + setproperty!(kernel.current_module, name, x) + end + end + end + + setfield!(kernel, name, x) +end + +function Base.wait(kernel::Kernel) + if isassigned(kernel.waitloop_task) + wait(kernel.waitloop_task[]) + end +end + +function Base.close(kernel::Kernel) + # Reset the IO streams first so that any later errors get printed + if kernel.capture_stdout + redirect_stdout(orig_stdout[]) + close(kernel.watch_stdout_timer[]) + close(kernel.read_stdout[]) + wait(kernel.watch_stdout_task[]) + end + if kernel.capture_stderr + redirect_stderr(orig_stderr[]) + close(kernel.watch_stderr_timer[]) + close(kernel.read_stderr[]) + wait(kernel.watch_stderr_task[]) + end + if kernel.capture_stdin + redirect_stdin(orig_stdin[]) + end + + # Reset the logger so that @log statements work and pop the InlineDisplay + if isassigned(orig_logger) + # orig_logger seems to not be set during precompilation + Logging.global_logger(orig_logger[]) + end + popdisplay() + + # Close all sockets + close(kernel.publish[]) + close(kernel.raw_input[]) + close(kernel.requests[]) + close(kernel.control[]) + stop_heartbeat(kernel) + + # The waitloop should now be ready to exit + kernel.inited = false + notify(kernel.stop_event) + wait(kernel) + + # Reset global variables + IJulia.ans = nothing + IJulia.n = 0 + IJulia._default_kernel = nothing + IJulia.CommManager.comms = Dict{String, CommManager.Comm}() + IJulia.profile = Dict{String, Any}() +end + +function Kernel(f::Function, profile; kwargs...) + kernel = Kernel(; kwargs...) + init([], kernel, profile) + + try + f(kernel) + finally + close(kernel) + end +end + +_default_kernel::Union{Kernel, Nothing} = nothing -# in the Jupyter front-end, enable verbose output via IJulia.set_verbose() -verbose = IJULIA_DEBUG """ set_verbose(v=true) @@ -64,8 +242,12 @@ This consists of log messages printed to the terminal window where `jupyter` was launched, displaying information about every message sent or received by the kernel. Used for debugging IJulia. """ -function set_verbose(v::Bool=true) - global verbose = v +function set_verbose(v::Bool=true, kernel=_default_kernel) + if isnothing(kernel) + error("Kernel has not been initialized, cannot set its verbosity.") + end + + kernel.verbose = v end """ @@ -74,25 +256,15 @@ kernel is running, i.e. in a running IJulia notebook. To test whether you are in an IJulia notebook, therefore, you can check `isdefined(Main, :IJulia) && IJulia.inited`. """ -inited = false - -const _capture_docstring = """ -The IJulia kernel captures all [stdout and stderr](https://en.wikipedia.org/wiki/Standard_streams) -output and redirects it to the notebook. When debugging IJulia problems, -however, it can be more convenient to *not* capture stdout and stderr output -(since the notebook may not be functioning). This can be done by editing -`IJulia.jl` to set `capture_stderr` and/or `capture_stdout` to `false`. -""" +inited::Bool = false -@doc _capture_docstring -const capture_stdout = true - -# set this to false for debugging, to disable stderr redirection -@doc _capture_docstring -const capture_stderr = !IJULIA_DEBUG +function set_current_module(m::Module; kernel=_default_kernel) + if isnothing(kernel) + error("Kernel has not been initialized, cannot set the current module.") + end -set_current_module(m::Module) = current_module[] = m -const current_module = Ref{Module}(Main) + kernel.current_module = m +end _shutting_down::Threads.Atomic{Bool} = Threads.Atomic{Bool}(false) @@ -108,8 +280,8 @@ somewhat analogous to the `%load` magics in IPython. If the optional argument `replace` is `true`, then `s` replaces the *current* cell rather than creating a new cell. """ -function load_string(s::AbstractString, replace::Bool=false) - push!(execute_payloads, Dict( +function load_string(s::AbstractString, replace::Bool=false, kernel=_default_kernel) + push!(kernel.execute_payloads, Dict( "source"=>"set_next_input", "text"=>s, "replace"=>replace @@ -125,8 +297,8 @@ IJulia notebook, analogous to the `%load` magics in IPython. If the optional argument `replace` is `true`, then the file contents replace the *current* cell rather than creating a new cell. """ -load(filename::AbstractString, replace::Bool=false) = - load_string(read(filename, String), replace) +load(filename::AbstractString, replace::Bool=false, kernel=_default_kernel) = + load_string(read(filename, String), replace, kernel) ####################################################################### # History: global In/Out and other exported history variables @@ -135,33 +307,33 @@ load(filename::AbstractString, replace::Bool=false) = returns the string for input cell `n` of the notebook (as it was when it was *last evaluated*). """ -const In = Dict{Int,String}() +In::Dict{String, Any} = Dict{String, Any}() """ `Out` is a global dictionary of output values, where `Out[n]` returns the output from the last evaluation of cell `n` in the notebook. """ -const Out = Dict{Int,Any}() +Out::Dict{String, Any} = Dict{String, Any}() """ `ans` is a global variable giving the value returned by the last notebook cell evaluated. """ -ans = nothing +ans::Any = nothing # execution counter """ `IJulia.n` is the (integer) index of the last-evaluated notebook cell. """ -n = 0 +n::Int = 0 ####################################################################### # methods to clear history or any subset thereof -function clear_history(indices) +function clear_history(indices; kernel=_default_kernel) for n in indices - delete!(In, n) - if haskey(Out, n) - delete!(Out, n) + delete!(kernel.In, n) + if haskey(kernel.Out, n) + delete!(kernel.Out, n) end end end @@ -170,10 +342,10 @@ end clear_history(r::AbstractRange{<:Integer}) = invoke(clear_history, Tuple{Any}, intersect(r, 1:n)) -function clear_history() - empty!(In) - empty!(Out) - global ans = nothing +function clear_history(; kernel=_default_kernel) + empty!(kernel.In) + empty!(kernel.Out) + kernel.ans = nothing end """ @@ -192,18 +364,18 @@ clear_history ####################################################################### # methods to print history or any subset thereof -function history(io::IO, indices::AbstractVector{<:Integer}) - for n in intersect(indices, 1:IJulia.n) - if haskey(In, n) - print(io, In[n]) +function history(io::IO, indices::AbstractVector{<:Integer}; kernel=_default_kernel) + for n in intersect(indices, 1:kernel.n) + if haskey(kernel.In, n) + println(io, kernel.In[n]) end end end -history(io::IO, x::Union{Integer,AbstractVector{<:Integer}}...) = history(io, vcat(x...)) -history(x...) = history(stdout, x...) -history(io::IO, x...) = throw(MethodError(history, (io, x...,))) -history() = history(1:n) +history(io::IO, x::Union{Integer,AbstractVector{<:Integer}}...; kernel=_default_kernel) = history(io, vcat(x...); kernel) +history(x...; kernel=_default_kernel) = history(stdout, x...; kernel) +history(io::IO, x...; kernel=_default_kernel) = throw(MethodError(history, (io, x...,))) +history(; kernel=_default_kernel) = history(1:kernel.n; kernel) """ history([io], [indices...]) @@ -224,57 +396,60 @@ history # executing an input cell, e.g. to "close" the current plot in Pylab. # Modules should only use these if isdefined(Main, IJulia) is true. -const postexecute_hooks = Function[] +function _pop_hook!(f, hooks) + hook_idx = findlast(isequal(f), hooks) + if isnothing(hook_idx) + error("Could not find hook: $(f)") + else + splice!(hooks, hook_idx) + end +end + """ push_postexecute_hook(f::Function) Push a function `f()` onto the end of a list of functions to execute after executing any notebook cell. """ -push_postexecute_hook(f::Function) = push!(postexecute_hooks, f) +push_postexecute_hook(f::Function; kernel=_default_kernel) = push!(kernel.postexecute_hooks, f) """ pop_postexecute_hook(f::Function) Remove a function `f()` from the list of functions to execute after executing any notebook cell. """ -pop_postexecute_hook(f::Function) = - splice!(postexecute_hooks, findlast(isequal(f), postexecute_hooks)) +pop_postexecute_hook(f::Function; kernel=_default_kernel) = _pop_hook!(f, kernel.postexecute_hooks) -const preexecute_hooks = Function[] """ push_preexecute_hook(f::Function) Push a function `f()` onto the end of a list of functions to execute before executing any notebook cell. """ -push_preexecute_hook(f::Function) = push!(preexecute_hooks, f) +push_preexecute_hook(f::Function; kernel=_default_kernel) = push!(kernel.preexecute_hooks, f) """ pop_preexecute_hook(f::Function) Remove a function `f()` from the list of functions to execute before executing any notebook cell. """ -pop_preexecute_hook(f::Function) = - splice!(preexecute_hooks, findlast(isequal(f), preexecute_hooks)) +pop_preexecute_hook(f::Function; kernel=_default_kernel) = _pop_hook!(f, kernel.preexecute_hooks) # similar, but called after an error (e.g. to reset plotting state) -const posterror_hooks = Function[] """ pop_posterror_hook(f::Function) Remove a function `f()` from the list of functions to execute after an error occurs when a notebook cell is evaluated. """ -push_posterror_hook(f::Function) = push!(posterror_hooks, f) +push_posterror_hook(f::Function; kernel=_default_kernel) = push!(kernel.posterror_hooks, f) """ pop_posterror_hook(f::Function) Remove a function `f()` from the list of functions to execute after an error occurs when a notebook cell is evaluated. """ -pop_posterror_hook(f::Function) = - splice!(posterror_hooks, findlast(isequal(f), posterror_hooks)) +pop_posterror_hook(f::Function; kernel=_default_kernel) = _pop_hook!(f, kernel.posterror_hooks) ####################################################################### @@ -288,13 +463,14 @@ Call `clear_output()` to clear visible output from the current notebook cell. Using `wait=true` clears the output only when new output is available, which reduces flickering and is useful for simple animations. """ -function clear_output(wait=false) +function clear_output(wait=false, kernel=_default_kernel) # flush pending stdio flush_all() - empty!(displayqueue) # discard pending display requests - send_ipython(publish[], msg_pub(execute_msg::Msg, "clear_output", - Dict("wait" => wait))) - stdio_bytes[] = 0 # reset output throttling + empty!(kernel.displayqueue) # discard pending display requests + send_ipython(kernel.publish[], kernel, msg_pub(kernel.execute_msg::Msg, "clear_output", + Dict("wait" => wait))) + kernel.stdio_bytes = 0 # reset output throttling + return nothing end @@ -305,10 +481,19 @@ Sets the maximum number of bytes, `max_output`, that can be written to stdout an stderr before getting truncated. A large value here allows a lot of output to be displayed in the notebook, potentially bogging down the browser. """ -function set_max_stdio(max_output::Integer) - max_output_per_request[] = max_output +function set_max_stdio(max_output::Integer; kernel) + kernel.max_output_per_request[] = max_output end +""" + reset_stdio_count() + +Reset the count of the number of bytes written to stdout/stderr. See +[`set_max_stdio`](@ref) for more details. +""" +function reset_stdio_count(kernel=_default_kernel) + kernel.stdio_bytes = 0 +end ####################################################################### @@ -325,5 +510,6 @@ include("handlers.jl") include("heartbeat.jl") include("inline.jl") include("kernel.jl") +include("precompile.jl") end # IJulia diff --git a/src/comm_manager.jl b/src/comm_manager.jl index 0d37f862..5a13d9d6 100644 --- a/src/comm_manager.jl +++ b/src/comm_manager.jl @@ -2,27 +2,13 @@ module CommManager using IJulia -import IJulia: Msg, uuid4, send_ipython, msg_pub +import IJulia: Msg, uuid4, send_ipython, msg_pub, Comm -export Comm, comm_target, msg_comm, send_comm, close_comm, +export comm_target, msg_comm, send_comm, close_comm, register_comm, comm_msg, comm_open, comm_close, comm_info_request -mutable struct Comm{target} - id::String - primary::Bool - on_msg::Function - on_close::Function - function (::Type{Comm{target}})(id, primary, on_msg, on_close) where {target} - comm = new{target}(id, primary, on_msg, on_close) - comms[id] = comm - return comm - end -end - -# This dict holds a map from CommID to Comm so that we can -# pick out the right Comm object when messages arrive -# from the front-end. -const comms = Dict{String, Comm}() +# Global variable kept around for backwards compatibility +comms::Dict{String, CommManager.Comm} = Dict{String, CommManager.Comm}() noop_callback(msg) = nothing function Comm(target, @@ -30,13 +16,14 @@ function Comm(target, primary=true, on_msg=noop_callback, on_close=noop_callback; + kernel=IJulia._default_kernel, data=Dict(), metadata=Dict()) - comm = Comm{Symbol(target)}(id, primary, on_msg, on_close) + comm = Comm{Symbol(target)}(id, primary, on_msg, on_close, kernel) if primary # Request a secondary object be created at the front end - send_ipython(IJulia.publish[], - msg_comm(comm, IJulia.execute_msg, "comm_open", + send_ipython(kernel.publish[], kernel, + msg_comm(comm, kernel.execute_msg, "comm_open", data, metadata, target_name=string(target))) end return comm @@ -44,13 +31,13 @@ end comm_target(comm :: Comm{target}) where {target} = target -function comm_info_request(sock, msg) +function comm_info_request(sock, kernel, msg) reply = if haskey(msg.content, "target_name") t = Symbol(msg.content["target_name"]) - filter(kv -> comm_target(kv.second) == t, comms) + filter(kv -> comm_target(kv.second) == t, kernel.comms) else # reply with all comms. - comms + kernel.comms end _comms = Dict{String, Dict{Symbol,Symbol}}() @@ -59,7 +46,7 @@ function comm_info_request(sock, msg) end content = Dict(:comms => _comms) - send_ipython(sock, + send_ipython(sock, kernel, msg_reply(msg, "comm_info_reply", content)) end @@ -76,17 +63,17 @@ function msg_comm(comm::Comm, m::IJulia.Msg, msg_type, end function send_comm(comm::Comm, data::Dict, - metadata::Dict = Dict(); kwargs...) - msg = msg_comm(comm, IJulia.execute_msg, "comm_msg", data, + metadata::Dict = Dict(); kernel=IJulia._default_kernel, kwargs...) + msg = msg_comm(comm, kernel.execute_msg, "comm_msg", data, metadata; kwargs...) - send_ipython(IJulia.publish[], msg) + send_ipython(kernel.publish[], kernel, msg) end function close_comm(comm::Comm, data::Dict = Dict(), - metadata::Dict = Dict(); kwargs...) - msg = msg_comm(comm, IJulia.execute_msg, "comm_close", data, + metadata::Dict = Dict(); kernel=IJulia._default_kernel, kwargs...) + msg = msg_comm(comm, kernel.execute_msg, "comm_close", data, metadata; kwargs...) - send_ipython(IJulia.publish[], msg) + send_ipython(kernel.publish[], kernel, msg) end function register_comm(comm::Comm, data) @@ -97,7 +84,7 @@ end # handlers for incoming comm_* messages -function comm_open(sock, msg) +function comm_open(sock, kernel, msg) if haskey(msg.content, "comm_id") comm_id = msg.content["comm_id"] if haskey(msg.content, "target_name") @@ -105,24 +92,24 @@ function comm_open(sock, msg) if !haskey(msg.content, "data") msg.content["data"] = Dict() end - comm = Comm(target, comm_id, false) + comm = Comm(target, comm_id, false; kernel) invokelatest(register_comm, comm, msg) - comms[comm_id] = comm + kernel.comms[comm_id] = comm else # Tear down comm to maintain consistency # if a target_name is not present - send_ipython(IJulia.publish[], - msg_comm(Comm(:notarget, comm_id), + send_ipython(kernel.publish[], kernel, + msg_comm(Comm(:notarget, comm_id, false; kernel), msg, "comm_close")) end end end -function comm_msg(sock, msg) +function comm_msg(sock, kernel, msg) if haskey(msg.content, "comm_id") comm_id = msg.content["comm_id"] - if haskey(comms, comm_id) - comm = comms[comm_id] + if haskey(kernel.comms, comm_id) + comm = kernel.comms[comm_id] else # We don't have that comm open return @@ -135,17 +122,17 @@ function comm_msg(sock, msg) end end -function comm_close(sock, msg) +function comm_close(sock, kernel, msg) if haskey(msg.content, "comm_id") comm_id = msg.content["comm_id"] - comm = comms[comm_id] + comm = kernel.comms[comm_id] if !haskey(msg.content, "data") msg.content["data"] = Dict() end comm.on_close(msg) - delete!(comms, comm.id) + delete!(kernel.comms, comm.id) end end diff --git a/src/display.jl b/src/display.jl index 8b9c66c0..89d9dde2 100644 --- a/src/display.jl +++ b/src/display.jl @@ -47,7 +47,7 @@ const ijulia_jsonmime_types = Vector{Union{MIME, Vector{MIME}}}([ Register a new MIME type. """ register_mime(x::Union{MIME, Vector{MIME}}) = push!(ijulia_mime_types, x) -register_mime(x::AbstractVector{<:MIME}) = push!(ijulia_mime_types, Vector{Mime}(x)) +register_mime(x::AbstractVector{<:MIME}) = push!(ijulia_mime_types, Vector{MIME}(x)) """ register_jsonmime(x::Union{MIME, Vector{MIME}}) @@ -56,7 +56,7 @@ register_mime(x::AbstractVector{<:MIME}) = push!(ijulia_mime_types, Vector{Mime} Register a new JSON MIME type. """ register_jsonmime(x::Union{MIME, Vector{MIME}}) = push!(ijulia_jsonmime_types, x) -register_jsonmime(x::AbstractVector{<:MIME}) = push!(ijulia_jsonmime_types, Vector{Mime}(x)) +register_jsonmime(x::AbstractVector{<:MIME}) = push!(ijulia_jsonmime_types, Vector{MIME}(x)) # return a String=>Any dictionary to attach as metadata # in Jupyter display_data and pyout messages @@ -137,13 +137,10 @@ function display_dict(x) end -# queue of objects to display at end of cell execution -const displayqueue = Any[] - # remove x from the display queue -function undisplay(x) - i = findfirst(isequal(x), displayqueue) - i !== nothing && splice!(displayqueue, i) +function undisplay(x, kernel) + i = findfirst(isequal(x), kernel.displayqueue) + i !== nothing && splice!(kernel.displayqueue, i) return x end @@ -181,8 +178,11 @@ function error_content(e, bt=catch_backtrace(); if !isempty(msg) pushfirst!(tb, msg) end - Dict("ename" => ename, "evalue" => evalue, - "traceback" => tb) + + # Specify the value type as Any because types other than String may be in + # the returned JSON. + Dict{String, Any}("ename" => ename, "evalue" => evalue, + "traceback" => tb) end ####################################################################### diff --git a/src/eventloop.jl b/src/eventloop.jl index 558f614f..2022a1bb 100644 --- a/src/eventloop.jl +++ b/src/eventloop.jl @@ -1,17 +1,28 @@ """ - eventloop(socket) + eventloop(socket, kernel) Generic event loop for one of the [kernel sockets](https://jupyter-client.readthedocs.io/en/latest/messaging.html#introduction). """ -function eventloop(socket) +function eventloop(socket, kernel) task_local_storage(:IJulia_task, "write task") try while true - msg = recv_ipython(socket) + local msg try - send_status("busy", msg) - invokelatest(get(handlers, msg.header["msg_type"], unknown_request), socket, msg) + msg = recv_ipython(socket, kernel) + catch e + if isa(e, EOFError) + # The socket was closed + return + else + rethrow() + end + end + + try + send_status("busy", kernel, msg) + invokelatest(get(handlers, msg.header["msg_type"], unknown_request), socket, kernel, msg) catch e if e isa InterruptException && _shutting_down[] # If we're shutting down, just return immediately @@ -23,11 +34,11 @@ function eventloop(socket) # kernel interruption to interrupt long calculations.) content = error_content(e, msg="KERNEL EXCEPTION") map(s -> println(orig_stderr[], s), content["traceback"]) - send_ipython(publish[], msg_pub(execute_msg, "error", content)) + send_ipython(kernel.publish[], kernel, msg_pub(kernel.execute_msg, "error", content)) end finally flush_all() - send_status("idle", msg) + send_status("idle", kernel, msg) end end catch e @@ -38,34 +49,39 @@ function eventloop(socket) # the Jupyter manager may send us a SIGINT if the user # chooses to interrupt the kernel; don't crash on this if isa(e, InterruptException) - eventloop(socket) + eventloop(socket, kernel) + elseif isa(e, ZMQ.StateError) + # This is almost certainly because of a closed socket + return else rethrow() end end end -const requests_task = Ref{Task}() - """ - waitloop() + waitloop(kernel) Main loop of a kernel. Runs the event loops for the control and shell sockets (note: in IJulia the shell socket is called `requests`). """ -function waitloop() - @async eventloop(control[]) - requests_task[] = @async eventloop(requests[]) - while true +function waitloop(kernel) + control_task = @async eventloop(kernel.control[], kernel) + kernel.requests_task[] = @async eventloop(kernel.requests[], kernel) + + while kernel.inited try - wait() + wait(kernel.stop_event) catch e # send interrupts (user SIGINT) to the code-execution task if isa(e, InterruptException) - @async Base.throwto(requests_task[], e) + @async Base.throwto(kernel.requests_task[], e) else rethrow() end + finally + wait(control_task) + wait(kernel.requests_task[]) end end end diff --git a/src/execute_request.jl b/src/execute_request.jl index bc1dcc02..09aa9b13 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -13,45 +13,36 @@ else # Pkg.jl#3777 Pkg.REPLMode.do_cmds(cmd, stdout) end -# global variable so that display can be done in the correct Msg context -execute_msg = Msg(["julia"], Dict("username"=>"jlkernel", "session"=>uuid4()), Dict()) -# global variable tracking the number of bytes written in the current execution -# request -const stdio_bytes = Ref(0) - import REPL: helpmode -# use a global array to accumulate "payloads" for the execute_reply message -const execute_payloads = Dict[] """ - execute_request(socket, msg) + execute_request(socket, kernel, msg) Handle a [execute request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#execute). This will execute Julia code, along with Pkg and shell commands. """ -function execute_request(socket, msg) +function execute_request(socket, kernel, msg) code = msg.content["code"] @vprintln("EXECUTING ", code) - global execute_msg = msg - global n, In, Out, ans - stdio_bytes[] = 0 + kernel.execute_msg = msg + kernel.stdio_bytes = 0 silent = msg.content["silent"] store_history = get(msg.content, "store_history", !silent) - empty!(execute_payloads) + empty!(kernel.execute_payloads) if !silent - n += 1 - send_ipython(publish[], + kernel.n += 1 + send_ipython(kernel.publish[], kernel, msg_pub(msg, "execute_input", - Dict("execution_count" => n, + Dict("execution_count" => kernel.n, "code" => code))) end silent = silent || REPL.ends_with_semicolon(code) if store_history - In[n] = code + kernel.In[kernel.n] = code end # "; ..." cells are interpreted as shell commands for run @@ -69,32 +60,32 @@ function execute_request(socket, msg) hcode = replace(code, r"^\s*\?" => "") try - for hook in preexecute_hooks + for hook in kernel.preexecute_hooks invokelatest(hook) end - ans = result = if hcode != code # help request + kernel.ans = result = if hcode != code # help request Core.eval(Main, helpmode(hcode)) else #run the code! occursin(magics_regex, code) && match(magics_regex, code).offset == 1 ? magics_help(code) : - SOFTSCOPE[] ? softscope_include_string(current_module[], code, "In[$n]") : - include_string(current_module[], code, "In[$n]") + SOFTSCOPE[] ? softscope_include_string(kernel.current_module, code, "In[$(kernel.n)]") : + include_string(kernel.current_module, code, "In[$(kernel.n)]") end if silent result = nothing - elseif (result !== nothing) && (result !== Out) + elseif (result !== nothing) && (result !== kernel.Out) if store_history - Out[n] = result + kernel.Out[kernel.n] = result end end user_expressions = Dict() for (v,ex) in msg.content["user_expressions"] try - value = include_string(current_module[], ex) + value = include_string(kernel.current_module, ex) # Like the IPython reference implementation, we return # something that looks like a `display_data` but also has a # `status` field: @@ -111,48 +102,55 @@ function execute_request(socket, msg) end end - for hook in postexecute_hooks + for hook in kernel.postexecute_hooks invokelatest(hook) end # flush pending stdio flush_all() + yield() + if haskey(kernel.bufs, "stdout") + send_stdout(kernel) + end + if haskey(kernel.bufs, "stderr") + send_stderr(kernel) + end - undisplay(result) # dequeue if needed, since we display result in pyout - invokelatest(display) # flush pending display requests + undisplay(result, kernel) # dequeue if needed, since we display result in pyout + @invokelatest display(kernel) # flush pending display requests if result !== nothing result_metadata = invokelatest(metadata, result) result_data = invokelatest(display_dict, result) - send_ipython(publish[], + send_ipython(kernel.publish[], kernel, msg_pub(msg, "execute_result", - Dict("execution_count" => n, - "metadata" => result_metadata, - "data" => result_data))) + Dict("execution_count" => kernel.n, + "metadata" => result_metadata, + "data" => result_data))) end - send_ipython(requests[], + send_ipython(kernel.requests[], kernel, msg_reply(msg, "execute_reply", Dict("status" => "ok", - "payload" => execute_payloads, - "execution_count" => n, - "user_expressions" => user_expressions))) - empty!(execute_payloads) + "payload" => kernel.execute_payloads, + "execution_count" => kernel.n, + "user_expressions" => user_expressions))) + empty!(kernel.execute_payloads) catch e bt = catch_backtrace() try # flush pending stdio flush_all() - for hook in posterror_hooks + for hook in kernel.posterror_hooks invokelatest(hook) end catch end - empty!(displayqueue) # discard pending display requests on an error + empty!(kernel.displayqueue) # discard pending display requests on an error content = error_content(e,bt) - send_ipython(publish[], msg_pub(msg, "error", content)) + send_ipython(kernel.publish[], kernel, msg_pub(msg, "error", content)) content["status"] = "error" content["execution_count"] = n - send_ipython(requests[], msg_reply(msg, "execute_reply", content)) + send_ipython(kernel.requests[], kernel, msg_reply(msg, "execute_reply", content)) end end diff --git a/src/handlers.jl b/src/handlers.jl index fa0d5a1c..72532334 100644 --- a/src/handlers.jl +++ b/src/handlers.jl @@ -87,7 +87,7 @@ function complete_type(T::DataType) end #Get typeMap for Jupyter completions -function complete_types(comps) +function complete_types(comps, kernel=_default_kernel) typeMap = [] for c in comps ctype = "" @@ -102,7 +102,7 @@ function complete_types(comps) expr = Meta.parse(c, raise=false) if typeof(expr) == Symbol try - ctype = complete_type(Core.eval(current_module[], :(typeof($expr)))) + ctype = complete_type(Core.eval(kernel.current_module, :(typeof($expr)))) catch end elseif !isa(expr, Expr) @@ -117,12 +117,12 @@ function complete_types(comps) end """ - complete_request(socket, msg) + complete_request(socket, kernel, msg) Handle a [completion request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#completion). """ -function complete_request(socket, msg) +function complete_request(socket, kernel, msg) code = msg.content["code"] cursor_chr = msg.content["cursor_pos"] cursorpos = chr2ind(msg, code, cursor_chr) @@ -131,7 +131,7 @@ function complete_request(socket, msg) cursorpos = min(cursorpos, lastindex(code)) if all(isspace, code[1:cursorpos]) - send_ipython(requests[], msg_reply(msg, "complete_reply", + send_ipython(kernel.requests[], kernel, msg_reply(msg, "complete_reply", Dict("status" => "ok", "metadata" => Dict(), "matches" => String[], @@ -141,7 +141,7 @@ function complete_request(socket, msg) end codestart = find_parsestart(code, cursorpos) - comps_, positions, should_complete = REPLCompletions.completions(code[codestart:end], cursorpos-codestart+1, current_module[]) + comps_, positions, should_complete = REPLCompletions.completions(code[codestart:end], cursorpos-codestart+1, kernel.current_module) comps = unique!(repl_completion_text.(comps_)) # positions = positions .+ (codestart - 1) on Julia 0.7 positions = (first(positions) + codestart - 1):(last(positions) + codestart - 1) @@ -156,14 +156,14 @@ function complete_request(socket, msg) cursor_start = ind2chr(msg, code, prevind(code, first(positions))) cursor_end = ind2chr(msg, code, last(positions)) if should_complete - metadata["_jupyter_types_experimental"] = complete_types(comps) + metadata["_jupyter_types_experimental"] = complete_types(comps, kernel) else # should_complete is false for cases where we only want to show # a list of possible completions but not complete, e.g. foo(\t pushfirst!(comps, code[positions]) end end - send_ipython(requests[], msg_reply(msg, "complete_reply", + send_ipython(kernel.requests[], kernel, msg_reply(msg, "complete_reply", Dict("status" => "ok", "matches" => comps, "metadata" => metadata, @@ -172,13 +172,13 @@ function complete_request(socket, msg) end """ - kernel_info_request(socket, msg) + kernel_info_request(socket, kernel, msg) Handle a [kernel info request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#kernel-info). """ -function kernel_info_request(socket, msg) - send_ipython(socket, +function kernel_info_request(socket, kernel, msg) + send_ipython(socket, kernel, msg_reply(msg, "kernel_info_reply", Dict("protocol_version" => "5.4", "implementation" => "ijulia", @@ -204,30 +204,30 @@ function kernel_info_request(socket, msg) end """ - connect_request(socket, msg) + connect_request(socket, kernel, msg) Handle a [connect request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#connect). """ -function connect_request(socket, msg) - send_ipython(requests[], +function connect_request(socket, kernel, msg) + send_ipython(kernel.requests[], kernel, msg_reply(msg, "connect_reply", - Dict("shell_port" => profile["shell_port"], - "iopub_port" => profile["iopub_port"], - "stdin_port" => profile["stdin_port"], - "hb_port" => profile["hb_port"]))) + Dict("shell_port" => kernel.profile["shell_port"], + "iopub_port" => kernel.profile["iopub_port"], + "stdin_port" => kernel.profile["stdin_port"], + "hb_port" => kernel.profile["hb_port"]))) end """ - shutdown_request(socket, msg) + shutdown_request(socket, kernel, msg) Handle a [shutdown request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#kernel-shutdown). After sending the reply this will exit the process. """ -function shutdown_request(socket, msg) +function shutdown_request(socket, kernel, msg) # stop heartbeat thread - stop_heartbeat(heartbeat[], heartbeat_context[]) + stop_heartbeat(kernel) # Shutdown the `requests` socket handler before sending any messages. This # is necessary because otherwise the event loop will be calling @@ -235,12 +235,14 @@ function shutdown_request(socket, msg) # deadlock when we try to send a message to it from the `control` socket # handler. global _shutting_down[] = true - @async Base.throwto(requests_task[], InterruptException()) + @async Base.throwto(kernel.requests_task[], InterruptException()) - send_ipython(requests[], msg_reply(msg, "shutdown_reply", - msg.content)) + # In protocol 5.4 the shutdown reply moved to the control socket + shutdown_socket = VersionNumber(msg) >= v"5.4" ? kernel.control[] : kernel.requests[] + send_ipython(shutdown_socket, kernel, + msg_reply(msg, "shutdown_reply", msg.content)) sleep(0.1) # short delay (like in ipykernel), to hopefully ensure shutdown_reply is sent - exit() + kernel.shutdown() end docdict(s::AbstractString) = display_dict(Core.eval(Main, helpmode(devnull, s))) @@ -375,12 +377,12 @@ function get_token(code, pos) end """ - inspect_request(socket, msg) + inspect_request(socket, kernel, msg) Handle a [introspection request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#introspection). """ -function inspect_request(socket, msg) +function inspect_request(socket, kernel, msg) try code = msg.content["code"] s = get_token(code, chr2ind(msg, code, msg.content["cursor_pos"])) @@ -390,59 +392,60 @@ function inspect_request(socket, msg) d = docdict(s) content = Dict("status" => "ok", "found" => !isempty(d), - "data" => d) + "data" => d, + "metadata" => Dict()) end - send_ipython(requests[], msg_reply(msg, "inspect_reply", content)) + send_ipython(kernel.requests[], kernel, msg_reply(msg, "inspect_reply", content)) catch e content = error_content(e, backtrace_top=:inspect_request); content["status"] = "error" - send_ipython(requests[], + send_ipython(kernel.requests[], kernel, msg_reply(msg, "inspect_reply", content)) end end """ - history_request(socket, msg) + history_request(socket, kernel, msg) Handle a [history request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#history). This is currently only a dummy implementation that doesn't actually do anything. """ -function history_request(socket, msg) +function history_request(socket, kernel, msg) # we will just send back empty history for now, pending clarification # as requested in ipython/ipython#3806 - send_ipython(requests[], + send_ipython(kernel.requests[], kernel, msg_reply(msg, "history_reply", Dict("history" => []))) end """ - is_complete_request(socket, msg) + is_complete_request(socket, kernel, msg) Handle a [completeness request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#code-completeness). """ -function is_complete_request(socket, msg) +function is_complete_request(socket, kernel, msg) ex = Meta.parse(msg.content["code"], raise=false) status = Meta.isexpr(ex, :incomplete) ? "incomplete" : Meta.isexpr(ex, :error) ? "invalid" : "complete" - send_ipython(requests[], + send_ipython(kernel.requests[], kernel, msg_reply(msg, "is_complete_reply", Dict("status"=>status, "indent"=>""))) end """ - interrupt_request(socket, msg) + interrupt_request(socket, kernel, msg) Handle a [interrupt request](https://jupyter-client.readthedocs.io/en/latest/messaging.html#kernel-interrupt). This will throw an `InterruptException` to the currently executing request handler. """ -function interrupt_request(socket, msg) - @async Base.throwto(requests_task[], InterruptException()) - send_ipython(socket, msg_reply(msg, "interrupt_reply", Dict())) +function interrupt_request(socket, kernel, msg) + @async Base.throwto(kernel.requests_task[], InterruptException()) + send_ipython(socket, kernel, msg_reply(msg, "interrupt_reply", Dict())) end -function unknown_request(socket, msg) +function unknown_request(socket, kernel, msg) @vprintln("UNKNOWN MESSAGE TYPE $(msg.header["msg_type"])") end diff --git a/src/heartbeat.jl b/src/heartbeat.jl index 75d9c399..9a423aea 100644 --- a/src/heartbeat.jl +++ b/src/heartbeat.jl @@ -4,8 +4,6 @@ # call in libzmq, which simply blocks forever, so the usual lack of # thread safety in Julia should not be an issue here. -const threadid = zeros(Int, 128) # sizeof(uv_thread_t) <= 8 on Linux, OSX, Win - # entry point for new thread function heartbeat_thread(heartbeat::Ptr{Cvoid}) @static if VERSION ≥ v"1.9.0-DEV.1588" # julia#46609 @@ -24,15 +22,16 @@ function heartbeat_thread(heartbeat::Ptr{Cvoid}) return ret end -function start_heartbeat(heartbeat) +function start_heartbeat(kernel) + heartbeat = kernel.heartbeat[] heartbeat.linger = 0 heartbeat_c = @cfunction(heartbeat_thread, Cint, (Ptr{Cvoid},)) ccall(:uv_thread_create, Cint, (Ptr{Int}, Ptr{Cvoid}, Ptr{Cvoid}), - threadid, heartbeat_c, heartbeat) + kernel.heartbeat_threadid, heartbeat_c, heartbeat) end -function stop_heartbeat(heartbeat, context) - if !isopen(context) +function stop_heartbeat(kernel) + if !isopen(kernel.heartbeat_context[]) # Do nothing if it has already been stopped (which can happen in the tests) return end @@ -41,12 +40,12 @@ function stop_heartbeat(heartbeat, context) # returns. We don't call ZMQ.close(::Context) directly because that # currently isn't threadsafe: # https://github.com/JuliaInterop/ZMQ.jl/issues/256 - ZMQ.lib.zmq_ctx_shutdown(context) - @ccall uv_thread_join(threadid::Ptr{Int})::Cint + ZMQ.lib.zmq_ctx_shutdown(kernel.heartbeat_context[]) + @ccall uv_thread_join(kernel.heartbeat_threadid::Ptr{Int})::Cint # Now that the heartbeat thread has joined and its guaranteed to no longer # be working on the heartbeat socket, we can safely close it and then the # context. - close(heartbeat) - close(context) + close(kernel.heartbeat[]) + close(kernel.heartbeat_context[]) end diff --git a/src/hmac.jl b/src/hmac.jl index 7389133c..553c5cd0 100644 --- a/src/hmac.jl +++ b/src/hmac.jl @@ -1,11 +1,8 @@ -const sha_ctx = Ref{SHA.SHA_CTX}() -const hmac_key = Ref{Vector{UInt8}}() - -function hmac(s1, s2, s3, s4) - if !isassigned(sha_ctx) +function hmac(s1, s2, s3, s4, kernel) + if !isassigned(kernel.sha_ctx) return "" else - hmac = SHA.HMAC_CTX(copy(sha_ctx[]), hmac_key[]) + hmac = SHA.HMAC_CTX(copy(kernel.sha_ctx[]), kernel.hmac_key) for s in (s1, s2, s3, s4) SHA.update!(hmac, codeunits(s)) end diff --git a/src/init.jl b/src/init.jl index a60c92f6..713a7a9f 100644 --- a/src/init.jl +++ b/src/init.jl @@ -1,36 +1,27 @@ import Random: seed! -import Logging: ConsoleLogger - -# use our own random seed for msg_id so that we -# don't alter the user-visible random state (issue #336) -const IJulia_RNG = seed!(Random.MersenneTwister(0)) -import UUIDs -uuid4() = string(UUIDs.uuid4(IJulia_RNG)) +import Sockets +import Logging +import Logging: AbstractLogger, ConsoleLogger const orig_stdin = Ref{IO}() const orig_stdout = Ref{IO}() const orig_stderr = Ref{IO}() +const orig_logger = Ref{AbstractLogger}() const SOFTSCOPE = Ref{Bool}() + +# Global variable kept around for backwards compatibility +profile::Dict{String, Any} = Dict{String, Any}() + + function __init__() seed!(IJulia_RNG) orig_stdin[] = stdin orig_stdout[] = stdout orig_stderr[] = stderr + orig_logger[] = Logging.global_logger() SOFTSCOPE[] = lowercase(get(ENV, "IJULIA_SOFTSCOPE", "yes")) in ("yes", "true") end -# the following constants need to be initialized in init(). -const publish = Ref{Socket}() -const raw_input = Ref{Socket}() -const requests = Ref{Socket}() -const control = Ref{Socket}() -const heartbeat = Ref{Socket}() -const heartbeat_context = Ref{Context}() -const profile = Dict{String,Any}() -const read_stdout = Ref{Base.PipeEndpoint}() -const read_stderr = Ref{Base.PipeEndpoint}() -const socket_locks = Dict{Socket,ReentrantLock}() - # needed for executing pkg commands on earlier Julia versions @static if VERSION < v"1.11" # similar to Pkg.REPLMode.MiniREPL, a minimal REPL-like emulator @@ -45,41 +36,65 @@ const socket_locks = Dict{Socket,ReentrantLock}() const minirepl = Ref{MiniREPL}() end +function getports(port_hint, n) + ports = Int[] + + for i in 1:n + port, server = Sockets.listenany(Sockets.localhost, port_hint) + close(server) + push!(ports, port) + port_hint = port + 1 + end + + return ports +end + +function create_profile(port_hint=8080; key=uuid4()) + ports = getports(port_hint, 5) + + Dict( + "transport" => "tcp", + "ip" => "127.0.0.1", + "control_port" => ports[1], + "shell_port" => ports[2], + "stdin_port" => ports[3], + "hb_port" => ports[4], + "iopub_port" => ports[5], + "signature_scheme" => "hmac-sha256", + "key" => key + ) +end + """ - init(args) + init(args, kernel) Initialize a kernel. `args` may either be empty or have one element containing the path to an existing connection file. If `args` is empty a connection file will be generated. """ -function init(args) - inited && error("IJulia is already running") +function init(args, kernel, profile=nothing) + !isnothing(_default_kernel) && error("IJulia is already running") if length(args) > 0 - merge!(profile, open(JSON.parse,args[1])) - verbose && println("PROFILE = $profile") - global connection_file = args[1] + merge!(kernel.profile, open(JSON.parse,args[1])) + kernel.verbose && println("PROFILE = $profile") + kernel.connection_file = args[1] + elseif !isnothing(profile) + merge!(kernel.profile, profile) else # generate profile and save let port0 = 5678 - merge!(profile, Dict{String,Any}( - "ip" => "127.0.0.1", - "transport" => "tcp", - "stdin_port" => port0, - "control_port" => port0+1, - "hb_port" => port0+2, - "shell_port" => port0+3, - "iopub_port" => port0+4, - "key" => uuid4() - )) + merge!(kernel.profile, create_profile(port0)) fname = "profile-$(getpid()).json" - global connection_file = "$(pwd())/$fname" - println("connect ipython with --existing $connection_file") + kernel.connection_file = "$(pwd())/$fname" + println("connect ipython with --existing $(kernel.connection_file)") open(fname, "w") do f - JSON.print(f, profile) + JSON.print(f, kernel.profile) end end end + profile = kernel.profile + if !isempty(profile["key"]) signature_scheme = get(profile, "signature_scheme", "hmac-sha256") isempty(signature_scheme) && (signature_scheme = "hmac-sha256") @@ -90,46 +105,57 @@ function init(args) error("Signature schemes other than SHA are not supported on IJulia anymore, requested signature_scheme is: $(signature_scheme)") end - global sha_ctx[] = getproperty(SHA, Symbol(uppercase(sigschm[2]), "_CTX"))() - global hmac_key[] = collect(UInt8, profile["key"]) + kernel.sha_ctx[] = getproperty(SHA, Symbol(uppercase(sigschm[2]), "_CTX"))() + kernel.hmac_key = collect(UInt8, profile["key"]) end - publish[] = Socket(PUB) - raw_input[] = Socket(ROUTER) - requests[] = Socket(ROUTER) - control[] = Socket(ROUTER) - heartbeat_context[] = Context() - heartbeat[] = Socket(heartbeat_context[], ROUTER) + kernel.publish[] = Socket(PUB) + kernel.raw_input[] = Socket(ROUTER) + kernel.requests[] = Socket(ROUTER) + kernel.control[] = Socket(ROUTER) + kernel.heartbeat_context[] = Context() + kernel.heartbeat[] = Socket(kernel.heartbeat_context[], ROUTER) sep = profile["transport"]=="ipc" ? "-" : ":" - bind(publish[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["iopub_port"])") - bind(requests[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["shell_port"])") - bind(control[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["control_port"])") - bind(raw_input[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["stdin_port"])") - bind(heartbeat[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["hb_port"])") + bind(kernel.publish[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["iopub_port"])") + bind(kernel.requests[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["shell_port"])") + bind(kernel.control[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["control_port"])") + bind(kernel.raw_input[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["stdin_port"])") + bind(kernel.heartbeat[], "$(profile["transport"])://$(profile["ip"])$(sep)$(profile["hb_port"])") # associate a lock with each socket so that multi-part messages # on a given socket don't get inter-mingled between tasks. - for s in (publish[], raw_input[], requests[], control[]) - socket_locks[s] = ReentrantLock() + for s in (kernel.publish[], kernel.raw_input[], kernel.requests[], kernel.control[]) + kernel.socket_locks[s] = ReentrantLock() end - start_heartbeat(heartbeat[]) - if capture_stdout - read_stdout[], = redirect_stdout() - redirect_stdout(IJuliaStdio(stdout,"stdout")) + start_heartbeat(kernel) + if kernel.capture_stdout + kernel.read_stdout[], = redirect_stdout() + redirect_stdout(IJuliaStdio(stdout, kernel, "stdout")) end - if capture_stderr - read_stderr[], = redirect_stderr() - redirect_stderr(IJuliaStdio(stderr,"stderr")) + if kernel.capture_stderr + kernel.read_stderr[], = redirect_stderr() + redirect_stderr(IJuliaStdio(stderr, kernel, "stderr")) end - redirect_stdin(IJuliaStdio(stdin,"stdin")) + if kernel.capture_stdin + redirect_stdin(IJuliaStdio(stdin, kernel, "stdin")) + end + @static if VERSION < v"1.11" minirepl[] = MiniREPL(TextDisplay(stdout)) end + watch_stdio(kernel) + pushdisplay(IJulia.InlineDisplay()) + logger = ConsoleLogger(Base.stderr) Base.CoreLogging.global_logger(logger) + IJulia._default_kernel = kernel + IJulia.CommManager.comms = kernel.comms + IJulia.profile = kernel.profile + + send_status("starting", kernel) + kernel.inited = true - send_status("starting") - global inited = true + kernel.waitloop_task[] = @async waitloop(kernel) end diff --git a/src/inline.jl b/src/inline.jl index a784e573..4b04510a 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -72,9 +72,10 @@ end for mime in ipy_mime @eval begin function display(d::InlineDisplay, ::MIME{Symbol($mime)}, x) + kernel = _default_kernel flush_all() # so that previous stream output appears in order - send_ipython(publish[], - msg_pub(execute_msg, "display_data", + send_ipython(kernel.publish[], kernel, + msg_pub(kernel.execute_msg, "display_data", Dict( "metadata" => metadata(x), # optional "transient" => transient(x), # optional @@ -94,14 +95,15 @@ display(d::InlineDisplay, m::MIME"text/javascript", x) = display(d, MIME("applic # the display message, also sending text/plain for text data. displayable(d::InlineDisplay, M::MIME) = istextmime(M) function display(d::InlineDisplay, M::MIME, x) + kernel = _default_kernel sx = limitstringmime(M, x) d = Dict(string(M) => sx) if istextmime(M) d["text/plain"] = sx # directly show text data, e.g. text/csv end flush_all() # so that previous stream output appears in order - send_ipython(publish[], - msg_pub(execute_msg, "display_data", + send_ipython(kernel.publish[], kernel, + msg_pub(kernel.execute_msg, "display_data", Dict("metadata" => metadata(x), # optional "transient" => transient(x), # optional "data" => d))) @@ -110,10 +112,11 @@ end # override display to send IPython a dictionary of all supported # output types, so that IPython can choose what to display. function display(d::InlineDisplay, x) - undisplay(x) # dequeue previous redisplay(x) + kernel = _default_kernel + undisplay(x, kernel) # dequeue previous redisplay(x) flush_all() # so that previous stream output appears in order - send_ipython(publish[], - msg_pub(execute_msg, "display_data", + send_ipython(kernel.publish[], kernel, + msg_pub(kernel.execute_msg, "display_data", Dict("metadata" => metadata(x), # optional "transient" => transient(x), # optional "data" => display_dict(x)))) @@ -124,15 +127,16 @@ end # an input cell has finished executing. function redisplay(d::InlineDisplay, x) - if !in(x,displayqueue) - push!(displayqueue, x) + kernel = _default_kernel + if !in(x, kernel.displayqueue) + push!(kernel.displayqueue, x) end end -function display() - q = copy(displayqueue) - empty!(displayqueue) # so that undisplay in display(x) is no-op +function display(kernel::Kernel) + q = copy(kernel.displayqueue) + empty!(kernel.displayqueue) # so that undisplay in display(x) is no-op for x in q - display(x) + display(x, kernel) end end diff --git a/src/jupyter.jl b/src/jupyter.jl index 6189ca6a..b56c5aa9 100644 --- a/src/jupyter.jl +++ b/src/jupyter.jl @@ -151,10 +151,14 @@ end Launches [qtconsole](https://qtconsole.readthedocs.io) for the current kernel. IJulia must be initialized already. """ -function qtconsole() +function qtconsole(kernel=_default_kernel) + if isnothing(kernel) + error("IJulia has not been started, cannot run qtconsole") + end + qtconsole = find_jupyter_subcommand("qtconsole") if inited - run(`$qtconsole --existing $connection_file`; wait=false) + run(`$qtconsole --existing $(kernel.connection_file)`; wait=false) else error("IJulia is not running. qtconsole must be called from an IJulia session.") end diff --git a/src/kernel.jl b/src/kernel.jl index 9d4b5aa2..daa1d58d 100644 --- a/src/kernel.jl +++ b/src/kernel.jl @@ -7,7 +7,8 @@ function run_kernel() ENV["LINES"] = get(ENV, "LINES", 30) ENV["COLUMNS"] = get(ENV, "COLUMNS", 80) - IJulia.init(ARGS) + println(IJulia.orig_stdout[], "Starting kernel event loops.") + IJulia.init(ARGS, IJulia.Kernel()) let startupfile = !isempty(DEPOT_PATH) ? abspath(DEPOT_PATH[1], "config", "startup_ijulia.jl") : "" isfile(startupfile) && Base.JLOptions().startupfile != 2 && Base.include(Main, startupfile) @@ -16,11 +17,6 @@ function run_kernel() # import things that we want visible in IJulia but not in REPL's using IJulia @eval Main import IJulia: ans, In, Out, clear_history - pushdisplay(IJulia.InlineDisplay()) - - println(IJulia.orig_stdout[], "Starting kernel event loops.") - IJulia.watch_stdio() - # check whether Revise is running and as needed configure it to run before every prompt if isdefined(Main, :Revise) let mode = get(ENV, "JULIA_REVISE", "auto") @@ -28,5 +24,5 @@ function run_kernel() end end - IJulia.waitloop() + wait(IJulia._default_kernel::Kernel) end diff --git a/src/msg.jl b/src/msg.jl index 910bcaab..3d2414d1 100644 --- a/src/msg.jl +++ b/src/msg.jl @@ -2,21 +2,6 @@ import Base.show export Msg, msg_pub, msg_reply, send_status, send_ipython -""" -IPython message struct. -""" -mutable struct Msg - idents::Vector{String} - header::Dict - content::Dict - parent_header::Dict - metadata::Dict - function Msg(idents, header::Dict, content::Dict, - parent_header=Dict{String,Any}(), metadata=Dict{String,Any}()) - new(idents,header,content,parent_header,metadata) - end -end - """ msg_header(m::Msg, msg_type::String) @@ -29,6 +14,8 @@ msg_header(m::Msg, msg_type::String) = Dict("msg_id" => uuid4(), "msg_type" => msg_type, "version" => "5.4") +Base.VersionNumber(m::Msg) = VersionNumber(m.header["version"]) + # PUB/broadcast messages use the msg_type as the ident, except for # stream messages which use the stream name (e.g. "stdout"). # [According to minrk, "this isn't well defined, or even really part @@ -48,12 +35,12 @@ function show(io::IO, msg::Msg) end """ - send_ipython(socket, m::Msg) + send_ipython(socket, kernel, m::Msg) Send a message `m`. This will lock `socket`. """ -function send_ipython(socket, m::Msg) - lock(socket_locks[socket]) +function send_ipython(socket, kernel, m::Msg) + lock(kernel.socket_locks[socket]) try @vprintln("SENDING ", m) for i in m.idents @@ -64,23 +51,23 @@ function send_ipython(socket, m::Msg) parent_header = json(m.parent_header) metadata = json(m.metadata) content = json(m.content) - send(socket, hmac(header, parent_header, metadata, content), more=true) + send(socket, hmac(header, parent_header, metadata, content, kernel), more=true) send(socket, header, more=true) send(socket, parent_header, more=true) send(socket, metadata, more=true) send(socket, content) finally - unlock(socket_locks[socket]) + unlock(kernel.socket_locks[socket]) end end """ - recv_ipython(socket) + recv_ipython(socket, kernel) Wait for and get a message. This will lock `socket`. """ -function recv_ipython(socket) - lock(socket_locks[socket]) +function recv_ipython(socket, kernel) + lock(kernel.socket_locks[socket]) try idents = String[] s = recv(socket, String) @@ -96,23 +83,34 @@ function recv_ipython(socket) parent_header = recv(socket, String) metadata = recv(socket, String) content = recv(socket, String) - if signature != hmac(header, parent_header, metadata, content) + if signature != hmac(header, parent_header, metadata, content, kernel) error("Invalid HMAC signature") # What should we do here? end + + # Note: don't remove these lines, they're useful for creating a + # precompilation workload. + # @show idents + # @show signature + # @show header + # @show parent_header + # @show metadata + # @show content + m = Msg(idents, JSON.parse(header), JSON.parse(content), JSON.parse(parent_header), JSON.parse(metadata)) @vprintln("RECEIVED $m") return m finally - unlock(socket_locks[socket]) + unlock(kernel.socket_locks[socket]) end end """ - send_status(state::AbstractString, parent_msg::Msg=execute_msg) + send_status(state::AbstractString, kernel, parent_msg::Msg=execute_msg) Publish a status message. """ -function send_status(state::AbstractString, parent_msg::Msg=execute_msg) - send_ipython(publish[], Msg([ "status" ], msg_header(parent_msg, "status"), - Dict("execution_state" => state), parent_msg.header)) +function send_status(state::AbstractString, kernel, parent_msg::Msg=kernel.execute_msg) + send_ipython(kernel.publish[], kernel, + Msg([ "status" ], msg_header(parent_msg, "status"), + Dict("execution_state" => state), parent_msg.header)) end diff --git a/src/precompile.jl b/src/precompile.jl new file mode 100644 index 00000000..a13adb6a --- /dev/null +++ b/src/precompile.jl @@ -0,0 +1,69 @@ +import PrecompileTools: @compile_workload + +# This key is used by the tests and precompilation workload to keep some +# consistency in the message signatures. +const _TEST_KEY = "a0436f6c-1916-498b-8eb9-e81ab9368e84" + +# How to update the precompilation workload: +# 1. Uncomment the `@show` expressions in `recv_ipython()` in msg.jl. +# 2. Copy this workload into tests/kernel.jl and update as desired: +# +# Kernel(profile; capture_stdout=false, capture_stderr=false) do kernel +# jupyter_client(profile) do client +# kernel_info(client) +# execute(client, "42") +# execute(client, "error(42)") +# end +# end +# +# 3. When the above runs it will print out the contents of the received messages +# as strings. You can copy these verbatim into the precompilation workload +# below. Note that if you modify any step of the workload you will need to +# update *all* the messages to ensure they have the right parent +# headers/signatures. +@compile_workload begin + local profile = create_profile(45_000; key=_TEST_KEY) + + Kernel(profile; capture_stdout=false, capture_stderr=false, capture_stdin=false) do kernel + # Connect as a client to the kernel + requests_socket = ZMQ.Socket(ZMQ.DEALER) + ip = profile["ip"] + port = profile["shell_port"] + ZMQ.connect(requests_socket, "tcp://$(ip):$(port)") + + # kernel_info + idents = ["d2bd8e47-b2c9cd130d2967a19f52c1a3"] + signature = "3c4f523a0e8b80e5b3e35756d75f62d12b851e1fd67c609a9119872e911f83d2" + header = "{\"msg_id\": \"d2bd8e47-b2c9cd130d2967a19f52c1a3_3534705_0\", \"msg_type\": \"kernel_info_request\", \"username\": \"james\", \"session\": \"d2bd8e47-b2c9cd130d2967a19f52c1a3\", \"date\": \"2025-02-20T22:29:47.616834Z\", \"version\": \"5.4\"}" + parent_header = "{}" + metadata = "{}" + content = "{}" + + ZMQ.send_multipart(requests_socket, [only(idents), "", signature, header, parent_header, metadata, content]) + ZMQ.recv_multipart(requests_socket, String) + + # Execute `42` + idents = ["d2bd8e47-b2c9cd130d2967a19f52c1a3"] + signature = "758c034ba5efb4fd7fd5a5600f913bc634739bf6a2c1e1d87e88b008706337bc" + header = "{\"msg_id\": \"d2bd8e47-b2c9cd130d2967a19f52c1a3_3534705_1\", \"msg_type\": \"execute_request\", \"username\": \"james\", \"session\": \"d2bd8e47-b2c9cd130d2967a19f52c1a3\", \"date\": \"2025-02-20T22:29:49.835131Z\", \"version\": \"5.4\"}" + parent_header = "{}" + metadata = "{}" + content = "{\"code\": \"42\", \"silent\": false, \"store_history\": true, \"user_expressions\": {}, \"allow_stdin\": true, \"stop_on_error\": true}" + + ZMQ.send_multipart(requests_socket, [only(idents), "", signature, header, parent_header, metadata, content]) + ZMQ.recv_multipart(requests_socket, String) + + # Execute `error(42)` + idents = ["d2bd8e47-b2c9cd130d2967a19f52c1a3"] + signature = "953702763b65d9b0505f34ae0eb195574b9c2c65eebedbfa8476150133649801" + header = "{\"msg_id\": \"d2bd8e47-b2c9cd130d2967a19f52c1a3_3534705_2\", \"msg_type\": \"execute_request\", \"username\": \"james\", \"session\": \"d2bd8e47-b2c9cd130d2967a19f52c1a3\", \"date\": \"2025-02-20T22:29:50.320836Z\", \"version\": \"5.4\"}" + parent_header = "{}" + metadata = "{}" + content = "{\"code\": \"error(42)\", \"silent\": false, \"store_history\": true, \"user_expressions\": {}, \"allow_stdin\": true, \"stop_on_error\": true}" + + ZMQ.send_multipart(requests_socket, [only(idents), "", signature, header, parent_header, metadata, content]) + ZMQ.recv_multipart(requests_socket, String) + + close(requests_socket) + end +end diff --git a/src/stdio.jl b/src/stdio.jl index a0979587..f41c4037 100644 --- a/src/stdio.jl +++ b/src/stdio.jl @@ -7,11 +7,12 @@ Wrapper type around redirected stdio streams, both for overloading things like """ struct IJuliaStdio{IO_t <: IO} <: Base.AbstractPipe io::IOContext{IO_t} + kernel::Kernel end -IJuliaStdio(io::IO, stream::AbstractString="unknown") = +IJuliaStdio(io::IO, kernel::Kernel, stream::AbstractString="unknown") = IJuliaStdio{typeof(io)}(IOContext(io, :color=>Base.have_color, :jupyter_stream=>stream, - :displaysize=>displaysize())) + :displaysize=>displaysize()), kernel) Base.pipe_reader(io::IJuliaStdio) = io.io.io Base.pipe_writer(io::IJuliaStdio) = io.io.io Base.lock(io::IJuliaStdio) = lock(io.io.io) @@ -24,18 +25,6 @@ Base.displaysize(io::IJuliaStdio) = displaysize(io.io) Base.unwrapcontext(io::IJuliaStdio) = Base.unwrapcontext(io.io) Base.setup_stdio(io::IJuliaStdio, readable::Bool) = Base.setup_stdio(io.io.io, readable) -if VERSION < v"1.7.0-DEV.254" - for s in ("stdout", "stderr", "stdin") - f = Symbol("redirect_", s) - sq = QuoteNode(Symbol(s)) - @eval function Base.$f(io::IJuliaStdio) - io[:jupyter_stream] != $s && throw(ArgumentError(string("expecting ", $s, " stream"))) - Core.eval(Base, Expr(:(=), $sq, io)) - return io - end - end -end - # logging in verbose mode goes to original stdio streams. Use macros # so that we do not even evaluate the arguments in no-verbose modes @@ -48,7 +37,7 @@ end macro vprintln(x...) quote - if verbose::Bool + if _default_kernel.verbose println(orig_stdout[], get_log_preface(), $(map(esc, x)...)) end end @@ -56,20 +45,15 @@ end macro verror_show(e, bt) quote - if verbose::Bool + if _default_kernel.verbose showerror(orig_stderr[], $(esc(e)), $(esc(bt))) end end end -#name=>iobuffer for each stream ("stdout","stderr") so they can be sent in flush -const bufs = Dict{String, IOBuffer}() -const bufs_locks = Dict{String, ReentrantLock}() const stream_interval = 0.1 # maximum number of bytes in libuv/os buffer before emptying const max_bytes = 10*1024 -# max output per code cell is 512 kb by default -const max_output_per_request = Ref(1 << 19) """ watch_stream(rd::IO, name::AbstractString) @@ -80,31 +64,33 @@ when buffer contains more than `max_bytes` bytes. Otherwise, if data is availabl `stream_interval` seconds (see the `Timer`'s set up in `watch_stdio`). Truncate the output to `max_output_per_request` bytes per execution request since excessive output can bring browsers to a grinding halt. """ -function watch_stream(rd::IO, name::AbstractString) +function watch_stream(rd::IO, name::AbstractString, kernel) task_local_storage(:IJulia_task, "read $name task") try buf = IOBuffer() - bufs[name] = buf - bufs_locks[name] = ReentrantLock() + buf_lock = ReentrantLock() + kernel.bufs[name] = buf + kernel.bufs_locks[name] = buf_lock + while !eof(rd) # blocks until something is available nb = bytesavailable(rd) if nb > 0 - stdio_bytes[] += nb + kernel.stdio_bytes += nb # if this stream has surpassed the maximum output limit then ignore future bytes - if stdio_bytes[] >= max_output_per_request[] + if kernel.stdio_bytes >= kernel.max_output_per_request[] read(rd, nb) # read from libuv/os buffer and discard - if stdio_bytes[] - nb < max_output_per_request[] - send_ipython(publish[], msg_pub(execute_msg, "stream", - Dict("name" => "stderr", "text" => "Excessive output truncated after $(stdio_bytes[]) bytes."))) + if kernel.stdio_bytes - nb < kernel.max_output_per_request[] + send_ipython(kernel.publish[], kernel, msg_pub(kernel.execute_msg, "stream", + Dict("name" => "stderr", "text" => "Excessive output truncated after $(kernel.stdio_bytes) bytes."))) end else - @lock bufs_locks[name] write(buf, read(rd, nb)) + @lock buf_lock write(buf, read(rd, nb)) end end - @lock bufs_locks[name] if buf.size > 0 + @lock buf_lock if buf.size > 0 if buf.size >= max_bytes #send immediately - send_stream(name) + send_stream(name, kernel) end end end @@ -112,22 +98,22 @@ function watch_stream(rd::IO, name::AbstractString) # the IPython manager may send us a SIGINT if the user # chooses to interrupt the kernel; don't crash on this if isa(e, InterruptException) - watch_stream(rd, name) + watch_stream(rd, name, kernel) else rethrow() end end end -function send_stdio(name) - if verbose::Bool && !haskey(task_local_storage(), :IJulia_task) +function send_stdio(name, kernel) + if kernel.verbose && !haskey(task_local_storage(), :IJulia_task) task_local_storage(:IJulia_task, "send $name task") end - send_stream(name) + send_stream(name, kernel) end -send_stdout(t::Timer) = send_stdio("stdout") -send_stderr(t::Timer) = send_stdio("stderr") +send_stdout(kernel) = send_stdio("stdout", kernel) +send_stderr(kernel) = send_stdio("stderr", kernel) """ Jupyter associates cells with message headers. Once a cell's execution state has @@ -139,14 +125,15 @@ is updating Signal graph state, it's execution state is busy, meaning Jupyter will not drop stream messages if Interact can set the header message under which the stream messages will be sent. Hence the need for this function. """ -function set_cur_msg(msg) - global execute_msg = msg +function set_cur_msg(msg, kernel) + kernel.execute_msg = msg end -function send_stream(name::AbstractString) - buf = bufs[name] +function send_stream(name::AbstractString, kernel) + buf = kernel.bufs[name] + buf_lock = kernel.bufs_locks[name] - @lock bufs_locks[name] if buf.size > 0 + @lock buf_lock if buf.size > 0 d = take!(buf) n = num_utf8_trailing(d) dextra = d[end-(n-1):end] @@ -166,8 +153,8 @@ function send_stream(name::AbstractString) print(sbuf, '\n') s = String(take!(sbuf)) end - send_ipython(publish[], - msg_pub(execute_msg, "stream", + send_ipython(kernel.publish[], kernel, + msg_pub(kernel.execute_msg, "stream", Dict("name" => name, "text" => s))) end end @@ -197,15 +184,15 @@ Display the `prompt` string, request user input, and return the string entered by the user. If `password` is `true`, the user's input is not displayed during typing. """ -function readprompt(prompt::AbstractString; password::Bool=false) - if !execute_msg.content["allow_stdin"] +function readprompt(prompt::AbstractString; kernel=_default_kernel, password::Bool=false) + if !kernel.execute_msg.content["allow_stdin"] error("IJulia: this front-end does not implement stdin") end - send_ipython(raw_input[], - msg_reply(execute_msg, "input_request", + send_ipython(kernel.raw_input[], kernel, + msg_reply(kernel.execute_msg, "input_request", Dict("prompt"=>prompt, "password"=>password))) while true - msg = recv_ipython(raw_input[]) + msg = recv_ipython(kernel.raw_input[], kernel) if msg.header["msg_type"] == "input_reply" return msg.content["value"] else @@ -246,17 +233,19 @@ function readline(io::IJuliaStdio) end end -function watch_stdio() +function watch_stdio(kernel) task_local_storage(:IJulia_task, "init task") - if capture_stdout - read_task = @async watch_stream(read_stdout[], "stdout") + if kernel.capture_stdout + kernel.watch_stdout_task[] = @async watch_stream(kernel.read_stdout[], "stdout", kernel) + errormonitor(kernel.watch_stdout_task[]) #send stdout stream msgs every stream_interval secs (if there is output to send) - Timer(send_stdout, stream_interval, interval=stream_interval) + kernel.watch_stdout_timer[] = Timer(_ -> send_stdout(kernel), stream_interval, interval=stream_interval) end - if capture_stderr - readerr_task = @async watch_stream(read_stderr[], "stderr") + if kernel.capture_stderr + kernel.watch_stderr_task[] = @async watch_stream(kernel.read_stderr[], "stderr", kernel) + errormonitor(kernel.watch_stderr_task[]) #send STDERR stream msgs every stream_interval secs (if there is output to send) - Timer(send_stderr, stream_interval, interval=stream_interval) + kernel.watch_stderr_timer[] = Timer(_ -> send_stderr(kernel), stream_interval, interval=stream_interval) end end @@ -280,5 +269,5 @@ import Base.flush function flush(io::IJuliaStdio) flush(io.io) oslibuv_flush() - send_stream(get(io,:jupyter_stream,"unknown")) + send_stream(get(io,:jupyter_stream,"unknown"), io.kernel) end diff --git a/test/Project.toml b/test/Project.toml index 4354dbcf..5bd300d0 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,13 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" +ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" diff --git a/test/comm.jl b/test/comm.jl index e39b8a3d..e91a68f3 100644 --- a/test/comm.jl +++ b/test/comm.jl @@ -2,16 +2,18 @@ using Test import IJulia: Comm, comm_target @testset "comm" begin + kernel = IJulia.Kernel() + target = :notarget comm_id = "6BA197D8A67A455196279A59EB2FE844" - comm = Comm(target, comm_id, false) + comm = Comm(target, comm_id, false; kernel) @test :notarget == comm_target(comm) @test !comm.primary # comm_info_request in comm_manager.jl comms = Dict{String, Comm}( - "id" => Comm(Symbol("jupyter.widget"), "id", false) + "id" => Comm(Symbol("jupyter.widget"), "id", false; kernel) ) msg_content = Dict("target_name" => "jupyter.widget") reply = if haskey(msg_content, "target_name") @@ -33,4 +35,4 @@ import IJulia: Comm, comm_target @test Dict("id"=>Dict(:target_name=>Symbol("jupyter.widget"))) == _comms content = Dict(:comms => _comms) @test Dict(:comms=>Dict("id"=>Dict(:target_name=>Symbol("jupyter.widget")))) == content -end \ No newline at end of file +end diff --git a/test/kernel.jl b/test/kernel.jl new file mode 100644 index 00000000..cb6cb70d --- /dev/null +++ b/test/kernel.jl @@ -0,0 +1,390 @@ +ENV["JULIA_CONDAPKG_ENV"] = "@ijulia-tests" +ENV["JULIA_CONDAPKG_BACKEND"] = "MicroMamba" +ENV["JULIA_CONDAPKG_VERBOSITY"] = -1 + +# If you're running the tests locally you could uncomment the two environment +# variables below. This will be a bit faster since it stops CondaPkg from +# re-resolving the environment each time (but you do need to run it at least +# once locally to initialize the `@ijulia-tests` environment). +# ENV["JULIA_PYTHONCALL_EXE"] = joinpath(Base.DEPOT_PATH[1], "conda_environments", "ijulia-tests", "bin", "python") +# ENV["JULIA_CONDAPKG_BACKEND"] = "Null" + + +using Test +import Sockets +import Sockets: listenany + +import ZMQ +import PythonCall +import PythonCall: Py, pyimport, pyconvert, pytype, pystr + +# A little bit of hackery to fix the version number sent by the client. See: +# https://github.com/jupyter/jupyter_client/pull/1054 +jupyter_client_lib = pyimport("jupyter_client") +jupyter_client_lib.session.protocol_version = "5.4" + +const BlockingKernelClient = jupyter_client_lib.BlockingKernelClient + +import IJulia: Kernel +# These symbols are imported so that we can test that setproperty!(::Kernel) +# will propagate changes from the corresponding Kernel fields to the +# module-global variables. +import IJulia: ans, In, Out + + +function test_py_get!(get_func, result) + try + result[] = get_func(timeout=0) + return true + catch ex + exception_type = pyconvert(String, ex.t.__name__) + if exception_type != "Empty" + rethrow() + end + + return false + end +end + +function recursive_pyconvert(x) + x_type = pyconvert(String, pytype(x).__name__) + + if x_type == "dict" + x = pyconvert(Dict{String, Any}, x) + for key in copy(keys(x)) + if x[key] isa Py + x[key] = recursive_pyconvert(x[key]) + elseif x[key] isa PythonCall.PyDict + x[key] = recursive_pyconvert(x[key].py) + end + end + elseif x_type == "str" + x = pyconvert(String, x) + end + + return x +end + +# Calling methods directly with `reply=true` on the BlockingKernelClient will +# cause a deadlock because the client will block the whole thread while polling +# the socket, which means that the thread will never enter a GC safepoint so any +# other code that happens to allocate will get blocked. Instead, we send +# requests by themselves and then poll the appropriate socket with `timeout=0` +# so that the Python code will never block and we never get into a deadlock. +function make_request(request_func, get_func, args...; wait=true, kwargs...) + request_func(args...; kwargs..., reply=false) + if !wait + return nothing + end + + result = Ref{Py}() + timeout = haskey(ENV, "CI") ? 120 : 20 + if timedwait(() -> test_py_get!(get_func, result), timeout) == :timed_out + error("Jupyter channel get timed out") + end + + return recursive_pyconvert(result[]) +end + +kernel_info(client) = make_request(client.kernel_info, client.get_shell_msg) +comm_info(client) = make_request(client.comm_info, client.get_shell_msg) +history(client) = make_request(client.history, client.get_shell_msg) +shutdown(client; wait=true) = make_request(client.shutdown, client.get_control_msg; wait) +execute(client, code) = make_request(client.execute, client.get_shell_msg; code) +inspect(client, code) = make_request(client.inspect, client.get_shell_msg; code) +get_stdin_msg(client) = make_request(Returns(nothing), client.get_stdin_msg) +get_iopub_msg(client) = make_request(Returns(nothing), client.get_iopub_msg) + +function get_iopub_msgtype(client, msg_type) + while true + msg = get_iopub_msg(client) + if msg["header"]["msg_type"] == msg_type + return msg + end + end +end + +get_execute_result(client) = get_iopub_msgtype(client, "execute_result") +get_comm_close(client) = get_iopub_msgtype(client, "comm_close") +get_comm_msg(client) = get_iopub_msgtype(client, "comm_msg") + +function msg_ok(msg) + ok = msg["content"]["status"] == "ok" + if !ok + @error "Kernel is not ok" msg["content"] + end + + return ok +end + +msg_error(msg) = msg["content"]["status"] == "error" + +function jupyter_client(f, profile) + client = BlockingKernelClient() + client.load_connection_info(profile) + client.start_channels() + + try + f(client) + finally + client.stop_channels() + end +end + +@testset "Kernel" begin + profile = IJulia.create_profile(; key=IJulia._TEST_KEY) + profile_kwargs = Dict([Symbol(key) => value for (key, value) in profile]) + profile_kwargs[:key] = pystr(profile_kwargs[:key]).encode() + + @testset "getproperty()/setproperty!()" begin + kernel = Kernel() + + # Test setting special fields that should be mirrored to global variables + for field in (:n, :ans, :inited) + # Save the old value so we can restore them afterwards + old_value = getproperty(kernel, field) + + test_value = field === :inited ? true : 10 + setproperty!(kernel, field, test_value) + @test getproperty(IJulia, field) == test_value + @test getproperty(kernel, field) == test_value + + setproperty!(kernel, field, old_value) + end + end + + @testset "Explicit tests with jupyter_client" begin + # Some of these tests have their own kernel instance to avoid + # interfering with the state of other tests. + + # Test clear_history() + Kernel(profile; capture_stdout=false, capture_stderr=false) do kernel + jupyter_client(profile) do client + for i in 1:10 + @test msg_ok(execute(client, "$(i)")) + end + @test length(kernel.In) == 10 + @test msg_ok(execute(client, "IJulia.clear_history(-1:5)")) + @test Set(keys(kernel.In)) == Set(6:11) # The 11th entry is the call to clear_history() + @test msg_ok(execute(client, "IJulia.clear_history()")) + @test isempty(kernel.In) + @test isempty(kernel.Out) + end + end + + # Test input + Kernel(profile; capture_stdout=false, capture_stderr=false) do kernel + jupyter_client(profile) do client + # The input system in Jupyter is a bit convoluted. First we + # make a request to the kernel: + client.execute("readline()") + # Then wait for readline(::IJuliaStdio) to send its own + # `input_request` message on the stdin socket. + @test msg_ok(get_stdin_msg(client)) + # Send an `input_reply` back + client.input("foo") + + # Wait for the original `execute_request` to complete and + # send an `execute_result` message with the 'input'. + msg = get_execute_result(client) + @test msg["content"]["data"]["text/plain"] == "\"foo\"" + end + end + + shutdown_called = false + Kernel(profile; capture_stdout=false, capture_stderr=false, shutdown=() -> shutdown_called = true) do kernel + jupyter_client(profile) do client + @testset "Comms" begin + # Try opening a Comm without a target_name, which should + # only trigger a comm_close message. + open_msg = IJulia.Msg(["foo"], + Dict("username" => "user", + "session" => "session1"), + Dict("comm_id" => "foo", + "data" => Dict())) + IJulia.comm_open(kernel.requests[], kernel, open_msg) + @test get_comm_close(client)["content"]["comm_id"] == "foo" + + # Setting the target_name should cause the Comm to be created + open_msg.content["target_name"] = "foo" + IJulia.comm_open(kernel.requests[], kernel, open_msg) + @test kernel.comms["foo"] isa IJulia.Comm{:foo} + + @test haskey(comm_info(client)["content"]["comms"], "foo") + + # Smoke test for comm_msg (incoming to the kernel) + msg_msg = IJulia.Msg(["foo"], + Dict("username" => "user", + "session" => "session1"), + Dict("comm_id" => "foo", + "data" => Dict())) + IJulia.comm_msg(kernel.requests[], kernel, msg_msg) + + # Test comm_msg (outgoing from the kernel) + IJulia.send_comm(kernel.comms["foo"], Dict(1 => 2)) + @test get_comm_msg(client)["content"]["data"]["1"] == 2 + + # Test comm_close (outgoing from the kernel) + IJulia.close_comm(kernel.comms["foo"]) + # Should this also delete the Comm from kernel.comms? + @test get_comm_close(client)["content"]["comm_id"] == "foo" + + # Test comm_close (incoming to the kernel) + close_msg = IJulia.Msg(["foo"], + Dict("username" => "user", + "session" => "session1"), + Dict("comm_id" => "foo", + "data" => Dict())) + IJulia.comm_close(kernel.requests[], kernel, close_msg) + @test !haskey(kernel.comms, "foo") + end + + # Test load()/load_string() + mktemp() do path, _ + write(path, "42") + + msg = execute(client, "IJulia.load($(repr(path)))") + @test msg_ok(msg) + @test length(msg["content"]["payload"]) == 1 + end + + # Test hooks + @testset "Hooks" begin + preexecute = false + postexecute = false + posterror = false + preexecute_hook = () -> preexecute = !preexecute + postexecute_hook = () -> postexecute = !postexecute + posterror_hook = () -> posterror = !posterror + IJulia.push_preexecute_hook(preexecute_hook) + IJulia.push_postexecute_hook(postexecute_hook) + IJulia.push_posterror_hook(posterror_hook) + @test msg_ok(execute(client, "42")) + + # The pre/post hooks should've been called but not the posterror hook + @test preexecute + @test postexecute + @test !posterror + + # With a throwing cell the posterror hook should be called + @test msg_error(execute(client, "error(42)")) + @test posterror + + # After popping the hooks they should no longer be executed + preexecute = false + postexecute = false + posterror = false + IJulia.pop_preexecute_hook(preexecute_hook) + IJulia.pop_postexecute_hook(postexecute_hook) + IJulia.pop_posterror_hook(posterror_hook) + @test msg_ok(execute(client, "42")) + @test msg_error(execute(client, "error(42)")) + @test !preexecute + @test !postexecute + @test !posterror + end + + # Smoke tests + @test msg_ok(kernel_info(client)) + @test msg_ok(comm_info(client)) + @test msg_ok(history(client)) + @test msg_ok(execute(client, "IJulia.set_verbose(false)")) + @test msg_ok(execute(client, "flush(stdout)")) + + # Test history(). This test requires `capture_stdout=false`. + IJulia.clear_history() + @test msg_ok(execute(client, "1")) + @test msg_ok(execute(client, "42")) + stdout_pipe = Pipe() + redirect_stdout(stdout_pipe) do + IJulia.history() + end + close(stdout_pipe.in) + @test collect(eachline(stdout_pipe)) == ["1", "42"] + + # Test that certain global variables are updated in kernel.current_module + @test msg_ok(execute(client, "42")) + @test msg_ok(execute(client, "ans == 42")) + @test kernel.ans + + # Test shutdown_request + @test msg_ok(shutdown(client)) + @test timedwait(() -> shutdown_called, 10) == :ok + end + end + end + + @testset "jupyter_kernel_test" begin + stdout_pipe = Pipe() + stderr_pipe = Pipe() + Base.link_pipe!(stdout_pipe) + Base.link_pipe!(stderr_pipe) + stdout_str = "" + stderr_str = "" + test_proc = nothing + + Kernel(profile; shutdown=Returns(nothing)) do kernel + test_file = joinpath(@__DIR__, "kernel_test.py") + + mktemp() do connection_file, io + # Write the connection file + jupyter_client_lib.connect.write_connection_file(; fname=connection_file, profile_kwargs...) + + try + # Run jupyter_kernel_test + cmd = ignorestatus(`$(PythonCall.C.python_executable_path()) $(test_file)`) + cmd = addenv(cmd, "IJULIA_TESTS_CONNECTION_FILE" => connection_file) + cmd = pipeline(cmd; stdout=stdout_pipe, stderr=stderr_pipe) + test_proc = run(cmd) + finally + close(stdout_pipe.in) + close(stderr_pipe.in) + stdout_str = read(stdout_pipe, String) + stderr_str = read(stderr_pipe, String) + close(stdout_pipe) + close(stderr_pipe) + end + end + end + + if !isempty(stdout_str) + @info "jupyter_kernel_test stdout:" + println(stdout_str) + end + if !isempty(stderr_str) + @info "jupyter_kernel_test stderr:" + println(stderr_str) + end + if !success(test_proc) + error("jupyter_kernel_test failed") + end + end + + # run_kernel() is the function that's actually run by Jupyter + @testset "run_kernel()" begin + julia = joinpath(Sys.BINDIR, "julia") + + mktemp() do connection_file, io + # Write the connection file + jupyter_client_lib.connect.write_connection_file(; fname=connection_file, profile_kwargs...) + + cmd = `$julia --startup-file=no --project=$(Base.active_project()) -e 'import IJulia; IJulia.run_kernel()' $(connection_file)` + kernel_proc = run(pipeline(cmd; stdout, stderr); wait=false) + try + jupyter_client(profile) do client + @test msg_ok(kernel_info(client)) + @test msg_ok(execute(client, "42")) + + # Note that we don't wait for a reply because the kernel + # will shut down almost immediately and it's not guaranteed + # we'll receive the reply. + shutdown(client; wait=false) + end + + @test timedwait(() -> process_exited(kernel_proc), 10) == :ok + finally + kill(kernel_proc) + end + end + end +end diff --git a/test/kernel_test.py b/test/kernel_test.py new file mode 100644 index 00000000..ef06003b --- /dev/null +++ b/test/kernel_test.py @@ -0,0 +1,136 @@ +import os +import unittest +import typing as t + +import jupyter_kernel_test +import jupyter_client +from jupyter_client import KernelManager, BlockingKernelClient + +# A little bit of hackery to fix the version number sent by the client. See: +# https://github.com/jupyter/jupyter_client/pull/1054 +jupyter_client.session.protocol_version = "5.4" + +# This is a modified version of jupyter_client.start_new_kernel() that uses an +# existing kernel from a connection file rather than trying to launch one. +def start_new_kernel2( + startup_timeout: float = 1, kernel_name: str = "python", **kwargs: t.Any +) -> t.Tuple[KernelManager, BlockingKernelClient]: + """Start a new kernel, and return its Manager and Client""" + connection_file = os.environ["IJULIA_TESTS_CONNECTION_FILE"] + + km = KernelManager(owns_kernel=False) + km.load_connection_file(connection_file) + km._connect_control_socket() + + kc = BlockingKernelClient() + kc.load_connection_file(connection_file) + + kc.start_channels() + try: + kc.wait_for_ready(timeout=startup_timeout) + except RuntimeError: + kc.stop_channels() + km.shutdown_kernel() + raise + + return km, kc + +# Modified version of: +# https://github.com/jupyter/jupyter_kernel_test/blob/main/test_ipykernel.py +# +# We skip the pager and history tests because they aren't supporteed. +class IJuliaTests(jupyter_kernel_test.KernelTests): + # Required -------------------------------------- + + # The name identifying an installed kernel to run the tests against + kernel_name = "IJuliaKernel" + + # Optional -------------------------------------- + + # language_info.name in a kernel_info_reply should match this + language_name = "julia" + + # the normal file extension (including the leading dot) for this language + # checked against language_info.file_extension in kernel_info_reply + file_extension = ".jl" + + # Code in the kernel's language to write "hello, world" to stdout + code_hello_world = 'println("hello, world")' + + # code which should cause (any) text to be written to STDERR + code_stderr = 'println(stderr, "foo")' + + # samples for the autocompletion functionality + # for each dictionary, `text` is the input to try and complete, and + # `matches` the list of all complete matching strings which should be found + completion_samples = [ + { + "text": "zi", + "matches": {"zip"}, + }, + ] + + # samples for testing code-completeness (used by console only) + # these samples should respectively be unambiguously complete statements + # (which should be executed on ), incomplete statements or code + # which should be identified as invalid + complete_code_samples = ["1", 'print("hello, world")', "f(x) = x*2"] + incomplete_code_samples = ['print("hello', "f(x) = x*"] + invalid_code_samples = ["import = 7q"] + + # code which should generate a (user-level) error in the kernel, and send + # a traceback to the client + code_generate_error = "error(42)" + + # Samples of code which generate a result value (ie, some text + # displayed as Out[n]) + code_execute_result = [{"code": "6*7", "result": "42"}] + + # Samples of code which should generate a rich display output, and + # the expected MIME type. + # Note that we slice down the image so it doesn't display such a massive + # amount of text when debugging. + code_display_data = [ + { + "code": 'using FileIO, ImageShow; display(load("mandrill.png")[1:5, 1:5])', + "mime": "image/png" + }, + { + "code": 'display(MIME("image/svg+xml"), read("plus.svg", String))', + "mime": "image/svg+xml" + }, + { + "code": 'display(MIME("text/latex"), "\\frac{1}{2}")', + "mime": "text/latex" + }, + { + "code": 'display(MIME("text/markdown"), "# header")', + "mime": "text/markdown" + }, + { + "code": 'display(MIME("text/html"), "foo")', + "mime": "text/html" + }, + { + "code": 'display("foo")', + "mime": "text/plain" + } + ] + + # test the support for object inspection + # the sample should be a name about which the kernel can give some help + # information (a built-in function is probably a good choice) + # only the default inspection level (equivalent to ipython "obj?") + # is currently tested + code_inspect_sample = "zip" + + # a code sample which should cause a `clear_output` message to be sent to + # the client + code_clear_output = "IJulia.clear_output()" + + @classmethod + def setUpClass(cls) -> None: + cls.km, cls.kc = start_new_kernel2(kernel_name=cls.kernel_name) + +if __name__ == "__main__": + unittest.main() diff --git a/test/mandrill.png b/test/mandrill.png new file mode 100644 index 00000000..8164b6f6 Binary files /dev/null and b/test/mandrill.png differ diff --git a/test/plus.svg b/test/plus.svg new file mode 100644 index 00000000..b74f17a6 --- /dev/null +++ b/test/plus.svg @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 223fa65d..319b4907 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,4 +11,11 @@ for file in TEST_FILES include(file) end -Aqua.test_all(IJulia; piracies=(; broken=true)) +# MicroMamba (and thus CondaPkg and PythonCall) are not supported on 32bit +if Sys.WORD_SIZE != 32 + include("kernel.jl") +else + @warn "Skipping the Kernel tests on 32bit" +end + +Aqua.test_all(IJulia) diff --git a/test/stdio.jl b/test/stdio.jl index 3c22b3af..bd47d017 100644 --- a/test/stdio.jl +++ b/test/stdio.jl @@ -2,9 +2,10 @@ using Test using IJulia @testset "stdio" begin + kernel = IJulia.Kernel() mktemp() do path, io - redirect_stdout(IJulia.IJuliaStdio(io, "stdout")) do + redirect_stdout(IJulia.IJuliaStdio(io, kernel, "stdout")) do println(Base.stdout, "stdout") println("print") end @@ -12,17 +13,17 @@ using IJulia seek(io, 0) @test read(io, String) == "stdout\nprint\n" if VERSION < v"1.7.0-DEV.254" - @test_throws ArgumentError redirect_stdout(IJulia.IJuliaStdio(io, "stderr")) - @test_throws ArgumentError redirect_stdout(IJulia.IJuliaStdio(io, "stdin")) - @test_throws ArgumentError redirect_stderr(IJulia.IJuliaStdio(io, "stdout")) - @test_throws ArgumentError redirect_stderr(IJulia.IJuliaStdio(io, "stdin")) - @test_throws ArgumentError redirect_stdin(IJulia.IJuliaStdio(io, "stdout")) - @test_throws ArgumentError redirect_stdin(IJulia.IJuliaStdio(io, "stderr")) + @test_throws ArgumentError redirect_stdout(IJulia.IJuliaStdio(io, kernel, "stderr")) + @test_throws ArgumentError redirect_stdout(IJulia.IJuliaStdio(io, kernel, "stdin")) + @test_throws ArgumentError redirect_stderr(IJulia.IJuliaStdio(io, kernel, "stdout")) + @test_throws ArgumentError redirect_stderr(IJulia.IJuliaStdio(io, kernel, "stdin")) + @test_throws ArgumentError redirect_stdin(IJulia.IJuliaStdio(io, kernel, "stdout")) + @test_throws ArgumentError redirect_stdin(IJulia.IJuliaStdio(io, kernel, "stderr")) end end mktemp() do path, io - redirect_stderr(IJulia.IJuliaStdio(io, "stderr")) do + redirect_stderr(IJulia.IJuliaStdio(io, kernel, "stderr")) do println(Base.stderr, "stderr") end flush(io) @@ -31,11 +32,14 @@ using IJulia end mktemp() do path, io - redirect_stdin(IJulia.IJuliaStdio(io, "stdin")) do + redirect_stdin(IJulia.IJuliaStdio(io, kernel, "stdin")) do # We can't actually do anything here because `IJuliaexecute_msg` has not # yet been initialized, so we just make sure that redirect_stdin does # not error. end end -end \ No newline at end of file + kernel.stdio_bytes = 42 + IJulia.reset_stdio_count(kernel) + @test kernel.stdio_bytes == 0 +end