Allow server and worker code to be Revised during quarto preview (#263)

MichaelHatherly · web-flow · commit 7fd563608c9a · 2025-02-22T08:06:51.000Z
* Allow server code to be `Revise`d during `quarto preview`

* Add support for revising the worker package code

* Ensure `Revise` changes apply to every `render` call

* Move some code out of `worker_init` and into startup.jl

* Drop some `invokelatest` calls, update docs
diff --git a/Project.toml b/Project.toml
@@ -14,6 +14,7 @@ JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 ProgressLogging = "33c8b6b6-d38a-422a-b730-caa89a2f386c"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -34,6 +35,7 @@ JSON3 = "1"
 Logging = "1.6"
 Pkg = "1.6"
 PrecompileTools = "1"
+Preferences = "1"
 ProgressLogging = "0.1.4"
 REPL = "1.6"
 Random = "1.6"
diff --git a/README.md b/README.md
@@ -124,6 +124,11 @@ Import `RemoteREPL` into a notebook cell and run
 `Main.QuartoNotebookWorker.remote_repl()`. Then in a separate REPL run
 `RemoteREPL.connect_repl()` to connect to the notebook process for debugging.
 
+### Integration with `Revise.jl`
+
+See [revise/README.md](./revise/README.md) for information on how to use
+`Revise` with this package while developing changes.
+
 ### Adding package "integrations"
 
 Some packages require custom integrations to be written to make them behave as
diff --git a/justfile b/justfile
@@ -0,0 +1,15 @@
+default:
+    just -l
+
+changelog:
+    julia --project=.ci .ci/changelog.jl
+
+close:
+    kill $(ps aux | grep '[j]ulia.*quartonotebookrunner\.jl' | awk '{print $2}')
+
+format:
+    julia --project=.ci .ci/format.jl
+
+revise *args:
+    julia --startup-file=no --project=revise -e 'import Pkg; Pkg.develop(; path = ".")'
+    julia --startup-file=no revise/quarto.jl {{args}}
diff --git a/revise/LocalPreferences.toml b/revise/LocalPreferences.toml
@@ -0,0 +1,2 @@
+[QuartoNotebookRunner]
+enable_revise = true
diff --git a/revise/Project.toml b/revise/Project.toml
@@ -0,0 +1,3 @@
+[deps]
+QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
diff --git a/revise/README.md b/revise/README.md
@@ -0,0 +1,31 @@
+# `Revise` integration for `quarto preview`
+
+This environment allows for running `QuartoNotebookRunner` with `Revise`
+enabled. That means that you can edit code within the `QuartoNotebookRunner`
+module that a `quarto` process is using for `engine: julia` notebooks and have
+those changes take effect on the next render request that is triggered by a
+notebook file change. Without this feature you need to manually close and
+restart the `julia` process on each change.
+
+## Usage
+
+A `justfile` is provided that will start a `quarto` server with `Revise` enabled.
+
+```
+$ just revise quarto preview filename.qmd
+```
+
+If you terminate the command and need to force close any leftover `julia` server
+processes then run:
+
+```
+$ just close
+```
+
+## `QuartoNotebookWorker` code revision
+
+To enable `Revise` for the `QuartoNotebookWorker` module, you must have
+`Revise` installed as a dependency in the notebook environment, it cannot be a
+global environment dependency since this is not part of the `LOAD_PATH` for
+notebooks. Then set the `QUARTO_ENABLE_REVISE=true` environment variable for
+your notebook in the `julia.env` frontmatter key.
diff --git a/revise/quarto.jl b/revise/quarto.jl
@@ -0,0 +1,2 @@
+isempty(ARGS) && error("No arguments provided")
+run(addenv(`$ARGS`, "QUARTO_JULIA_PROJECT" => @__DIR__))
diff --git a/src/QuartoNotebookRunner.jl b/src/QuartoNotebookRunner.jl
@@ -27,6 +27,7 @@ import IterTools
 import JSON3
 import Logging
 import PrecompileTools
+import Preferences
 import ProgressLogging
 import REPL
 import Random
diff --git a/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerReviseExt.jl b/src/QuartoNotebookWorker/ext/QuartoNotebookWorkerReviseExt.jl
@@ -1,10 +1,15 @@
 module QuartoNotebookWorkerReviseExt
 
 import Revise
-import QuartoNotebookWorker
+import QuartoNotebookWorker as QNW
 
 function __init__()
     @debug "extension has been loaded" Revise QuartoNotebookWorker
 end
 
+function QNW._revise_hook(::Nothing)
+    isempty(Revise.revision_queue) || Base.invokelatest(Revise.revise; throw = true)
+    return nothing
+end
+
 end
diff --git a/src/QuartoNotebookWorker/src/refresh.jl b/src/QuartoNotebookWorker/src/refresh.jl
@@ -24,6 +24,9 @@ function refresh!(path, original_options, options = original_options)
     return nothing
 end
 
+revise_hook() = _revise_hook(nothing)
+_revise_hook(::Any) = nothing
+
 function rget(dict, keys, default)
     value = dict
     for key in keys
diff --git a/src/server.jl b/src/server.jl
@@ -155,10 +155,10 @@ function refresh!(file::File, options::Dict)
             cd(() -> Malt.Worker(; exe, exeflags = _exeflags, env), dirname(file.path))
         file.exe = exe
         file.exeflags = exeflags
+        file.env = env
         init!(file, options)
     end
-    expr = :(refresh!($(options)))
-    remote_eval_fetch_channeled(file.worker, expr)
+    remote_eval_fetch_channeled(file.worker, :(refresh!($(options)); revise_hook()))
 end
 
 """
diff --git a/src/socket.jl b/src/socket.jl
@@ -212,7 +212,48 @@ function serve(;
     return SocketServer(socket_server, notebook_server, port, task, key)
 end
 
-function _handle_response(
+if Preferences.@load_preference("enable_revise", false)
+    let mod = Ref{Module}(),
+        pkgid = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise")
+
+        global function _try_revise()
+            if !isassigned(mod)
+                try
+                    mod[] = Base.require(pkgid)
+                    package_module = @__MODULE__
+                    _, package_files = Base.invokelatest(mod[].modulefiles, package_module)
+                    if !isnothing(package_files)
+                        Base.invokelatest(mod[].track, package_module, package_files)
+                    end
+                catch error
+                    return "Could not load `Revise`: $(error)"
+                end
+            end
+            if !isempty(mod[].revision_queue)
+                try
+                    Base.invokelatest(mod[].revise; throw = true)
+                catch error
+                    return "Failed to run `Revise.revise`: $(error)"
+                end
+            end
+            return nothing
+        end
+    end
+
+    @noinline function _handle_response(socket, args...)
+        revise_error = _try_revise()
+        if isnothing(revise_error)
+            return Base.invokelatest(_handle_response_internal, socket, args...)
+        else
+            return _write_json(socket, _log_error(revise_error))
+        end
+    end
+else
+    @inline _try_revise() = nothing
+    @inline _handle_response(args...) = _handle_response_internal(args...)
+end
+
+function _handle_response_internal(
     socket,
     notebooks::Server,
     request::@NamedTuple{type::String, content::Union{String,Dict{String,Any}}},
diff --git a/src/startup.jl b/src/startup.jl
@@ -110,7 +110,23 @@ capture() do
     end
 end
 
-# Step 4:
+# Step: 4
+#
+# Import `Revise` if the `QUARTO_ENABLE_REVISE` environment variable is set.
+# This happens prior to importing `QuartoNotebookWorker` so that `Revise` can
+# track the worker package, as well as anything that the user has loaded. This
+# setting is an internal setting and should not be used by end-users. They
+# should do a manual `import Revise` in their notebook if they need `Revise`
+# support.
+const QUARTO_ENABLE_REVISE = get(ENV, "QUARTO_ENABLE_REVISE", "false") == "true"
+capture() do
+    if QUARTO_ENABLE_REVISE
+        pkgid = Base.PkgId(Base.UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise")
+        Base.require(pkgid)
+    end
+end
+
+# Step 5:
 #
 # Now load in the worker package. This may trigger package precompilation on
 # first load, hence it is run under a `capture` should it fail to run.
@@ -123,13 +139,34 @@ const QuartoNotebookWorker = capture() do
     )
 end
 
-# Step 5:
+# Step 6:
+#
+# Define the notebook interface that the server process will call.
+render(args...; kwargs...) = QuartoNotebookWorker.render(args...; kwargs...)
+revise_hook() = @static QUARTO_ENABLE_REVISE ? QuartoNotebookWorker.revise_hook() : nothing
+
+# Issue #192
+#
+# Malt itself uses a new task for each `remote_eval` and because of
+# this, random number streams are not consistent across runs even if
+# seeded, as each task introduces a new state for its task-local RNG.
+# As a workaround, we feed all `remote_eval` requests through these
+# channels, such that the task executing code is always the same.
+const stable_execution_task_channel_out = Channel()
+const stable_execution_task_channel_in = Channel() do chan
+    for expr in chan
+        result = Core.eval(Main, expr)
+        put!(stable_execution_task_channel_out, result)
+    end
+end
+
+# Step 6:
 #
 # Ensures that the LOAD_PATH is returned to it's previous state without the
 # `@stdlib` that was pushed to it near the start of the file.
 popfirst!(LOAD_PATH)
 
-# Step 6:
+# Step 7:
 #
 # This calls into the main socket server loop, which does not terminate until
 # the process is finished off and the notebook needs closing. So anything
diff --git a/src/worker.jl b/src/worker.jl
@@ -1,28 +1,11 @@
 function worker_init(f::File, options::Dict)
     return quote
-        # Issue #192
-        #
-        # Malt itself uses a new task for each `remote_eval` and because of
-        # this, random number streams are not consistent across runs even if
-        # seeded, as each task introduces a new state for its task-local RNG.
-        # As a workaround, we feed all `remote_eval` requests through these
-        # channels, such that the task executing code is always the same.
-        const stable_execution_task_channel_out = Channel()
-        const stable_execution_task_channel_in = Channel() do chan
-            for expr in chan
-                result = Core.eval(Main, expr)
-                put!(stable_execution_task_channel_out, result)
-            end
-        end
-
         let QNW = Main.QuartoNotebookWorker
             QNW.NotebookState.PROJECT[] = Base.active_project()
             QNW.NotebookState.OPTIONS[] = $(options)
             QNW.NotebookState.define_notebook_module!(Main)
             global refresh!(args...) = QNW.refresh!($(f.path), $(options), args...)
-            global render(args...; kwargs...) = QNW.render(args...; kwargs...)
         end
-
         nothing
     end
 end

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+[QuartoNotebookRunner]`
	`2`	`+enable_revise = true`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[deps]`
	`2`	`+QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4"`
	`3`	`+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+isempty(ARGS) && error("No arguments provided")`
	`2`	+run(addenv(`$ARGS`, "QUARTO_JULIA_PROJECT" => @__DIR__))