Better docstring for at-spawn

Author: jpsamaroo

src/thunk.jl

@@ -280,54 +280,6 @@ function Base.showerror(io::IO, ex::ThunkFailedException)
     print(io, "  This Thunk:  $t_str")
 end
 
-"""
-    spawn(f, args...; kwargs...) -> DTask
-
-Spawns a task with `f` as the function, `args` as the arguments, and `kwargs`
-as the keyword arguments, returning an `DTask`. Uses a scheduler running
-in the background to execute code.
-"""
-function spawn(f, args...; kwargs...)
-    @nospecialize f args kwargs
-
-    # Get all options and determine which propagate beyond this task
-    options = get_options()
-    propagates = get(options, :propagates, ())
-    propagates = Tuple(unique(Symbol[propagates..., keys(options)...]))
-    if length(args) >= 1 && first(args) isa Options
-        spawn_options = first(args).options
-        options = merge(options, spawn_options)
-        args = args[2:end]
-    end
-
-    # Wrap f in a Chunk if necessary
-    processor = haskey(options, :processor) ? options.processor : nothing
-    scope = haskey(options, :scope) ? options.scope : nothing
-    if !isnothing(processor) || !isnothing(scope)
-        f = tochunk(f,
-                    something(processor, get_options(:processor, OSProc())),
-                    something(scope, get_options(:scope, DefaultScope())))
-    end
-
-    # Process the args and kwargs into Pair form
-    args_kwargs = args_kwargs_to_pairs(args, kwargs)
-
-    # Get task queue, and don't let it propagate
-    task_queue = get_options(:task_queue, EagerTaskQueue())
-    options = NamedTuple(filter(opt->opt[1] != :task_queue, Base.pairs(options)))
-    propagates = filter(prop->prop != :task_queue, propagates)
-    options = merge(options, (;propagates))
-
-    # Construct task spec and handle
-    spec = EagerTaskSpec(f, args_kwargs, options)
-    task = eager_spawn(spec)
-
-    # Enqueue the task into the task queue
-    enqueue!(task_queue, spec=>task)
-
-    return task
-end
-
 """
     @par [opts] f(args...; kwargs...) -> Thunk
 
@@ -358,12 +310,40 @@ macro par(exs...)
 end
 
 """
-    @spawn [opts] f(args...) -> Thunk
-
-Convenience macro like `Dagger.@par`, but eagerly executed from the moment it's
-called (equivalent to `spawn`).
-
-See the docs for `@par` for more information and usage examples.
+    Dagger.@spawn [option=value]... f(args...; kwargs...) -> DTask
+
+Spawns a Dagger `DTask` that will call `f(args...; kwargs...)`. This `DTask` is like a Julia `Task`, and has many similarities:
+- The `DTask` can be `wait`'d on and `fetch`'d from to see its final result
+- By default, the `DTask` will be automatically run on the first available compute resource
+- If all dependencies are satisfied, the `DTask` will be run as soon as possible
+- The `DTask` may be run in parallel with other `DTask`s, and the scheduler will automatically manage dependencies
+- If a `DTask` throws an exception, it will be propagated to any calls to `fetch`, but not to calls to `wait`
+
+However, the `DTask` also has many key differences from a `Task`:
+- The `DTask` may run on any thread of any Julia process, and even on a remote machine, in your cluster (see `Distributed.addprocs`)
+- The `DTask` might automatically utilize GPUs or other accelerators, if available
+- If arguments to a `DTask` are also `DTask`s, then the scheduler will execute those arguments' `DTask`s first, before running the "downstream" task
+- If an argument to a `DTask` `t2` is a `DTask` `t1`, then the *result* of `t1` (gotten via `fetch(t1)`) will be passed to `t2` (no need for `t2` to call `fetch`!)
+- `DTask`s are generally expected to be defined "functionally", meaning that they should not mutate global state, mutate their arguments, or have side effects
+- `DTask`s are function call-focused, meaning that `Dagger.@spawn` expects a single function call, and not a block of code
+- All `DTask` arguments are expected to be safe to serialize and send to other Julia processes; if not, use the `scope` option or `Dagger.@mutable` to control execution location
+
+Options to the `DTask` can be set before the call to `f` with key-value syntax, e.g.
+`Dagger.@spawn myopt=2 do_something(1, 3.0)`, which would set the option
+`myopt` to `2` for this task. Multiple options may be provided, which are
+specified like `Dagger.@spawn myopt=2 otheropt=4 do_something(1, 3.0)`.
+
+These options control a variety of properties of the resulting `DTask`:
+- `scope`: The execution "scope" of the task, which determines where the task will run. By default, the task will run on the first available compute resource. If you have multiple compute resources, you can specify a scope to run the task on a specific resource. For example, `Dagger.@spawn scope=Dagger.scope(worker=2) do_something(1, 3.0)` would run `do_something(1, 3.0)` on worker 2.
+- `meta`: If `true`, instead of the scheduler automatically fetching values from other tasks, the raw `Chunk` objects will be passed to `f`. Useful for doing manual fetching or manipulation of `Chunk` references. Non-`Chunk` arguments are still passed as-is.
+
+Other options exist; see `Dagger.Sch.ThunkOptions` for the full list.
+
+This macro is a semi-thin wrapper around `Dagger.spawn` - it creates a call to
+`Dagger.spawn` on `f` with arguments `args` and keyword arguments `kwargs`, and
+also passes along any options in an `Options` struct. For example,
+`Dagger.@spawn myopt=2 do_something(1, 3.0)` would essentially become
+`Dagger.spawn(do_something, Dagger.Options(;myopt=2), 1, 3.0)`.
 """
 macro spawn(exs...)
     opts = exs[1:end-1]
@@ -417,6 +397,54 @@ end
 _par(ex::Symbol; kwargs...) = esc(ex)
 _par(ex; kwargs...) = ex
 
+"""
+    Dagger.spawn(f, args...; kwargs...) -> DTask
+
+Spawns a `DTask` that will call `f(args...; kwargs...)`. Also supports passing a
+`Dagger.Options` struct as the first argument to set task options. See
+`Dagger.@spawn` for more details on `DTask`s.
+"""
+function spawn(f, args...; kwargs...)
+    @nospecialize f args kwargs
+
+    # Get all options and determine which propagate beyond this task
+    options = get_options()
+    propagates = get(options, :propagates, ())
+    propagates = Tuple(unique(Symbol[propagates..., keys(options)...]))
+    if length(args) >= 1 && first(args) isa Options
+        spawn_options = first(args).options
+        options = merge(options, spawn_options)
+        args = args[2:end]
+    end
+
+    # Wrap f in a Chunk if necessary
+    processor = haskey(options, :processor) ? options.processor : nothing
+    scope = haskey(options, :scope) ? options.scope : nothing
+    if !isnothing(processor) || !isnothing(scope)
+        f = tochunk(f,
+                    something(processor, get_options(:processor, OSProc())),
+                    something(scope, get_options(:scope, DefaultScope())))
+    end
+
+    # Process the args and kwargs into Pair form
+    args_kwargs = args_kwargs_to_pairs(args, kwargs)
+
+    # Get task queue, and don't let it propagate
+    task_queue = get_options(:task_queue, EagerTaskQueue())
+    options = NamedTuple(filter(opt->opt[1] != :task_queue, Base.pairs(options)))
+    propagates = filter(prop->prop != :task_queue, propagates)
+    options = merge(options, (;propagates))
+
+    # Construct task spec and handle
+    spec = EagerTaskSpec(f, args_kwargs, options)
+    task = eager_spawn(spec)
+
+    # Enqueue the task into the task queue
+    enqueue!(task_queue, spec=>task)
+
+    return task
+end
+
 persist!(t::Thunk) = (t.persist=true; t)
 cache_result!(t::Thunk) = (t.cache=true; t)