Resume causes 500 internal error and aborts task #1305
Description
Running a plan via the CLI, pausing it, then trying to resume fails with the server raising
ValueError: There is no context set for tracing despite the fact that a task is running, something has gone wrong...
The full traceback
2025-12-15 13:05:18,192 INFO Change state on <bluesky.run_engine.RunEngine object at 0x70ca40f97750> from 'running' -> 'idle'
2025-12-15 13:05:18,200 ERROR Exception in ASGI application
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 622, in contingency_wrapper
ret = yield from plan
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plans.py", line 1166, in inner_scan_nd
yield from per_step(detectors, step, pos_cache)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1975, in __iter__
return (yield from self._iter)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plan_stubs.py", line 1743, in one_nd_step
yield from take_reading(list(detectors) + list(motors)) # type: ignore # Movable issue
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1975, in __iter__
return (yield from self._iter)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plan_stubs.py", line 1485, in trigger_and_read
return (yield from rewindable_wrapper(inner_trigger_and_read(), rewindable))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 749, in rewindable_wrapper
return (yield from plan)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plan_stubs.py", line 1480, in inner_trigger_and_read
ret = yield from contingency_wrapper(read_plan(), except_plan=exception_path, else_plan=standard_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 641, in contingency_wrapper
yield from else_plan()
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plan_stubs.py", line 1474, in standard_path
yield from save()
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1975, in __iter__
return (yield from self._iter)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plan_stubs.py", line 128, in save
return (yield Msg("save"))
^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1677, in _run
new_response = await coro(msg)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2077, in _save
await current_run.save(msg)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/bundlers.py", line 574, in save
indices_generated = await self._pack_external_assets(self._asset_docs_cache, message_stream_name=desc_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/bundlers.py", line 896, in _pack_external_assets
await self.emit(DocumentNames(name), doc)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2661, in emit
self.emit_sync(name, doc)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2658, in emit_sync
self.dispatcher.process(name, doc)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2681, in process
exceptions = self.cb_registry.process(name, name.name, doc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 443, in process
func(*args, **kwargs)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 533, in __call__
return mtd(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/worker/task_worker.py", line 581, in _on_document
raise ValueError(
ValueError: There is no context set for tracing despite the fact that a task is running, something has gone wrong...
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/peter/.local/share/uv/python/cpython-3.11.11-linux-x86_64-gnu/lib/python3.11/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/runner.py", line 176, in import_and_run_function
value = func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/interface.py", line 229, in resume_worker
worker().resume()
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/observability_utils/tracing/decorators.py", line 151, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/worker/task_worker.py", line 410, in resume
self._ctx.run_engine.resume()
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1018, in resume
plan_return = self._resume_task()
^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1121, in _resume_task
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1756, in _run
raise err
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1610, in _run
msg = self._plan_stack[-1].send(resp)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/dodal/plan_stubs/data_session.py", line 57, in attach_data_session_metadata_wrapper
yield from plan
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/dodal/plans/spec_path.py", line 47, in spec_scan
yield from bp.scan_nd(tuple(detectors), _as_cycler(spec), md=_md)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/plans.py", line 1168, in scan_nd
return (yield from inner_scan_nd())
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 1013, in stage_wrapper
return (yield from finalize_wrapper(inner(), unstage_devices()))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 548, in finalize_wrapper
ret = yield from plan
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 1011, in inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
return (yield from plan)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 370, in run_wrapper
yield from contingency_wrapper(plan, except_plan=except_plan, else_plan=close_run)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/preprocessors.py", line 632, in contingency_wrapper
ret = yield from except_plan(e)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1677, in _run
new_response = await coro(msg)
^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 1901, in _close_run
ret = await current_run.close_run(msg)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/bundlers.py", line 192, in close_run
await self.emit(DocumentNames.stop, doc)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2661, in emit
self.emit_sync(name, doc)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2658, in emit_sync
self.dispatcher.process(name, doc)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/run_engine.py", line 2681, in process
exceptions = self.cb_registry.process(name, name.name, doc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 443, in process
func(*args, **kwargs)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/bluesky/utils/__init__.py", line 533, in __call__
return mtd(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/worker/task_worker.py", line 581, in _on_document
raise ValueError(
ValueError: There is no context set for tracing despite the fact that a task is running, something has gone wrong...
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/fastapi/applications.py", line 1139, in __call__
await super().__call__(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/applications.py", line 107, in __call__
await self.middleware_stack(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/opentelemetry/instrumentation/asgi/__init__.py", line 810, in __call__
await self.app(scope, otel_receive, otel_send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 186, in __call__
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 164, in __call__
await self.app(scope, receive, _send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/opentelemetry/instrumentation/fastapi/__init__.py", line 307, in __call__
await self.app(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 191, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/home/peter/.local/share/uv/python/cpython-3.11.11-linux-x86_64-gnu/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_utils.py", line 85, in collapse_excgroups
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 193, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/main.py", line 597, in log_request_details
response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 168, in call_next
raise app_exc from app_exc.__cause__ or app_exc.__context__
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 144, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 191, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/home/peter/.local/share/uv/python/cpython-3.11.11-linux-x86_64-gnu/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_utils.py", line 85, in collapse_excgroups
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 193, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/main.py", line 615, in inject_propagated_observability_context
response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 168, in call_next
raise app_exc from app_exc.__cause__ or app_exc.__context__
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 144, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 191, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/home/peter/.local/share/uv/python/cpython-3.11.11-linux-x86_64-gnu/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_utils.py", line 85, in collapse_excgroups
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 193, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/main.py", line 584, in add_api_version_header
response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 168, in call_next
raise app_exc from app_exc.__cause__ or app_exc.__context__
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 144, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/routing.py", line 716, in __call__
await self.middleware_stack(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/routing.py", line 736, in app
await route.handle(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/routing.py", line 290, in handle
await self.app(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 120, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 106, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 430, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/fastapi/routing.py", line 318, in run_endpoint_function
return await run_in_threadpool(dependant.call, **values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/starlette/concurrency.py", line 32, in run_in_threadpool
return await anyio.to_thread.run_sync(func)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/anyio/to_thread.py", line 61, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 2525, in run_sync_in_worker_thread
return await future
^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 986, in run
result = context.run(func, *args)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/observability_utils/tracing/decorators.py", line 151, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/main.py", line 502, in set_state
runner.run(interface.resume_worker)
File "/dls/athena/blueapi/.venv/lib/python3.11/site-packages/observability_utils/tracing/decorators.py", line 151, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/dls/athena/blueapi/src/blueapi/service/runner.py", line 136, in run
return self._subprocess.apply(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/peter/.local/share/uv/python/cpython-3.11.11-linux-x86_64-gnu/lib/python3.11/multiprocessing/pool.py", line 360, in apply
return self.apply_async(func, args, kwds).get()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/peter/.local/share/uv/python/cpython-3.11.11-linux-x86_64-gnu/lib/python3.11/multiprocessing/pool.py", line 774, in get
raise self._value
ValueError: There is no context set for tracing despite the fact that a task is running, something has gone wrong...
After the crash, querying the state returns Idle instead of either paused or running and the interrupted task appears to be dropped.