Skip to content

Occasional pytest_internalerror() was called on the worker error #153

@rouge8

Description

@rouge8

We're seeing our test suite occasionally fail to start with pytest_internalerror() was called on the worker:

gw0 C / gw1 C / gw2 C / gw3 C / gw4 C / gw5 C / gw6 C / gw7 CINTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/_pytest/main.py", line 264, in wrap_session
INTERNALERROR>     config._do_configure()
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/_pytest/config/__init__.py", line 995, in _do_configure
INTERNALERROR>     self.hook.pytest_configure.call_historic(kwargs=dict(config=self))
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_hooks.py", line 277, in call_historic
INTERNALERROR>     res = self._hookexec(self.name, self.get_hookimpls(), kwargs, False)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_manager.py", line 80, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_callers.py", line 60, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_result.py", line 60, in get_result
INTERNALERROR>     raise ex[1].with_traceback(ex[2])
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_callers.py", line 39, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 482, in pytest_configure
INTERNALERROR>     reactor_installers[config.getoption("reactor")]()
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 409, in init_default_reactor
INTERNALERROR>     _install_reactor(
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 458, in _install_reactor
INTERNALERROR>     init_twisted_greenlet()
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 204, in init_twisted_greenlet
INTERNALERROR>     signal.signal(
INTERNALERROR>   File "/home/circleci/.pyenv/versions/3.10.6/lib/python3.10/signal.py", line 56, in signal
INTERNALERROR>     handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
INTERNALERROR> ValueError: signal only works in main thread of the main interpreter
INTERNALERROR> def worker_internal_error(self, node, formatted_error):
INTERNALERROR>         """
INTERNALERROR>         pytest_internalerror() was called on the worker.
INTERNALERROR>     
INTERNALERROR>         pytest_internalerror() arguments are an excinfo and an excrepr, which can't
INTERNALERROR>         be serialized, so we go with a poor man's solution of raising an exception
INTERNALERROR>         here ourselves using the formatted message.
INTERNALERROR>         """
INTERNALERROR>         self._active_nodes.remove(node)
INTERNALERROR>         try:
INTERNALERROR> >           assert False, formatted_error
INTERNALERROR> E           AssertionError: Traceback (most recent call last):
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/_pytest/main.py", line 264, in wrap_session
INTERNALERROR> E                 config._do_configure()
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/_pytest/config/__init__.py", line 995, in _do_configure
INTERNALERROR> E                 self.hook.pytest_configure.call_historic(kwargs=dict(config=self))
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_hooks.py", line 277, in call_historic
INTERNALERROR> E                 res = self._hookexec(self.name, self.get_hookimpls(), kwargs, False)
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_manager.py", line 80, in _hookexec
INTERNALERROR> E                 return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_callers.py", line 60, in _multicall
INTERNALERROR> E                 return outcome.get_result()
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_result.py", line 60, in get_result
INTERNALERROR> E                 raise ex[1].with_traceback(ex[2])
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_callers.py", line 39, in _multicall
INTERNALERROR> E                 res = hook_impl.function(*args)
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 482, in pytest_configure
INTERNALERROR> E                 reactor_installers[config.getoption("reactor")]()
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 409, in init_default_reactor
INTERNALERROR> E                 _install_reactor(
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 458, in _install_reactor
INTERNALERROR> E                 init_twisted_greenlet()
INTERNALERROR> E               File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pytest_twisted.py", line 204, in init_twisted_greenlet
INTERNALERROR> E                 signal.signal(
INTERNALERROR> E               File "/home/circleci/.pyenv/versions/3.10.6/lib/python3.10/signal.py", line 56, in signal
INTERNALERROR> E                 handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
INTERNALERROR> E             ValueError: signal only works in main thread of the main interpreter
INTERNALERROR> E           assert False
INTERNALERROR> 
INTERNALERROR> .tox/unit-tests/lib/python3.10/site-packages/xdist/dsession.py:192: AssertionError
[gw6] node down: Not properly terminated

replacing crashed worker gw6
[gw8] linux Python 3.10.6 cwd: /home/circleci/project/zapgram
gw0 C / gw1 C / gw2 C / gw3 C / gw4 C / gw5 C / gw8 C / gw7 CINTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/_pytest/main.py", line 268, in wrap_session
INTERNALERROR>     session.exitstatus = doit(config, session) or 0
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/_pytest/main.py", line 322, in _main
INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_hooks.py", line 265, in __call__
INTERNALERROR>     return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_manager.py", line 80, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_callers.py", line 60, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_result.py", line 60, in get_result
INTERNALERROR>     raise ex[1].with_traceback(ex[2])
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/pluggy/_callers.py", line 39, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/xdist/dsession.py", line 117, in pytest_runtestloop
INTERNALERROR>     self.loop_once()
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/xdist/dsession.py", line 140, in loop_once
INTERNALERROR>     call(**kwargs)
INTERNALERROR>   File "/home/circleci/project/zapgram/.tox/unit-tests/lib/python3.10/site-packages/xdist/dsession.py", line 229, in worker_errordown
INTERNALERROR>     self._active_nodes.remove(node)
INTERNALERROR> KeyError: <WorkerController gw6>

I don't have a good reproducer and it's intermittent.

Dependencies:

pytest==7.1.2
pytest-flakefinder==1.0.0
pytest-forked==1.4.0
pytest-mock==3.8.2
pytest-randomly==3.12.0
pytest-twisted==1.13.4
pytest-unordered==0.5.1
pytest-xdist==2.5.0

Running Python 3.10.6 I believe.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions