Fix race condition (#946)

Naveed will recognize this: we were trying to help Bijan, and this was a red herring along the way. 

The problem sequence of events is this:
- We create a new `_ModuleHandler` (line 67), which constructs the `_SingletonEventLoopThread` (line 73). 
- Creating that singleton sets `self._loop` to None (old line 36), then spawns a separate thread (line 39) which would properly initialize `self._loop` (old line 42).
- Before the separate thread can run, though, we call `module_handler.emit()` (line 85), which calls `self._worker.get_loop()` (line 94), which throws an exception because the loop isn't initialized yet, because the second thread hasn't had a chance to run yet.

With the changes in the first commit, the loop is initialized immediately and merely run in the other thread. That way, other things (like `emit()`) can enqueue new coroutines in the loop before it starts running, and they'll execute shortly thereafter when the other thread gets going.

While I was at this, I noticed that we were using [a non-thread-safe](https://docs.python.org/3/library/asyncio-sync.html#event) `asyncio.Event` to signal between multiple threads, so I changed it to use a `threading.Event` instead. That's in a separate commit, in case that turns out to be a bad idea (I still don't have enough experience with asyncio to feel confident about that stuff).

Author: penguinland

src/viam/logging.py

@@ -4,7 +4,7 @@
 from copy import copy
 from datetime import datetime
 from logging import DEBUG, ERROR, FATAL, INFO, WARN, WARNING  # noqa: F401
-from threading import Lock, Thread
+from threading import Event, Lock, Thread
 from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union
 
 from grpclib.exceptions import StreamTerminatedError
@@ -23,8 +23,10 @@
 class _SingletonEventLoopThread:
     _instance = None
     _lock = Lock()
-    _ready_event = asyncio.Event()
-    _loop: Union[asyncio.AbstractEventLoop, None]
+    # We use a threading.Event instead of an asyncio.Event because the latter are not thread safe,
+    # and this is set in a separate thread than it is waited on.
+    _ready_event = Event()
+    _loop: asyncio.AbstractEventLoop
     _thread: Thread
 
     def __new__(cls):
@@ -33,13 +35,12 @@ def __new__(cls):
             with cls._lock:
                 if cls._instance is None:
                     cls._instance = super(_SingletonEventLoopThread, cls).__new__(cls)
-                    cls._instance._loop = None
+                    cls._instance._loop = asyncio.new_event_loop()
                     cls._instance._thread = Thread(target=cls._instance._run)
                     cls._instance._thread.start()
         return cls._instance
 
     def _run(self):
-        self._loop = asyncio.new_event_loop()
         asyncio.set_event_loop(self._loop)
         self._ready_event.set()
         self._loop.run_forever()
@@ -54,8 +55,8 @@ def get_loop(self):
             raise RuntimeError("Event loop is None. Did you call .start() and .wait_until_ready()?")
         return self._loop
 
-    async def wait_until_ready(self):
-        await self._ready_event.wait()
+    def wait_until_ready(self):
+        self._ready_event.wait()
 
 
 class _ModuleHandler(logging.Handler):
@@ -101,7 +102,7 @@ def emit(self, record: logging.LogRecord):
             self._logger.log(record.levelno, message)
 
     async def _asynchronously_emit(self, record: logging.LogRecord, name: str, message: str, stack: str, time: datetime):
-        await self._worker.wait_until_ready()
+        self._worker.wait_until_ready()
         task = self._worker.get_loop().create_task(
             self._parent.log(name, record.levelname, time, message, stack),
             name=f"{viam._TASK_PREFIX}-LOG-{record.created}",