reproducing the problem

Author: sanderegg

packages/service-library/src/servicelib/redis.py

@@ -49,6 +49,10 @@ class CouldNotConnectToRedisError(BaseRedisError):
     msg_template: str = "Connection to '{dsn}' failed"
 
 
+class LockLostError(BaseRedisError):
+    msg_template: str = "Lock {lock.name} has been lost"
+
+
 async def _cancel_or_warn(task: Task) -> None:
     if not task.cancelled():
         task.cancel()
@@ -179,13 +183,16 @@ async def lock_context(
             raise CouldNotAcquireLockError(lock=ttl_lock)
 
         async def _extend_lock(lock: Lock) -> None:
-            with (
-                log_context(_logger, logging.DEBUG, f"Extending lock {lock_unique_id}"),
-                log_catch(_logger, reraise=False),
-            ):
-                # TODO: if we cannot re-acquire that means the lock is lost, and we are not anymore safe and should raise all the way to the caller
-
-                await lock.reacquire()
+            try:
+                with (
+                    log_context(
+                        _logger, logging.DEBUG, f"Extending lock {lock_unique_id}"
+                    ),
+                ):
+                    # TODO: if we cannot re-acquire that means the lock is lost, and we are not anymore safe and should raise all the way to the caller
+                    await lock.reacquire()
+            except redis.exceptions.LockNotOwnedError as exc:
+                raise LockLostError(lock=lock) from exc
 
         try:
             async with periodic_task(

packages/service-library/tests/test_redis.py

@@ -17,6 +17,7 @@
 from servicelib import redis as servicelib_redis
 from servicelib.redis import (
     CouldNotAcquireLockError,
+    LockLostError,
     RedisClientSDK,
     RedisClientsManager,
     RedisManagerDBConfig,
@@ -35,7 +36,7 @@
 ]
 
 pytest_simcore_ops_services_selection = [
-    # "redis-commander",
+    "redis-commander",
 ]
 
 
@@ -48,7 +49,7 @@ async def _is_locked(redis_client_sdk: RedisClientSDK, lock_name: str) -> bool:
 async def redis_client_sdk(
     get_redis_client_sdk: Callable[
         [RedisDatabase], AbstractAsyncContextManager[RedisClientSDK]
-    ]
+    ],
 ) -> AsyncIterator[RedisClientSDK]:
     async with get_redis_client_sdk(RedisDatabase.RESOURCES) as client:
         yield client
@@ -165,6 +166,23 @@ async def test_lock_context(
     assert await ttl_lock.owned() is False
 
 
+async def test_issue_cannot_acquire_lock_anymore(
+    redis_client_sdk: RedisClientSDK, faker: Faker
+):
+    lock_name = faker.pystr()
+    with pytest.raises(LockLostError):
+        async with redis_client_sdk.lock_context(lock_name) as ttl_lock:
+            assert await _is_locked(redis_client_sdk, lock_name) is True
+            assert await ttl_lock.owned() is True
+            # let's force release the lock
+            await redis_client_sdk._client.delete(lock_name)
+
+            await asyncio.sleep(20)
+            # assert await _is_locked(redis_client_sdk, lock_name) is False
+            # assert await ttl_lock.owned() is False
+            # actually it should even raise here
+
+
 async def test_lock_context_with_already_locked_lock_raises(
     redis_client_sdk: RedisClientSDK, faker: Faker
 ):
@@ -272,7 +290,6 @@ async def _inc_counter() -> None:
 async def test_redis_client_sdks_manager(
     mock_redis_socket_timeout: None, redis_service: RedisSettings
 ):
-
     all_redis_configs: set[RedisManagerDBConfig] = {
         RedisManagerDBConfig(db) for db in RedisDatabase
     }