Clean removal of prometheus db files on shutdown (#2308)

* Cleanly remove metrics db files created by prometheus

Until now, multiple workers were competing on deleting the files, and some were
inevitably failing with "FileNotFound" errors.

* Fix deprecated poetry usage in runtimes tox.ini

Author: lc525

mlserver/metrics/prometheus.py

@@ -31,14 +31,22 @@ def configure_metrics(settings: Settings):
     values.ValueClass = values.get_value_class()
 
 
+async def try_remove(filename: str):
+    try:
+        await remove(filename)
+    except FileNotFoundError:
+        pass
+
+
 async def stop_metrics(settings: Settings, pid: int):
     if not settings.parallel_workers:
         return
 
     mark_process_dead(pid)
     pattern = os.path.join(settings.metrics_dir, f"*_{pid}.db")
     matching_files = glob.glob(pattern)
-    await asyncio.gather(*[remove(f) for f in matching_files])
+    if len(matching_files) > 0:
+        await asyncio.gather(*[try_remove(f) for f in matching_files])
 
 
 class PrometheusEndpoint:

runtimes/alibi-detect/tox.ini

@@ -6,7 +6,7 @@ setenv =
     TF_USE_LEGACY_KERAS = 1
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/alibi-explain/tox.ini

@@ -6,7 +6,7 @@ setenv =
     TF_USE_LEGACY_KERAS = 1
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/catboost/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/huggingface/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/lightgbm/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/mlflow/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/mllib/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/sklearn/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \

runtimes/xgboost/tox.ini

@@ -4,7 +4,7 @@ isolated_build = true
 [testenv]
 allowlist_externals = poetry
 commands_pre =
-    poetry install --sync --no-root
+    poetry sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
     python -m pytest {posargs} -n auto \