Merge pull request #310 from VariantEffect/maintenance/bencap/274/deps

Refresh Dependencies for High Severity Dependabot Alerts

Author: bencap

.github/workflows/run-tests-on-push.yml

@@ -18,7 +18,7 @@ jobs:
     - run: pip install --upgrade pip
     - run: pip install poetry
     - run: poetry install --with dev --extras server
-    - run: poetry run pytest tests/ --show-capture=stdout
+    - run: poetry run pytest tests/ --show-capture=stdout --cov=src
 
   run-tests-3_10:
     runs-on: ubuntu-latest
@@ -32,7 +32,7 @@ jobs:
     - run: pip install --upgrade pip
     - run: pip install poetry
     - run: poetry install --with dev --extras server
-    - run: poetry run pytest tests/ --show-capture=stdout
+    - run: poetry run pytest tests/ --show-capture=stdout --cov=src
 
   run-tests-3_11:
     runs-on: ubuntu-latest
@@ -46,7 +46,7 @@ jobs:
     - run: pip install --upgrade pip
     - run: pip install poetry
     - run: poetry install --with dev --extras server
-    - run: poetry run pytest tests/ --show-capture=stdout
+    - run: poetry run pytest tests/ --show-capture=stdout --cov=src
 
   run-mypy-3_10:
     runs-on: ubuntu-latest

poetry.lock

@@ -26,35 +26,36 @@ python = "^3.9"
 
 fqfa = "~1.3.0"
 pyhumps = "~3.8.0"
+pyyaml = "~5.1"
 IDUtils = "~1.2.0"
 mavehgvs = "~0.6.0"
 eutils = "~0.6.0"
-hgvs = "~1.5.4"
-biocommons = "~0.0.0"
-cdot = "~0.2.21"
+email_validator = "~2.1.1"
 numpy = "~1.26"
 httpx = "~0.26.0"
 pandas = "~1.4.1"
 pydantic = "~1.10"
 python-dotenv = "~0.20.0"
 python-json-logger = "~2.0.7"
 SQLAlchemy = "~2.0.0"
-starlette = "~0.27.0"
-starlette-context = "^0.3.6"
 
 # Optional dependencies for running this application as a server
 alembic = { version = "~1.7.6", optional = true }
 arq = { version = "~0.25.0", optional = true }
-authlib = { version = "~0.15.5", optional = true }
+authlib = { version = "~1.3.1", optional = true }
 boto3 = { version = "~1.34.97", optional = true }
-cryptography = { version = "~41.0.6", optional = true }
-email-validator = { version = "~2.1.1", optional = true }
+biocommons = { version = "~0.0.0", optional = true }
+cryptography = { version = "~43.0.1", optional = true }
+cdot = { version = "~0.2.21", optional = true }
 fastapi = { version = "~0.95.0", optional = true }
+hgvs = { version = "~1.5.4", optional = true }
 orcid = { version = "~1.0.3", optional = true }
 psycopg2 = { version = "~2.9.3", optional = true }
 python-jose = { extras = ["cryptography"], version = "~3.3.0", optional = true }
 python-multipart = { version = "~0.0.5", optional = true }
-requests = { version = "~2.31.0", optional = true }
+requests = { version = "~2.32.0", optional = true }
+starlette = { version = "~0.27.0", optional = true }
+starlette-context = { version = "^0.3.6", optional = true }
 slack-sdk = { version = "~3.21.3", optional = true }
 uvicorn = { extras = ["standard"], version = "*", optional = true }
 watchtower = { version = "~3.2.0", optional = true }
@@ -70,7 +71,8 @@ mypy = "~1.10.0"
 pre-commit = "*"
 jsonschema = "*"
 fakeredis = "~2.21.1"
-pytest = "~7.0.1"
+pytest = "~7.2.0"
+pytest-cov = "~5.0.0"
 pytest-postgresql = "~5.0.0"
 pytest-asyncio = "~0.23.5"
 pytest-socket = "~0.6.0"
@@ -84,7 +86,7 @@ SQLAlchemy = { extras = ["mypy"], version = "~2.0.0" }
 
 
 [tool.poetry.extras]
-server = ["alembic", "arq", "authlib", "boto3", "cryptography", "fastapi", "email-validator", "orcid", "psycopg2", "python-jose", "python-multipart", "requests", "slack-sdk", "uvicorn", "watchtower"]
+server = ["alembic", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "watchtower"]
 
 
 [tool.black]

pyproject.toml

@@ -3,12 +3,18 @@
 import os
 import sys
 
-from watchtower import CloudWatchLogHandler
-
 from .config import load_stock_config
 from .filters import canonical_only
 from .formatters import MavedbJsonFormatter
 
+WATCHTOWER_IMPORTED = False
+try:
+    from watchtower import CloudWatchLogHandler
+
+    WATCHTOWER_IMPORTED = True
+except ModuleNotFoundError:
+    pass
+
 LOG_CONFIG = os.environ.get("LOG_CONFIG")
 
 
@@ -35,12 +41,14 @@ def configure():
     # Formatter and handler are un-configurable via file config.
     cw_is_enabled = False
     root_logger = logging.getLogger("root")
-    for handler in root_logger.handlers:
-        if isinstance(handler, CloudWatchLogHandler):
-            handler.addFilter(canonical_only)
-            handler.formatter = MavedbJsonFormatter()
 
-            cw_is_enabled = True
+    if WATCHTOWER_IMPORTED:
+        for handler in root_logger.handlers:
+            if isinstance(handler, CloudWatchLogHandler):
+                handler.addFilter(canonical_only)
+                handler.formatter = MavedbJsonFormatter()
+
+                cw_is_enabled = True
 
     if not cw_is_enabled:
         root_logger.info("CloudWatch log handler is not enabled. Canonical logs will only be emitted to stdout.")

src/mavedb/logging/__init__.py

@@ -9,7 +9,6 @@ root:
   propagate: true
   handlers:
     - console
-    - watchtower
 
 loggers:
   urllib3:
@@ -18,10 +17,6 @@ loggers:
   botocore:
     level: INFO
 
-  # HGVS emits a log line on startup, silence it.
-  hgvs:
-    level: WARNING
-
   # To log all database queries, set this to INFO
   sqlalchemy:
     level: WARNING
@@ -35,19 +30,7 @@ handlers:
     stream: ext://sys.stdout
     formatter: json
 
-  watchtower:
-    class: watchtower.CloudWatchLogHandler
-    level: !coalesce
-      - !LOG_LEVEL
-      - INFO
-    log_group_name: !CLOUD_WATCH_LOG_GROUP
-    log_stream_name: "{machine_name}/{logger_name}/{strftime:%y-%m-%d}"
-    send_interval: 10
-    use_queues: !CLOUD_WATCH_USE_QUEUES
-    create_log_group: True
-    formatter: json
-
 formatters:
   json:
-    class: mavedb.logging.MavedbJsonFormatter
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
     format: "%(message)s"

src/mavedb/logging/configurations/default.yaml

@@ -0,0 +1,53 @@
+version: 1
+disable_existing_loggers: False
+
+root:
+  # Filtering of messages by level is done at the handler level by using NOTSET
+  # on the root logger to emit everything.  This lets us keep console output
+  # readable while emitting verbose output to alternate handlers.
+  level: NOTSET
+  propagate: true
+  handlers:
+    - console
+    - watchtower
+
+loggers:
+  urllib3:
+    level: INFO
+
+  botocore:
+    level: INFO
+
+  # HGVS emits a log line on startup, silence it.
+  hgvs:
+    level: WARNING
+
+  # To log all database queries, set this to INFO
+  sqlalchemy:
+    level: WARNING
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: !coalesce
+      - !LOG_LEVEL
+      - INFO
+    stream: ext://sys.stdout
+    formatter: json
+
+  watchtower:
+    class: watchtower.CloudWatchLogHandler
+    level: !coalesce
+      - !LOG_LEVEL
+      - INFO
+    log_group_name: !CLOUD_WATCH_LOG_GROUP
+    log_stream_name: "{machine_name}/{logger_name}/{strftime:%y-%m-%d}"
+    send_interval: 10
+    use_queues: !CLOUD_WATCH_USE_QUEUES
+    create_log_group: True
+    formatter: json
+
+formatters:
+  json:
+    class: mavedb.logging.MavedbJsonFormatter
+    format: "%(message)s"

src/mavedb/logging/configurations/server.yaml

@@ -0,0 +1,2 @@
+# Import view models up front so any deferred forward references are resolved prior to OpenAPI spec generation.
+import mavedb.view_models

src/mavedb/routers/__init__.py

@@ -22,7 +22,6 @@
     DoiIdentifierCreate,
     SavedDoiIdentifier,
 )
-from mavedb.view_models.experiment import Experiment, SavedExperiment
 from mavedb.view_models.license import ShortLicense
 from mavedb.view_models.publication_identifier import (
     PublicationIdentifier,
@@ -136,7 +135,13 @@ def targets_need_labels_when_multiple_targets_exist(cls, field_value, values):
                 if target.target_sequence and target.target_sequence.label is None:
                     raise ValidationError(
                         "Target sequence labels cannot be empty when multiple targets are defined.",
-                        custom_loc=["body", "targetGene", idx, "targetSequence", "label"],
+                        custom_loc=[
+                            "body",
+                            "targetGene",
+                            idx,
+                            "targetSequence",
+                            "label",
+                        ],
                     )
 
         return field_value
@@ -155,7 +160,13 @@ def target_labels_are_unique(cls, field_value, values):
                 # just one for now seems fine.
                 raise ValidationError(
                     "Target sequence labels cannot be duplicated.",
-                    custom_loc=["body", "targetGene", dup_indices[-1], "targetSequence", "label"],
+                    custom_loc=[
+                        "body",
+                        "targetGene",
+                        dup_indices[-1],
+                        "targetSequence",
+                        "label",
+                    ],
                 )
 
         return field_value
@@ -312,7 +323,7 @@ class ShortScoreSet(BaseModel):
     published_date: Optional[date]
     replaces_id: Optional[int]
     num_variants: int
-    experiment: Experiment
+    experiment: "Experiment"
     primary_publication_identifiers: list[SavedPublicationIdentifier]
     secondary_publication_identifiers: list[SavedPublicationIdentifier]
     license: ShortLicense
@@ -380,7 +391,7 @@ def camelize_dataset_columns_keys(cls, value) -> dict:
 class ScoreSet(SavedScoreSet):
     """Score set view model containing most properties visible to non-admin users, but no variant data."""
 
-    experiment: Experiment
+    experiment: "Experiment"
     doi_identifiers: Sequence[DoiIdentifier]
     primary_publication_identifiers: Sequence[PublicationIdentifier]
     secondary_publication_identifiers: Sequence[PublicationIdentifier]
@@ -424,3 +435,9 @@ class ScoreSetPublicDump(SavedScoreSet):
     processing_errors: Optional[Dict]
     mapping_state: Optional[MappingState]
     mapping_errors: Optional[Dict]
+
+
+from mavedb.view_models.experiment import Experiment
+
+ShortScoreSet.update_forward_refs()
+ScoreSet.update_forward_refs()