fix: query args validation of /user/secrets GET (#343)

Author: olevski

components/renku_data_services/base_api/error_handler.py

@@ -1,5 +1,8 @@
 """The error handler for the application."""
 
+import os
+import sys
+import traceback
 from collections.abc import Mapping, Set
 from sqlite3 import Error as SqliteError
 from typing import Any, Optional, Protocol, TypeVar, Union
@@ -127,6 +130,12 @@ def default(self, request: Request, exception: Exception) -> HTTPResponse:
                     message="The provided input is too large to be stored in the database"
                 )
         self.log(request, formatted_exception)
+        if formatted_exception.status_code == 500 and "PYTEST_CURRENT_TEST" in os.environ:
+            # TODO: Figure out how to do logging properly in here, I could not get the sanic logs to show up from here
+            # at all when running schemathesis. So 500 errors are hard to debug but print statements do show up.
+            # The above log statement does not show up in the logs that pytest shows after a test is done.
+            sys.stderr.write(f"A 500 error was raised because of {type(exception)} on request {request}\n")
+            traceback.print_exception(exception)
         return json(
             self.api_spec.ErrorResponse(
                 error=self.api_spec.Error(

components/renku_data_services/base_api/misc.py

@@ -3,10 +3,12 @@
 from collections.abc import Awaitable, Callable, Coroutine
 from dataclasses import dataclass
 from functools import wraps
-from typing import Any, NoReturn, ParamSpec, TypeVar, cast
+from typing import Any, Concatenate, NoReturn, ParamSpec, TypeVar, cast
 
+from pydantic import BaseModel
 from sanic import Request, json
 from sanic.response import JSONResponse
+from sanic_ext import validate
 
 from renku_data_services import errors
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
@@ -69,3 +71,29 @@ async def decorated_function(*args: _P.args, **kwargs: _P.kwargs) -> _T:
         return response
 
     return decorated_function
+
+
+def validate_query(
+    query: type[BaseModel],
+) -> Callable[
+    [Callable[Concatenate[Request, _P], Awaitable[_T]]],
+    Callable[Concatenate[Request, _P], Coroutine[Any, Any, _T]],
+]:
+    """Decorator for sanic query parameter validation.
+
+    Should be removed once sanic fixes this error in their validation code.
+    """
+
+    def decorator(
+        f: Callable[Concatenate[Request, _P], Awaitable[_T]],
+    ) -> Callable[Concatenate[Request, _P], Coroutine[Any, Any, _T]]:
+        @wraps(f)
+        async def decorated_function(request: Request, *args: _P.args, **kwargs: _P.kwargs) -> _T:
+            try:
+                return await validate(query=query)(f)(request, *args, **kwargs)
+            except KeyError:
+                raise errors.ValidationError(message="Failed to validate the query parameters")
+
+        return decorated_function
+
+    return decorator

components/renku_data_services/migrations/alembic.ini

@@ -75,38 +75,3 @@ version_path_separator = os
 # black.type = console_scripts
 # black.entrypoint = black
 # black.options = -l 79 REVISION_SCRIPT_FILENAME
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = INFO
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S

components/renku_data_services/migrations/env.py

@@ -1,15 +1,12 @@
 """Database migrations for Alembic."""
 
-from logging.config import fileConfig
-
-from alembic import context
-from alembic.config import Config
+from logging.config import dictConfig
 
 from renku_data_services.authz.orm import BaseORM as authz
 from renku_data_services.connected_services.orm import BaseORM as connected_services
 from renku_data_services.crc.orm import BaseORM as crc
 from renku_data_services.message_queue.orm import BaseORM as events
-from renku_data_services.migrations.utils import run_migrations
+from renku_data_services.migrations.utils import logging_config, run_migrations
 from renku_data_services.namespace.orm import BaseORM as namespaces
 from renku_data_services.platform.orm import BaseORM as platform
 from renku_data_services.project.orm import BaseORM as project
@@ -19,14 +16,9 @@
 from renku_data_services.user_preferences.orm import BaseORM as user_preferences
 from renku_data_services.users.orm import BaseORM as users
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config: Config = context.config
-
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
+dictConfig(logging_config)
 
 all_metadata = [
     authz.metadata,

components/renku_data_services/migrations/utils.py

@@ -191,3 +191,19 @@ def __del__(self) -> None:
         self._thread.join()
         del self._loop
         del self._thread
+
+
+logging_config: dict = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "loggers": {
+        "alembic": {
+            "level": "INFO",
+            "qualname": "alembic",
+        },
+        "sqlalchemy": {
+            "level": "WARN",
+            "qualname": "sqlalchemy.engine",
+        },
+    },
+}

components/renku_data_services/users/blueprints.py

@@ -10,6 +10,7 @@
 import renku_data_services.base_models as base_models
 from renku_data_services.base_api.auth import authenticate, only_authenticated
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
+from renku_data_services.base_api.misc import validate_query
 from renku_data_services.base_models.validation import validated_json
 from renku_data_services.errors import errors
 from renku_data_services.secrets.db import UserSecretsRepo
@@ -160,7 +161,7 @@ def get_all(self) -> BlueprintFactoryResponse:
 
         @authenticate(self.authenticator)
         @only_authenticated
-        @validate(query=GetSecretsParams)
+        @validate_query(query=GetSecretsParams)
         async def _get_all(request: Request, user: base_models.APIUser, query: GetSecretsParams) -> JSONResponse:
             secret_kind = SecretKind[query.kind.value]
             secrets = await self.secret_repo.get_user_secrets(requested_by=user, kind=secret_kind)

test/bases/renku_data_services/data_api/test_schemathesis.py

@@ -21,7 +21,7 @@ def requests_statistics():
     stats = sorted(stats)
     p95 = stats[math.floor(0.95 * len(stats))]
 
-    assert p95 < timedelta(milliseconds=100)
+    assert p95 < timedelta(milliseconds=100), f"The p95 response time {p95} was >= 100 ms"
 
 
 @pytest_asyncio.fixture