feat: add service for query cache results (#17781)

* feat: add service for query cache results

Using Redis as a backing store for serialized results of longer,
expensive queries that can be computed periodically, and then used in a
client-facing operation where the query would take to long inline.

Signed-off-by: Mike Fiedler <[email protected]>

* remove whitespace

that'll teach me to resolve conflicts in the browser...

---------

Signed-off-by: Mike Fiedler <[email protected]>

Author: miketheman

tests/conftest.py

@@ -48,6 +48,8 @@
 from warehouse.admin.flags import AdminFlag, AdminFlagValue
 from warehouse.attestations import services as attestations_services
 from warehouse.attestations.interfaces import IIntegrityService
+from warehouse.cache import services as cache_services
+from warehouse.cache.interfaces import IQueryResultsCache
 from warehouse.email import services as email_services
 from warehouse.email.interfaces import IEmailSender
 from warehouse.helpdesk import services as helpdesk_services
@@ -165,6 +167,7 @@ def pyramid_services(
     macaroon_service,
     helpdesk_service,
     notification_service,
+    query_results_cache_service,
     search_service,
 ):
     services = _Services()
@@ -189,6 +192,7 @@ def pyramid_services(
     services.register_service(macaroon_service, IMacaroonService, None, name="")
     services.register_service(helpdesk_service, IHelpDeskService, None)
     services.register_service(notification_service, IAdminNotificationService)
+    services.register_service(query_results_cache_service, IQueryResultsCache)
     services.register_service(search_service, ISearchService)
 
     return services
@@ -316,6 +320,7 @@ def get_app_config(database, nondefaults=None):
         "database.url": database,
         "docs.url": "http://docs.example.com/",
         "ratelimit.url": "memory://",
+        "db_results_cache.url": "redis://localhost:0/",
         "opensearch.url": "https://localhost/warehouse",
         "files.backend": "warehouse.packaging.services.LocalFileStorage",
         "archive_files.backend": "warehouse.packaging.services.LocalArchiveFileStorage",
@@ -528,6 +533,11 @@ def notification_service():
     return helpdesk_services.ConsoleAdminNotificationService()
 
 
+@pytest.fixture
+def query_results_cache_service(mockredis):
+    return cache_services.RedisQueryResults(redis_client=mockredis)
+
+
 @pytest.fixture
 def search_service():
     return search_services.NullSearchService()

tests/unit/cache/test_init.py

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+
+from warehouse.cache import includeme
+from warehouse.cache.interfaces import IQueryResultsCache
+from warehouse.cache.services import RedisQueryResults
+
+
+def test_includeme():
+    config = pretend.stub(
+        register_service_factory=pretend.call_recorder(lambda *a, **k: None)
+    )
+
+    includeme(config)
+
+    assert config.register_service_factory.calls == [
+        pretend.call(RedisQueryResults.create_service, IQueryResultsCache)
+    ]

tests/unit/cache/test_services.py

@@ -0,0 +1,69 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import uuid
+
+import pretend
+
+from zope.interface.verify import verifyClass
+
+from warehouse.cache.interfaces import IQueryResultsCache
+from warehouse.cache.services import RedisQueryResults
+
+
+class TestRedisQueryResults:
+    def test_interface_matches(self):
+        assert verifyClass(IQueryResultsCache, RedisQueryResults)
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"db_results_cache.url": "redis://"})
+        )
+        # Create the service
+        service = RedisQueryResults.create_service(None, request)
+
+        assert isinstance(service, RedisQueryResults)
+
+    def test_set_get_simple(self, query_results_cache_service):
+        # Set a value in the cache
+        query_results_cache_service.set("test_key", {"foo": "bar"})
+
+        # Get the value from the cache
+        result = query_results_cache_service.get("test_key")
+
+        assert result == {"foo": "bar"}
+
+    def test_set_get_complex(self, query_results_cache_service):
+        # Construct a complex object to store in the cache
+        obj = {
+            "uuid": uuid.uuid4(),
+            "datetime": datetime.datetime.now(),
+            "list": [1, 2, 3],
+            "dict": {"key": "value"},
+        }
+        # Set the complex object in the cache
+        query_results_cache_service.set("complex_key", obj)
+
+        # Get the complex object from the cache
+        result = query_results_cache_service.get("complex_key")
+
+        # Check that the result is the "same" as the original object, except
+        # for the UUID and datetime, which are now strings
+        assert result["list"] == obj["list"]
+        assert result["dict"] == obj["dict"]
+        assert result["uuid"] == str(obj["uuid"])
+        assert result["datetime"] == obj["datetime"].isoformat()
+        assert isinstance(result["list"], list)
+        assert isinstance(result["dict"], dict)
+        assert isinstance(result["uuid"], str)
+        assert isinstance(result["datetime"], str)

tests/unit/test_config.py

@@ -426,6 +426,7 @@ def __init__(self):
             pretend.call(".sessions"),
             pretend.call(".cache.http"),
             pretend.call(".cache.origin"),
+            pretend.call(".cache"),
             pretend.call(".email"),
             pretend.call(".accounts"),
             pretend.call(".macaroons"),

warehouse/cache/__init__.py

@@ -9,3 +9,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from __future__ import annotations
+
+import typing
+
+from .interfaces import IQueryResultsCache
+from .services import RedisQueryResults
+
+if typing.TYPE_CHECKING:
+    from pyramid.config import Configurator
+
+
+def includeme(config: Configurator) -> None:
+    config.register_service_factory(
+        RedisQueryResults.create_service, IQueryResultsCache
+    )

warehouse/cache/interfaces.py

@@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zope.interface import Interface
+
+
+class IQueryResultsCache(Interface):
+    """
+    A cache for expensive/slow database query results.
+
+    Example usage:
+
+    >>> some_expensive_query = request.db.query(...)
+    >>> cache_service = request.find_service(IQueryResultsCache)
+    >>> cache_service.set("some_key_name", some_expensive_query)
+
+    # Later, retrieve the cached results:
+    >>> results = cache_service.get("some_key_name")
+    """
+
+    def create_service(context, request):
+        """Create the service, bootstrap any configuration needed."""
+
+    def get(key: str):
+        """Get a cached result by key."""
+
+    def set(key: str, value):
+        """Set a cached result by key."""
+        # TODO: do we need a set-with-expiration, a la `setex`?

warehouse/cache/services.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import typing
+
+import orjson
+import redis
+
+from zope.interface import implementer
+
+from warehouse.cache.interfaces import IQueryResultsCache
+
+if typing.TYPE_CHECKING:
+    from pyramid.request import Request
+
+
+@implementer(IQueryResultsCache)
+class RedisQueryResults:
+    """
+    A Redis-based query results cache.
+
+    Anything using this service must assume that the key results may be empty,
+    and handle the case where the key is not found in the cache.
+
+    The key is a string, and the value is a JSON-serialized object as a string.
+    """
+
+    def __init__(self, redis_client):
+        self.redis_client = redis_client
+
+    @classmethod
+    def create_service(cls, _context, request: Request) -> RedisQueryResults:
+        redis_url = request.registry.settings["db_results_cache.url"]
+        redis_client = redis.StrictRedis.from_url(redis_url)
+        return cls(redis_client)
+
+    def get(self, key: str) -> list | dict | None:
+        """Get a cached result by key."""
+        result = self.redis_client.get(key)
+        # deserialize the value as a JSON object
+        return orjson.loads(result)
+
+    def set(self, key: str, value) -> None:
+        """Set a cached result by key."""
+        # serialize the value as a JSON string
+        value = orjson.dumps(value)
+        self.redis_client.set(key, value)

warehouse/config.py

@@ -388,6 +388,7 @@ def configure(settings=None):
     maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
     maybe_set_redis(settings, "sessions.url", "REDIS_URL", db=2)
     maybe_set_redis(settings, "ratelimit.url", "REDIS_URL", db=3)
+    maybe_set_redis(settings, "db_results_cache.url", "REDIS_URL", db=5)
     maybe_set(settings, "captcha.backend", "CAPTCHA_BACKEND")
     maybe_set(settings, "recaptcha.site_key", "RECAPTCHA_SITE_KEY")
     maybe_set(settings, "recaptcha.secret_key", "RECAPTCHA_SECRET_KEY")
@@ -810,6 +811,8 @@ def configure(settings=None):
     # Register our support for http and origin caching
     config.include(".cache.http")
     config.include(".cache.origin")
+    # Register our support for the database results cache
+    config.include(".cache")
 
     # Register support for sending emails
     config.include(".email")