Support serializable panels. This is a WIP and needs clean-up.

tim-schilling · tim-schilling · commit bbbbb34bb205 · 2023-08-20T21:32:08.000-05:00
The remainder of the work is to fix the individual panels' serialization errors.
diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py
@@ -12,6 +12,7 @@ class Panel:
     def __init__(self, toolbar, get_response):
         self.toolbar = toolbar
         self.get_response = get_response
+        self.from_store = False
 
     # Private panel properties
 
@@ -21,6 +22,12 @@ def panel_id(self):
 
     @property
     def enabled(self) -> bool:
+        if self.from_store:
+            # If the toolbar was loaded from the store the existence of
+            # recorded data indicates whether it was enabled or not.
+            # We can't use the remainder of the logic since we don't have
+            # a request to work off of.
+            return bool(self.get_stats())
         # The user's cookies should override the default value
         cookie_value = self.toolbar.request.COOKIES.get("djdt" + self.panel_id)
         if cookie_value is not None:
@@ -168,6 +175,9 @@ def record_stats(self, stats):
         Each call to ``record_stats`` updates the statistics dictionary.
         """
         self.toolbar.stats.setdefault(self.panel_id, {}).update(stats)
+        self.toolbar.store.save_panel(
+            self.toolbar.request_id, self.panel_id, self.toolbar.stats[self.panel_id]
+        )
 
     def get_stats(self):
         """
@@ -251,6 +261,15 @@ def generate_server_timing(self, request, response):
         Does not return a value.
         """
 
+    def load_stats_from_store(self, data):
+        """
+        Instantiate the panel from serialized data.
+
+        Return the panel instance.
+        """
+        self.toolbar.stats.setdefault(self.panel_id, {}).update(data)
+        self.from_store = True
+
     @classmethod
     def run_checks(cls):
         """
diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py
@@ -86,20 +86,19 @@ def content(self):
 
         Fetch every store for the toolbar and include it in the template.
         """
-        stores = {}
-        for id, toolbar in reversed(self.toolbar._store.items()):
-            stores[id] = {
-                "toolbar": toolbar,
+        toolbar_history = {}
+        for request_id in reversed(self.toolbar.store.request_ids()):
+            toolbar_history[request_id] = {
                 "form": HistoryStoreForm(
-                    initial={"request_id": id, "exclude_history": True}
+                    initial={"request_id": request_id, "exclude_history": True}
                 ),
             }
 
         return render_to_string(
             self.template,
             {
                 "current_request_id": self.toolbar.request_id,
-                "stores": stores,
+                "toolbar_history": toolbar_history,
                 "refresh_form": HistoryStoreForm(
                     initial={
                         "request_id": self.toolbar.request_id,
diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py
@@ -3,6 +3,7 @@
 
 from debug_toolbar.decorators import render_with_toolbar_language, require_show_toolbar
 from debug_toolbar.panels.history.forms import HistoryStoreForm
+from debug_toolbar.store import get_store
 from debug_toolbar.toolbar import DebugToolbar
 
 
@@ -46,19 +47,20 @@ def history_refresh(request):
     if form.is_valid():
         requests = []
         # Convert to list to handle mutations happening in parallel
-        for id, toolbar in list(DebugToolbar._store.items()):
+        for request_id in get_store().request_ids():
+            toolbar = DebugToolbar.fetch(request_id)
             requests.append(
                 {
-                    "id": id,
+                    "id": request_id,
                     "content": render_to_string(
                         "debug_toolbar/panels/history_tr.html",
                         {
-                            "id": id,
+                            "id": request_id,
                             "store_context": {
                                 "toolbar": toolbar,
                                 "form": HistoryStoreForm(
                                     initial={
-                                        "request_id": id,
+                                        "request_id": request_id,
                                         "exclude_history": True,
                                     }
                                 ),
diff --git a/debug_toolbar/panels/settings.py b/debug_toolbar/panels/settings.py
@@ -1,4 +1,5 @@
 from django.conf import settings
+from django.utils.encoding import force_str
 from django.utils.translation import gettext_lazy as _
 from django.views.debug import get_default_exception_reporter_filter
 
@@ -20,4 +21,11 @@ def title(self):
         return _("Settings from %s") % settings.SETTINGS_MODULE
 
     def generate_stats(self, request, response):
-        self.record_stats({"settings": dict(sorted(get_safe_settings().items()))})
+        self.record_stats(
+            {
+                "settings": {
+                    key: force_str(value)
+                    for key, value in sorted(get_safe_settings().items())
+                }
+            }
+        )
diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py
@@ -9,6 +9,7 @@
 from django.test.signals import template_rendered
 from django.test.utils import instrumented_test_render
 from django.urls import path
+from django.utils.encoding import force_str
 from django.utils.translation import gettext_lazy as _
 
 from debug_toolbar.panels import Panel
@@ -179,7 +180,7 @@ def generate_stats(self, request, response):
             else:
                 template.origin_name = _("No origin")
                 template.origin_hash = ""
-            info["template"] = template
+            info["template"] = force_str(template)
             # Clean up context for better readability
             if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]:
                 context_list = template_data.get("context", [])
@@ -188,7 +189,14 @@ def generate_stats(self, request, response):
 
         # Fetch context_processors/template_dirs from any template
         if self.templates:
-            context_processors = self.templates[0]["context_processors"]
+            context_processors = (
+                {
+                    key: force_str(value)
+                    for key, value in self.templates[0]["context_processors"].items()
+                }
+                if self.templates[0]["context_processors"]
+                else None
+            )
             template = self.templates[0]["template"]
             # django templates have the 'engine' attribute, while jinja
             # templates use 'backend'
diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py
@@ -126,5 +126,5 @@ def panel(cls, request_id: str, panel_id: str) -> Any:
             return deserialize(data)
 
 
-def get_store():
+def get_store() -> BaseStore:
     return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"])
diff --git a/debug_toolbar/templates/debug_toolbar/panels/history.html b/debug_toolbar/templates/debug_toolbar/panels/history.html
@@ -15,7 +15,7 @@
         </tr>
     </thead>
     <tbody id="djdtHistoryRequests">
-        {% for id, store_context in stores.items %}
+        {% for request_id, store_context in toolbar_history.items %}
             {% include "debug_toolbar/panels/history_tr.html" %}
         {% endfor %}
     </tbody>
diff --git a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html
@@ -1,5 +1,5 @@
 {% load i18n %}
-<tr class="{% if id == current_request_id %}djdt-highlighted{% endif %}" id="historyMain_{{ id }}" data-request-id="{{ id }}">
+<tr class="{% if request_id == current_request_id %}djdt-highlighted{% endif %}" id="historyMain_{{ request_id }}" data-request-id="{{ request_id }}">
     <td>
         {{ store_context.toolbar.stats.HistoryPanel.time|escape }}
     </td>
@@ -10,8 +10,8 @@
         <p>{{ store_context.toolbar.stats.HistoryPanel.request_url|truncatechars:100|escape }}</p>
     </td>
     <td>
-        <button type="button" class="djToggleSwitch" data-toggle-name="historyMain" data-toggle-id="{{ id }}">+</button>
-        <div class="djUnselected djToggleDetails_{{ id }}">
+        <button type="button" class="djToggleSwitch" data-toggle-name="historyMain" data-toggle-id="{{ request_id }}">+</button>
+        <div class="djUnselected djToggleDetails_{{ request_id }}">
           <table>
             <colgroup>
               <col class="djdt-width-20">
@@ -44,7 +44,7 @@
     <td class="djdt-actions">
         <form method="get" action="{% url 'djdt:history_sidebar' %}">
             {{ store_context.form }}
-            <button data-request-id="{{ id }}" class="switchHistory">Switch</button>
+            <button data-request-id="{{ request_id }}" class="switchHistory">Switch</button>
         </form>
     </td>
 </tr>
diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py
@@ -1,9 +1,8 @@
 """
 The main DebugToolbar class that loads and renders the Toolbar.
 """
-
+import logging
 import uuid
-from collections import OrderedDict
 from functools import lru_cache
 
 from django.apps import apps
@@ -17,13 +16,17 @@
 from django.utils.translation import get_language, override as lang_override
 
 from debug_toolbar import APP_NAME, settings as dt_settings
+from debug_toolbar.store import get_store
+
+logger = logging.getLogger(__name__)
 
 
 class DebugToolbar:
     # for internal testing use only
     _created = Signal()
+    store = None
 
-    def __init__(self, request, get_response):
+    def __init__(self, request, get_response, request_id=None):
         self.request = request
         self.config = dt_settings.get_config().copy()
         panels = []
@@ -33,16 +36,11 @@ def __init__(self, request, get_response):
             if panel.enabled:
                 get_response = panel.process_request
         self.process_request = get_response
-        # Use OrderedDict for the _panels attribute so that items can be efficiently
-        # removed using FIFO order in the DebugToolbar.store() method.  The .popitem()
-        # method of Python's built-in dict only supports LIFO removal.
-        self._panels = OrderedDict()
-        while panels:
-            panel = panels.pop()
-            self._panels[panel.panel_id] = panel
+        self._panels = {panel.panel_id: panel for panel in reversed(panels)}
         self.stats = {}
         self.server_timing_stats = {}
-        self.request_id = None
+        self.request_id = request_id
+        self.init_store()
         self._created.send(request, toolbar=self)
 
     # Manage panels
@@ -74,7 +72,7 @@ def render_toolbar(self):
         Renders the overall Toolbar with panels inside.
         """
         if not self.should_render_panels():
-            self.store()
+            self.init_store()
         try:
             context = {"toolbar": self}
             lang = self.config["TOOLBAR_LANGUAGE"] or get_language()
@@ -106,20 +104,20 @@ def should_render_panels(self):
 
     # Handle storing toolbars in memory and fetching them later on
 
-    _store = OrderedDict()
+    def init_store(self):
+        # Store already initialized.
+        if self.store is None:
+            self.store = get_store()
 
-    def store(self):
-        # Store already exists.
         if self.request_id:
             return
         self.request_id = uuid.uuid4().hex
-        self._store[self.request_id] = self
-        for _ in range(self.config["RESULTS_CACHE_SIZE"], len(self._store)):
-            self._store.popitem(last=False)
+        self.store.set(self.request_id)
 
     @classmethod
-    def fetch(cls, request_id):
-        return cls._store.get(request_id)
+    def fetch(cls, request_id, panel_id=None):
+        if get_store().exists(request_id):
+            return StoredDebugToolbar.from_store(request_id, panel_id=panel_id)
 
     # Manually implement class-level caching of panel classes and url patterns
     # because it's more obvious than going through an abstraction.
@@ -186,3 +184,38 @@ def observe_request(request):
     Determine whether to update the toolbar from a client side request.
     """
     return not DebugToolbar.is_toolbar_request(request)
+
+
+def from_store_get_response(request):
+    logger.warning(
+        "get_response was called for debug toolbar after being loaded from the store. No request exists in this scenario as the request is not stored, only the panel's data."
+    )
+    return None
+
+
+class StoredDebugToolbar(DebugToolbar):
+    def __init__(self, request, get_response, request_id=None):
+        self.request = None
+        self.config = dt_settings.get_config().copy()
+        self.process_request = get_response
+        self.stats = {}
+        self.server_timing_stats = {}
+        self.request_id = request_id
+        self.init_store()
+
+    @classmethod
+    def from_store(cls, request_id, panel_id=None):
+        toolbar = StoredDebugToolbar(
+            None, from_store_get_response, request_id=request_id
+        )
+        toolbar._panels = {}
+
+        for panel_class in reversed(cls.get_panel_classes()):
+            panel = panel_class(toolbar, from_store_get_response)
+            if panel_id and panel.panel_id != panel_id:
+                continue
+            data = toolbar.store.panel(toolbar.request_id, panel.panel_id)
+            if data:
+                panel.load_stats_from_store(data)
+                toolbar._panels[panel.panel_id] = panel
+        return toolbar
diff --git a/debug_toolbar/views.py b/debug_toolbar/views.py
@@ -10,7 +10,7 @@
 @render_with_toolbar_language
 def render_panel(request):
     """Render the contents of a panel"""
-    toolbar = DebugToolbar.fetch(request.GET["request_id"])
+    toolbar = DebugToolbar.fetch(request.GET["request_id"], request.GET["panel_id"])
     if toolbar is None:
         content = _(
             "Data for this panel isn't available anymore. "
diff --git a/docs/changes.rst b/docs/changes.rst
@@ -11,7 +11,12 @@ Serializable (don't include in main)
   warnings when a ``TypeError`` occurs during a panel's serialization.
 * Rename ``store_id`` properties to ``request_id`` and ``Toolbar.store`` to
   ``Toolbar.init_store``.
-
+* Support ``Panel`` instances with stored stats via
+  ``Panel.load_stats_from_store``.
+* Swapped ``Toolbar._store`` for the ``get_store()`` class.
+* Created a ``StoredDebugToolbar`` that support creating an instance of the
+  toolbar representing an old request. It should only be used for fetching
+  panels' contents.
 
 Pending
 -------
diff --git a/tests/base.py b/tests/base.py
@@ -3,6 +3,7 @@
 from django.http import HttpResponse
 from django.test import Client, RequestFactory, TestCase, TransactionTestCase
 
+from debug_toolbar.store import get_store
 from debug_toolbar.toolbar import DebugToolbar
 
 
@@ -82,6 +83,5 @@ def setUp(self):
         # The HistoryPanel keeps track of previous stores in memory.
         # This bleeds into other tests and violates their idempotency.
         # Clear the store before each test.
-        for key in list(DebugToolbar._store.keys()):
-            del DebugToolbar._store[key]
+        get_store().clear()
         super().setUp()
diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py
diff --git a/tests/settings.py b/tests/settings.py
diff --git a/tests/test_integration.py b/tests/test_integration.py
