Support storing serialized panel stats directly into the store.

tim-schilling · tim-schilling · commit 220373f620ea · 2021-01-03T17:22:58.000-06:00
In order to support the toolbar persisting stats, we need to move away
from calling record_stats with non-primitives.
diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py
@@ -1,6 +1,7 @@
 from django.template.loader import render_to_string
 
 from debug_toolbar import settings as dt_settings
+from debug_toolbar.store import store
 from debug_toolbar.utils import get_name_from_obj
 
 
@@ -146,19 +147,38 @@ def disable_instrumentation(self):
 
     # Store and retrieve stats (shared between panels for no good reason)
 
+    def deserialize_stats(self, data):
+        """
+        Deserialize stats coming from the store.
+
+        Provided to support future store mechanisms overriding a panel's content.
+        """
+        return data
+
+    def serialize_stats(self, stats):
+        """
+        Serialize stats for the store.
+
+        Provided to support future store mechanisms overriding a panel's content.
+        """
+        return stats
+
     def record_stats(self, stats):
         """
         Store data gathered by the panel. ``stats`` is a :class:`dict`.
 
         Each call to ``record_stats`` updates the statistics dictionary.
         """
         self.toolbar.stats.setdefault(self.panel_id, {}).update(stats)
+        store.save_panel(
+            self.toolbar.store_id, self.panel_id, self.serialize_stats(stats)
+        )
 
     def get_stats(self):
         """
         Access data stored by the panel. Returns a :class:`dict`.
         """
-        return self.toolbar.stats.get(self.panel_id, {})
+        return self.deserialize_stats(store.panel(self.toolbar.store_id, self.panel_id))
 
     def record_server_timing(self, key, title, value):
         """
diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py
@@ -208,7 +208,7 @@ def _store_call_info(
                 "kwargs": kwargs,
                 "trace": render_stacktrace(trace),
                 "template_info": template_info,
-                "backend": backend,
+                "backend": str(backend),
             }
         )
 
diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py
@@ -40,9 +40,9 @@ def nav_subtitle(self):
     def generate_stats(self, request, response):
         try:
             if request.method == "GET":
-                data = request.GET.copy()
+                data = dict(request.GET.copy())
             else:
-                data = request.POST.copy()
+                data = dict(request.POST.copy())
             # GraphQL tends to not be populated in POST. If the request seems
             # empty, check if it's a JSON request.
             if (
diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py
@@ -51,6 +51,7 @@ def __init__(
         self.id = id
         self.parent_ids = parent_ids
         self.hsv = hsv
+        self.has_subfuncs = False
 
     def parent_classes(self):
         return self.parent_classes
@@ -141,6 +142,20 @@ def cumtime_per_call(self):
     def indent(self):
         return 16 * self.depth
 
+    def as_context(self):
+        return {
+            "id": self.id,
+            "parent_ids": self.parent_ids,
+            "func_std_string": self.func_std_string(),
+            "has_subfuncs": self.has_subfuncs,
+            "cumtime": self.cumtime(),
+            "cumtime_per_call": self.cumtime_per_call(),
+            "tottime": self.tottime(),
+            "tottime_per_call": self.tottime_per_call(),
+            "count": self.count(),
+            "indent": self.indent(),
+        }
+
 
 class ProfilingPanel(Panel):
     """
@@ -157,7 +172,6 @@ def process_request(self, request):
 
     def add_node(self, func_list, func, max_depth, cum_time=0.1):
         func_list.append(func)
-        func.has_subfuncs = False
         if func.depth < max_depth:
             for subfunc in func.subfuncs():
                 if subfunc.stats[3] >= cum_time:
@@ -183,4 +197,4 @@ def generate_stats(self, request, response):
                 dt_settings.get_config()["PROFILER_MAX_DEPTH"],
                 root.stats[3] / 8,
             )
-            self.record_stats({"func_list": func_list})
+            self.record_stats({"func_list": [func.as_context() for func in func_list]})
diff --git a/debug_toolbar/panels/request.py b/debug_toolbar/panels/request.py
@@ -24,13 +24,11 @@ def nav_subtitle(self):
         return view_func.rsplit(".", 1)[-1]
 
     def generate_stats(self, request, response):
-        self.record_stats(
-            {
-                "get": get_sorted_request_variable(request.GET),
-                "post": get_sorted_request_variable(request.POST),
-                "cookies": get_sorted_request_variable(request.COOKIES),
-            }
-        )
+        stats = {
+            "get": get_sorted_request_variable(request.GET),
+            "post": get_sorted_request_variable(request.POST),
+            "cookies": get_sorted_request_variable(request.COOKIES),
+        }
 
         view_info = {
             "view_func": _("<no view>"),
@@ -47,14 +45,15 @@ def generate_stats(self, request, response):
             view_info["view_urlname"] = getattr(match, "url_name", _("<unavailable>"))
         except Http404:
             pass
-        self.record_stats(view_info)
+        stats.update(view_info)
 
         if hasattr(request, "session"):
-            self.record_stats(
+            stats.update(
                 {
                     "session": [
                         (k, request.session.get(k))
                         for k in sorted(request.session.keys())
                     ]
                 }
             )
+        self.record_stats(stats)
diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py
@@ -171,20 +171,24 @@ def disable_instrumentation(self):
     def generate_stats(self, request, response):
         template_context = []
         for template_data in self.templates:
-            info = {}
             # Clean up some info about templates
             template = template_data["template"]
             if hasattr(template, "origin") and template.origin and template.origin.name:
-                template.origin_name = template.origin.name
-                template.origin_hash = signing.dumps(template.origin.name)
+                origin_name = template.origin.name
+                origin_hash = signing.dumps(template.origin.name)
             else:
-                template.origin_name = _("No origin")
-                template.origin_hash = ""
-            info["template"] = template
-            # Clean up context for better readability
-            if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]:
-                context_list = template_data.get("context", [])
-                info["context"] = "\n".join(context_list)
+                origin_name = _("No origin")
+                origin_hash = ""
+            info = {
+                "template": {
+                    "origin_name": origin_name,
+                    "origin_hash": origin_hash,
+                    "name": template.name,
+                },
+                "context": "\n".join(template_data.get("context", []))
+                if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]
+                else "",
+            }
             template_context.append(info)
 
         # Fetch context_processors/template_dirs from any template
diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py
@@ -1,10 +1,32 @@
-from collections import OrderedDict
+import json
+from collections import OrderedDict, defaultdict
 
+from django.core.serializers.json import DjangoJSONEncoder
 from django.utils.module_loading import import_string
 
 from debug_toolbar import settings as dt_settings
 
 
+class DebugToolbarJSONEncoder(DjangoJSONEncoder):
+    def default(self, o):
+        try:
+            return super().default(o)
+        except TypeError:
+            return str(o)
+
+
+def serialize(data):
+    return json.dumps(data, cls=DebugToolbarJSONEncoder)
+
+
+def deserialize(data):
+    return json.loads(data)
+
+
+# Record stats in serialized fashion.
+# Remove use of fetching the toolbar as a whole from the store.
+
+
 class BaseStore:
     config = dt_settings.get_config().copy()
 
@@ -24,9 +46,14 @@ def set(cls, store_id, toolbar):
     def delete(cls, store_id):
         raise NotImplementedError
 
+    @classmethod
+    def record_stats(cls, store_id, panel_id, stats):
+        raise NotImplementedError
+
 
 class MemoryStore(BaseStore):
     _store = OrderedDict()
+    _stats = defaultdict(dict)
 
     @classmethod
     def get(cls, store_id):
@@ -46,5 +73,17 @@ def set(cls, store_id, toolbar):
     def delete(cls, store_id):
         del cls._store[store_id]
 
+    @classmethod
+    def save_panel(cls, store_id, panel_id, stats=None):
+        cls._stats[store_id][panel_id] = serialize(stats)
+
+    @classmethod
+    def panel(cls, store_id, panel_id):
+        try:
+            data = cls._stats[store_id][panel_id]
+        except KeyError:
+            data = None
+        return {} if data is None else deserialize(data)
+
 
 store = import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"])
diff --git a/debug_toolbar/templates/debug_toolbar/panels/cache.html b/debug_toolbar/templates/debug_toolbar/panels/cache.html
@@ -61,7 +61,8 @@ <h4>{% trans "Calls" %}</h4>
         </tr>
         <tr class="djUnselected djToggleDetails_{{ forloop.counter }}" id="cacheDetails_{{ forloop.counter }}">
           <td colspan="1"></td>
-          <td colspan="5"><pre class="djdt-stack">{{ call.trace }}</pre></td>
+          {# The trace property is escaped when serialized into the store #}
+          <td colspan="5"><pre class="djdt-stack">{{ call.trace|safe }}</pre></td>
         </tr>
       {% endfor %}
     </tbody>
diff --git a/debug_toolbar/templates/debug_toolbar/panels/profiling.html b/debug_toolbar/templates/debug_toolbar/panels/profiling.html
@@ -20,7 +20,8 @@
             {% else %}
               <span class="djNoToggleSwitch"></span>
             {% endif %}
-            <span class="djdt-stack">{{ call.func_std_string }}</span>
+            {# The func_std_string property is escaped when serialized into the store #}
+            <span class="djdt-stack">{{ call.func_std_string|safe }}</span>
           </div>
         </td>
         <td>{{ call.cumtime|floatformat:3 }}</td>
diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html
@@ -77,7 +77,8 @@
             {% if query.params %}
               {% if query.is_select %}
                 <form method="post">
-                  {{ query.form }}
+                  {# The form is rendered when serialized into storage #}
+                  {{ query.form|safe }}
                   <button formaction="{% url 'djdt:sql_select' %}" class="remoteCall">Sel</button>
                   <button formaction="{% url 'djdt:sql_explain' %}" class="remoteCall">Expl</button>
                   {% if query.vendor == 'mysql' %}
@@ -100,7 +101,8 @@
                 <p><strong>{% trans "Transaction status:" %}</strong> {{ query.trans_status }}</p>
               {% endif %}
               {% if query.stacktrace %}
-                <pre class="djdt-stack">{{ query.stacktrace }}</pre>
+                {# The stacktrace property is a rendered template. It is escaped when serialized into the store #}
+                <pre class="djdt-stack">{{ query.stacktrace|safe }}</pre>
               {% endif %}
               {% if query.template_info %}
                 <table>
diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py
@@ -33,7 +33,7 @@ def __init__(self, request, get_response):
             self._panels[panel.panel_id] = panel
         self.stats = {}
         self.server_timing_stats = {}
-        self.store_id = None
+        self.store_id = uuid.uuid4().hex
 
     # Manage panels
 
@@ -85,10 +85,6 @@ def should_render_panels(self):
         return render_panels
 
     def store(self):
-        # Store already exists.
-        if self.store_id:
-            return
-        self.store_id = uuid.uuid4().hex
         store.set(self.store_id, self)
 
     # Manually implement class-level caching of panel classes and url patterns
diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py
@@ -25,7 +25,7 @@ def test_post(self):
         response = self.panel.process_request(self.request)
         self.panel.generate_stats(self.request, response)
         data = self.panel.get_stats()["data"]
-        self.assertEqual(data["foo"], "bar")
+        self.assertEqual(data["foo"], ["bar"])
 
     def test_post_json(self):
         for data, expected_stats_data in (
diff --git a/tests/test_integration.py b/tests/test_integration.py
@@ -68,17 +68,17 @@ def test_url_resolving_positional(self):
         stats = self._resolve_stats("/resolving1/a/b/")
         self.assertEqual(stats["view_urlname"], "positional-resolving")
         self.assertEqual(stats["view_func"], "tests.views.resolving_view")
-        self.assertEqual(stats["view_args"], ("a", "b"))
+        self.assertEqual(stats["view_args"], ["a", "b"])
         self.assertEqual(stats["view_kwargs"], {})
 
     def test_url_resolving_named(self):
         stats = self._resolve_stats("/resolving2/a/b/")
-        self.assertEqual(stats["view_args"], ())
+        self.assertEqual(stats["view_args"], [])
         self.assertEqual(stats["view_kwargs"], {"arg1": "a", "arg2": "b"})
 
     def test_url_resolving_mixed(self):
         stats = self._resolve_stats("/resolving3/a/")
-        self.assertEqual(stats["view_args"], ("a",))
+        self.assertEqual(stats["view_args"], ["a"])
         self.assertEqual(stats["view_kwargs"], {"arg2": "default"})
 
     def test_url_resolving_bad(self):
@@ -159,7 +159,7 @@ def test_middleware_render_toolbar_json(self):
         """Verify the toolbar is rendered and data is stored for a json request."""
         self.assertEqual(len(store.all()), 0)
 
-        data = {"foo": "bar"}
+        data = {"foo": "bar", "spam[]": ["eggs", "ham"]}
         response = self.client.get("/json_view/", data, content_type="application/json")
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content.decode("utf-8"), '{"foo": "bar"}')
@@ -168,7 +168,7 @@ def test_middleware_render_toolbar_json(self):
         toolbar = list(store.all())[0][1]
         self.assertEqual(
             toolbar.get_panel_by_id("HistoryPanel").get_stats()["data"],
-            {"foo": ["bar"]},
+            {"foo": ["bar"], "spam[]": ["eggs", "ham"]},
         )
 
     def test_template_source_checks_show_toolbar(self):
@@ -289,13 +289,6 @@ def test_sql_profile_checks_show_toolbar(self):
             )
             self.assertEqual(response.status_code, 404)
 
-    @override_settings(DEBUG_TOOLBAR_CONFIG={"RENDER_PANELS": True})
-    def test_data_store_id_not_rendered_when_none(self):
-        url = "/regular/basic/"
-        response = self.client.get(url)
-        self.assertIn(b'id="djDebug"', response.content)
-        self.assertNotIn(b"data-store-id", response.content)
-
     def test_view_returns_template_response(self):
         response = self.client.get("/template_response/basic/")
         self.assertEqual(response.status_code, 200)

Original file line number	Diff line number	Diff line change
`@@ -208,7 +208,7 @@ def _store_call_info(`
`208`	`208`	`"kwargs": kwargs,`
`209`	`209`	`"trace": render_stacktrace(trace),`
`210`	`210`	`"template_info": template_info,`
`211`		`- "backend": backend,`
	`211`	`+ "backend": str(backend),`
`212`	`212`	`}`
`213`	`213`	`)`
`214`	`214`