From dd394db3da35d1a7f8060652e4c332afefcc1d67 Mon Sep 17 00:00:00 2001 From: tschilling Date: Tue, 16 May 2023 21:14:41 -0500 Subject: [PATCH 1/3] Add the Store API and initial documentation. --- debug_toolbar/settings.py | 1 + debug_toolbar/store.py | 127 ++++++++++++++++++++++++++++++++++++++ docs/changes.rst | 4 +- docs/configuration.rst | 9 +++ tests/test_store.py | 126 +++++++++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 debug_toolbar/store.py create mode 100644 tests/test_store.py diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index eb6b59209..fcd253c59 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -42,6 +42,7 @@ "SQL_WARNING_THRESHOLD": 500, # milliseconds "OBSERVE_REQUEST_CALLBACK": "debug_toolbar.toolbar.observe_request", "TOOLBAR_LANGUAGE": None, + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.MemoryStore", } diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py new file mode 100644 index 000000000..66cd89e8a --- /dev/null +++ b/debug_toolbar/store.py @@ -0,0 +1,127 @@ +import json +from collections import defaultdict, deque +from typing import Any, Dict, Iterable + +from django.core.serializers.json import DjangoJSONEncoder +from django.utils.encoding import force_str +from django.utils.module_loading import import_string + +from debug_toolbar import settings as dt_settings + + +class DebugToolbarJSONEncoder(DjangoJSONEncoder): + def default(self, o: Any) -> Any: + try: + return super().default(o) + except TypeError: + return force_str(o) + + +def serialize(data: Any) -> str: + return json.dumps(data, cls=DebugToolbarJSONEncoder) + + +def deserialize(data: str) -> Any: + return json.loads(data) + + +class BaseStore: + _config = dt_settings.get_config().copy() + + @classmethod + def ids(cls) -> Iterable: + """The stored ids""" + raise NotImplementedError + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the store""" + raise NotImplementedError + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the store""" + raise NotImplementedError + + @classmethod + def clear(cls): + """Remove all requests from the request store""" + raise NotImplementedError + + @classmethod + def delete(cls, request_id: str): + """Delete the store for the given request_id""" + raise NotImplementedError + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + raise NotImplementedError + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + raise NotImplementedError + + +class MemoryStore(BaseStore): + # ids is the collection of storage ids that have been used. + # Use a dequeue to support O(1) appends and pops + # from either direction. + _ids: deque = deque() + _request_store: Dict[str, Dict] = defaultdict(dict) + + @classmethod + def ids(cls) -> Iterable: + """The stored ids""" + return cls._ids + + @classmethod + def exists(cls, request_id: str) -> bool: + """Does the given request_id exist in the request store""" + return request_id in cls._ids + + @classmethod + def set(cls, request_id: str): + """Set a request_id in the request store""" + if request_id not in cls._ids: + cls._ids.append(request_id) + for _ in range(len(cls._ids) - cls._config["RESULTS_CACHE_SIZE"]): + removed_id = cls._ids.popleft() + cls._request_store.pop(removed_id, None) + + @classmethod + def clear(cls): + """Remove all requests from the request store""" + cls._ids.clear() + cls._request_store.clear() + + @classmethod + def delete(cls, request_id: str): + """Delete the stored request for the given request_id""" + cls._request_store.pop(request_id, None) + try: + cls._ids.remove(request_id) + except ValueError: + # The request_id doesn't exist in the collection of ids. + pass + + @classmethod + def save_panel(cls, request_id: str, panel_id: str, data: Any = None): + """Save the panel data for the given request_id""" + cls.set(request_id) + cls._request_store[request_id][panel_id] = serialize(data) + + @classmethod + def panel(cls, request_id: str, panel_id: str) -> Any: + """Fetch the panel data for the given request_id""" + try: + data = cls._request_store[request_id][panel_id] + except KeyError: + return {} + else: + return deserialize(data) + + +def get_store(): + return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/docs/changes.rst b/docs/changes.rst index ab69ef99f..33434b49b 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -9,10 +9,12 @@ Pending `__. * Converted cookie keys to lowercase. Fixed the ``samesite`` argument to ``djdt.cookie.set``. +* Defines the ``BaseStore`` interface for request storage mechanisms. +* Added the config setting ``TOOLBAR_STORE_CLASS`` to configure the request + storage mechanism. Defaults to ``debug_toolbar.store.MemoryStore``. 4.1.0 (2023-05-15) ------------------ - * Improved SQL statement formatting performance. Additionally, fixed the indentation of ``CASE`` statements and stopped simplifying ``.count()`` queries. diff --git a/docs/configuration.rst b/docs/configuration.rst index 887608c6e..f2f6b7de9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -150,6 +150,15 @@ Toolbar options the request doesn't originate from the toolbar itself, EG that ``is_toolbar_request`` is false for a given request. +.. _TOOLBAR_STORE_CLASS: + +* ``TOOLBAR_STORE_CLASS`` + + Default: ``"debug_toolbar.store.MemoryStore"`` + + The path to the class to be used for storing the toolbar's data per request. + + .. _TOOLBAR_LANGUAGE: * ``TOOLBAR_LANGUAGE`` diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 000000000..d3381084e --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,126 @@ +from django.test import TestCase +from django.test.utils import override_settings + +from debug_toolbar import store + + +class SerializationTestCase(TestCase): + def test_serialize(self): + self.assertEqual( + store.serialize({"hello": {"foo": "bar"}}), + '{"hello": {"foo": "bar"}}', + ) + + def test_serialize_force_str(self): + class Foo: + spam = "bar" + + def __str__(self): + return f"Foo spam={self.spam}" + + self.assertEqual( + store.serialize({"hello": Foo()}), + '{"hello": "Foo spam=bar"}', + ) + + def test_deserialize(self): + self.assertEqual( + store.deserialize('{"hello": {"foo": "bar"}}'), + {"hello": {"foo": "bar"}}, + ) + + +class BaseStoreTestCase(TestCase): + def test_methods_are_not_implemented(self): + # Find all the non-private and dunder class methods + methods = [ + member for member in vars(store.BaseStore) if not member.startswith("_") + ] + self.assertEqual(len(methods), 7) + with self.assertRaises(NotImplementedError): + store.BaseStore.ids() + with self.assertRaises(NotImplementedError): + store.BaseStore.exists("") + with self.assertRaises(NotImplementedError): + store.BaseStore.set("") + with self.assertRaises(NotImplementedError): + store.BaseStore.clear() + with self.assertRaises(NotImplementedError): + store.BaseStore.delete("") + with self.assertRaises(NotImplementedError): + store.BaseStore.save_panel("", "", None) + with self.assertRaises(NotImplementedError): + store.BaseStore.panel("", "") + + +class MemoryStoreTestCase(TestCase): + @classmethod + def setUpTestData(cls) -> None: + cls.store = store.MemoryStore + + def tearDown(self) -> None: + self.store.clear() + + def test_ids(self): + self.store.set("foo") + self.store.set("bar") + self.assertEqual(list(self.store.ids()), ["foo", "bar"]) + + def test_exists(self): + self.assertFalse(self.store.exists("missing")) + self.store.set("exists") + self.assertTrue(self.store.exists("exists")) + + def test_set(self): + self.store.set("foo") + self.assertEqual(list(self.store.ids()), ["foo"]) + + def test_set_max_size(self): + existing = self.store._config["RESULTS_CACHE_SIZE"] + self.store._config["RESULTS_CACHE_SIZE"] = 1 + self.store.save_panel("foo", "foo.panel", "foo.value") + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.assertEqual(list(self.store.ids()), ["bar"]) + self.assertEqual(self.store.panel("foo", "foo.panel"), {}) + self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + # Restore the existing config setting since this config is shared. + self.store._config["RESULTS_CACHE_SIZE"] = existing + + def test_clear(self): + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.store.clear() + self.assertEqual(list(self.store.ids()), []) + self.assertEqual(self.store.panel("bar", "bar.panel"), {}) + + def test_delete(self): + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.store.delete("bar") + self.assertEqual(list(self.store.ids()), []) + self.assertEqual(self.store.panel("bar", "bar.panel"), {}) + # Make sure it doesn't error + self.store.delete("bar") + + def test_save_panel(self): + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.assertEqual(list(self.store.ids()), ["bar"]) + self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + + def test_panel(self): + self.assertEqual(self.store.panel("missing", "missing"), {}) + self.store.save_panel("bar", "bar.panel", {"a": 1}) + self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) + + +class StubStore(store.BaseStore): + pass + + +class GetStoreTestCase(TestCase): + def test_get_store(self): + self.assertIs(store.get_store(), store.MemoryStore) + + @override_settings( + DEBUG_TOOLBAR_CONFIG={"TOOLBAR_STORE_CLASS": "tests.test_store.StubStore"} + ) + def test_get_store_with_setting(self): + self.assertIs(store.get_store(), StubStore) From 75c98a70cec0beb0c1aeeaa3ce37fefceb582514 Mon Sep 17 00:00:00 2001 From: tschilling Date: Tue, 16 May 2023 21:24:32 -0500 Subject: [PATCH 2/3] Remove config from docs as sphinx says it's misspelled. --- docs/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 33434b49b..c526a37b4 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -10,7 +10,7 @@ Pending * Converted cookie keys to lowercase. Fixed the ``samesite`` argument to ``djdt.cookie.set``. * Defines the ``BaseStore`` interface for request storage mechanisms. -* Added the config setting ``TOOLBAR_STORE_CLASS`` to configure the request +* Added the setting ``TOOLBAR_STORE_CLASS`` to configure the request storage mechanism. Defaults to ``debug_toolbar.store.MemoryStore``. 4.1.0 (2023-05-15) From 99f4473c3eb36fb5882a5dbde84cd4a5a90d3da8 Mon Sep 17 00:00:00 2001 From: tschilling Date: Sat, 17 Jun 2023 10:15:18 -0500 Subject: [PATCH 3/3] Switch to Store.request_ids and remove serialization force_str. If the serialization logic begins throwing exceptions we can consider subclassing the encoder class and using force_str on the object itself. --- debug_toolbar/store.py | 47 ++++++++++++++++++------------------------ tests/test_store.py | 26 +++++++---------------- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 66cd89e8a..b32d3b62a 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -1,24 +1,19 @@ +import contextlib import json from collections import defaultdict, deque from typing import Any, Dict, Iterable from django.core.serializers.json import DjangoJSONEncoder -from django.utils.encoding import force_str from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings -class DebugToolbarJSONEncoder(DjangoJSONEncoder): - def default(self, o: Any) -> Any: - try: - return super().default(o) - except TypeError: - return force_str(o) - - def serialize(data: Any) -> str: - return json.dumps(data, cls=DebugToolbarJSONEncoder) + # If this starts throwing an exceptions, consider + # Subclassing DjangoJSONEncoder and using force_str to + # make it JSON serializable. + return json.dumps(data, cls=DjangoJSONEncoder) def deserialize(data: str) -> Any: @@ -29,8 +24,8 @@ class BaseStore: _config = dt_settings.get_config().copy() @classmethod - def ids(cls) -> Iterable: - """The stored ids""" + def request_ids(cls) -> Iterable: + """The stored request ids""" raise NotImplementedError @classmethod @@ -68,43 +63,41 @@ class MemoryStore(BaseStore): # ids is the collection of storage ids that have been used. # Use a dequeue to support O(1) appends and pops # from either direction. - _ids: deque = deque() + _request_ids: deque = deque() _request_store: Dict[str, Dict] = defaultdict(dict) @classmethod - def ids(cls) -> Iterable: - """The stored ids""" - return cls._ids + def request_ids(cls) -> Iterable: + """The stored request ids""" + return cls._request_ids @classmethod def exists(cls, request_id: str) -> bool: """Does the given request_id exist in the request store""" - return request_id in cls._ids + return request_id in cls._request_ids @classmethod def set(cls, request_id: str): """Set a request_id in the request store""" - if request_id not in cls._ids: - cls._ids.append(request_id) - for _ in range(len(cls._ids) - cls._config["RESULTS_CACHE_SIZE"]): - removed_id = cls._ids.popleft() + if request_id not in cls._request_ids: + cls._request_ids.append(request_id) + for _ in range(len(cls._request_ids) - cls._config["RESULTS_CACHE_SIZE"]): + removed_id = cls._request_ids.popleft() cls._request_store.pop(removed_id, None) @classmethod def clear(cls): """Remove all requests from the request store""" - cls._ids.clear() + cls._request_ids.clear() cls._request_store.clear() @classmethod def delete(cls, request_id: str): """Delete the stored request for the given request_id""" cls._request_store.pop(request_id, None) - try: - cls._ids.remove(request_id) - except ValueError: - # The request_id doesn't exist in the collection of ids. - pass + # Suppress when request_id doesn't exist in the collection of ids. + with contextlib.suppress(ValueError): + cls._request_ids.remove(request_id) @classmethod def save_panel(cls, request_id: str, panel_id: str, data: Any = None): diff --git a/tests/test_store.py b/tests/test_store.py index d3381084e..c51afde1e 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -11,18 +11,6 @@ def test_serialize(self): '{"hello": {"foo": "bar"}}', ) - def test_serialize_force_str(self): - class Foo: - spam = "bar" - - def __str__(self): - return f"Foo spam={self.spam}" - - self.assertEqual( - store.serialize({"hello": Foo()}), - '{"hello": "Foo spam=bar"}', - ) - def test_deserialize(self): self.assertEqual( store.deserialize('{"hello": {"foo": "bar"}}'), @@ -38,7 +26,7 @@ def test_methods_are_not_implemented(self): ] self.assertEqual(len(methods), 7) with self.assertRaises(NotImplementedError): - store.BaseStore.ids() + store.BaseStore.request_ids() with self.assertRaises(NotImplementedError): store.BaseStore.exists("") with self.assertRaises(NotImplementedError): @@ -64,7 +52,7 @@ def tearDown(self) -> None: def test_ids(self): self.store.set("foo") self.store.set("bar") - self.assertEqual(list(self.store.ids()), ["foo", "bar"]) + self.assertEqual(list(self.store.request_ids()), ["foo", "bar"]) def test_exists(self): self.assertFalse(self.store.exists("missing")) @@ -73,14 +61,14 @@ def test_exists(self): def test_set(self): self.store.set("foo") - self.assertEqual(list(self.store.ids()), ["foo"]) + self.assertEqual(list(self.store.request_ids()), ["foo"]) def test_set_max_size(self): existing = self.store._config["RESULTS_CACHE_SIZE"] self.store._config["RESULTS_CACHE_SIZE"] = 1 self.store.save_panel("foo", "foo.panel", "foo.value") self.store.save_panel("bar", "bar.panel", {"a": 1}) - self.assertEqual(list(self.store.ids()), ["bar"]) + self.assertEqual(list(self.store.request_ids()), ["bar"]) self.assertEqual(self.store.panel("foo", "foo.panel"), {}) self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) # Restore the existing config setting since this config is shared. @@ -89,20 +77,20 @@ def test_set_max_size(self): def test_clear(self): self.store.save_panel("bar", "bar.panel", {"a": 1}) self.store.clear() - self.assertEqual(list(self.store.ids()), []) + self.assertEqual(list(self.store.request_ids()), []) self.assertEqual(self.store.panel("bar", "bar.panel"), {}) def test_delete(self): self.store.save_panel("bar", "bar.panel", {"a": 1}) self.store.delete("bar") - self.assertEqual(list(self.store.ids()), []) + self.assertEqual(list(self.store.request_ids()), []) self.assertEqual(self.store.panel("bar", "bar.panel"), {}) # Make sure it doesn't error self.store.delete("bar") def test_save_panel(self): self.store.save_panel("bar", "bar.panel", {"a": 1}) - self.assertEqual(list(self.store.ids()), ["bar"]) + self.assertEqual(list(self.store.request_ids()), ["bar"]) self.assertEqual(self.store.panel("bar", "bar.panel"), {"a": 1}) def test_panel(self):