From 6b930090afb3514935f3f1ee40b49785d0bf68ae Mon Sep 17 00:00:00 2001 From: tschilling Date: Sat, 10 Sep 2022 13:01:31 -0500 Subject: [PATCH 1/9] Give users control over including more profiling data. --- debug_toolbar/panels/profiling.py | 9 ++++++--- debug_toolbar/settings.py | 1 + docs/changes.rst | 5 +++++ docs/configuration.rst | 14 ++++++++++++++ tests/panels/test_profiling.py | 13 +++++++++++++ 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index 5fd5b3c84..1c2681455 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -128,14 +128,14 @@ def process_request(self, request): self.profiler = cProfile.Profile() return self.profiler.runcall(super().process_request, request) - def add_node(self, func_list, func, max_depth, cum_time=0.1): + def add_node(self, func_list, func, max_depth, cum_time): func_list.append(func) func.has_subfuncs = False if func.depth < max_depth: for subfunc in func.subfuncs(): if subfunc.stats[3] >= cum_time: func.has_subfuncs = True - self.add_node(func_list, subfunc, max_depth, cum_time=cum_time) + self.add_node(func_list, subfunc, max_depth, cum_time) def generate_stats(self, request, response): if not hasattr(self, "profiler"): @@ -150,10 +150,13 @@ def generate_stats(self, request, response): if root_func in self.stats.stats: root = FunctionCall(self.stats, root_func, depth=0) func_list = [] + cum_time_threshold = ( + root.stats[3] / dt_settings.get_config()["PROFILER_THRESHOLD_RATIO"] + ) self.add_node( func_list, root, dt_settings.get_config()["PROFILER_MAX_DEPTH"], - root.stats[3] / 8, + cum_time_threshold, ) self.record_stats({"func_list": func_list}) diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index 5bf9bb09f..c88780dbb 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -34,6 +34,7 @@ ), "PRETTIFY_SQL": True, "PROFILER_MAX_DEPTH": 10, + "PROFILER_THRESHOLD_RATIO": 8, "SHOW_TEMPLATE_CONTEXT": True, "SKIP_TEMPLATE_PREFIXES": ("django/forms/widgets/", "admin/widgets/"), "SQL_WARNING_THRESHOLD": 500, # milliseconds diff --git a/docs/changes.rst b/docs/changes.rst index df6be99f2..031726e71 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,6 +4,11 @@ Change log Pending ------- +* Added Profiling panel setting ``PROFILER_THRESHOLD_RATIO`` to give users + better control over how many function calls are included. A higher value + will include more data, but increase render time. + + 3.6.0 (2022-08-17) ------------------ diff --git a/docs/configuration.rst b/docs/configuration.rst index 6f4084ad5..c017cd1ec 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -259,6 +259,20 @@ Panel options This setting affects the depth of function calls in the profiler's analysis. +* ``PROFILER_THRESHOLD_RATIO`` + + Default: ``10`` + + Panel: profiling + + This setting affects the which calls are included in the profile. A higher + value will include more function calls. A lower value will result in a faster + render of the profiling panel, but will exclude data. + + This value is used to determine the threshold of cumulative time to include + the nested functions. The threshold is calculated by the root calls + cumulative time divided by this ratio. + * ``SHOW_TEMPLATE_CONTEXT`` Default: ``True`` diff --git a/tests/panels/test_profiling.py b/tests/panels/test_profiling.py index ca5c2463b..27e458518 100644 --- a/tests/panels/test_profiling.py +++ b/tests/panels/test_profiling.py @@ -35,6 +35,19 @@ def test_insert_content(self): self.assertIn("regular_view", content) self.assertValidHTML(content) + @override_settings(DEBUG_TOOLBAR_CONFIG={"PROFILER_THRESHOLD_RATIO": 1}) + def test_cum_time_threshold(self): + """ + Test that cumulative time threshold excludes calls + """ + self._get_response = lambda request: regular_view(request, "profiling") + response = self.panel.process_request(self.request) + self.panel.generate_stats(self.request, response) + # ensure the panel renders but doesn't include our function. + content = self.panel.content + self.assertNotIn("regular_view", content) + self.assertValidHTML(content) + def test_listcomp_escaped(self): self._get_response = lambda request: listcomp_view(request) response = self.panel.process_request(self.request) From cc4411954ec044a4d2f2506c5d08b111c153fc9b Mon Sep 17 00:00:00 2001 From: tschilling Date: Sat, 10 Sep 2022 13:19:44 -0500 Subject: [PATCH 2/9] The profiling panel will now include more user code. By checking that the code lives in the settings.BASE_DIR directory, we know that the code was likely written by the user and thus more important to a developer when debugging code. --- debug_toolbar/panels/profiling.py | 21 ++++++++++++++++++++- docs/changes.rst | 2 ++ docs/panels.rst | 5 +++++ tests/panels/test_profiling.py | 4 +++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index 1c2681455..798cfe0bc 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -3,6 +3,7 @@ from colorsys import hsv_to_rgb from pstats import Stats +from django.conf import settings from django.utils.html import format_html from django.utils.translation import gettext_lazy as _ @@ -32,6 +33,21 @@ def background(self): r, g, b = hsv_to_rgb(*self.hsv) return f"rgb({r * 100:f}%,{g * 100:f}%,{b * 100:f}%)" + def is_project_func(self): + """ + Check if the function is from the project code. + + Project code is identified by the BASE_DIR setting + which is used in Django projects by default. + """ + if hasattr(settings, "BASE_DIR"): + file_name, _, _ = self.func + return ( + str(settings.BASE_DIR) in file_name + and "/site-packages" not in file_name + ) + return None + def func_std_string(self): # match what old profile produced func_name = self.func if func_name[:2] == ("~", 0): @@ -133,7 +149,10 @@ def add_node(self, func_list, func, max_depth, cum_time): func.has_subfuncs = False if func.depth < max_depth: for subfunc in func.subfuncs(): - if subfunc.stats[3] >= cum_time: + # Always include the user's code + if subfunc.stats[3] >= cum_time or ( + subfunc.is_project_func() and subfunc.stats[3] > 0 + ): func.has_subfuncs = True self.add_node(func_list, subfunc, max_depth, cum_time) diff --git a/docs/changes.rst b/docs/changes.rst index 031726e71..9ed705069 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -7,6 +7,8 @@ Pending * Added Profiling panel setting ``PROFILER_THRESHOLD_RATIO`` to give users better control over how many function calls are included. A higher value will include more data, but increase render time. +* Update Profiling panel to include try to always include user code. This + code is more important to developers than dependency code. 3.6.0 (2022-08-17) diff --git a/docs/panels.rst b/docs/panels.rst index 8e5558aab..25c12af16 100644 --- a/docs/panels.rst +++ b/docs/panels.rst @@ -130,6 +130,11 @@ Profiling information for the processing of the request. This panel is included but inactive by default. You can activate it by default with the ``DISABLE_PANELS`` configuration option. +The panel will include all function calls made by your project if your using +the setting ``settings.BASE_DIR`` to point to your project's root directory. +If a function is in a file within that directory and does not include +``"/site-packages/"`` in the path, it will be included. + Third-party panels ------------------ diff --git a/tests/panels/test_profiling.py b/tests/panels/test_profiling.py index 27e458518..3c8eff9ea 100644 --- a/tests/panels/test_profiling.py +++ b/tests/panels/test_profiling.py @@ -33,6 +33,7 @@ def test_insert_content(self): # ensure the panel renders correctly. content = self.panel.content self.assertIn("regular_view", content) + self.assertIn("get_template", content) self.assertValidHTML(content) @override_settings(DEBUG_TOOLBAR_CONFIG={"PROFILER_THRESHOLD_RATIO": 1}) @@ -45,7 +46,8 @@ def test_cum_time_threshold(self): self.panel.generate_stats(self.request, response) # ensure the panel renders but doesn't include our function. content = self.panel.content - self.assertNotIn("regular_view", content) + self.assertIn("regular_view", content) + self.assertNotIn("get_template", content) self.assertValidHTML(content) def test_listcomp_escaped(self): From 09c1cb57f1612dad09abc36786ece13648fe7af8 Mon Sep 17 00:00:00 2001 From: tschilling Date: Sat, 10 Sep 2022 13:29:48 -0500 Subject: [PATCH 3/9] Highlight the project function calls in the profiling panel. --- debug_toolbar/static/debug_toolbar/css/toolbar.css | 6 ++++++ debug_toolbar/templates/debug_toolbar/panels/profiling.html | 2 +- docs/changes.rst | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/debug_toolbar/static/debug_toolbar/css/toolbar.css b/debug_toolbar/static/debug_toolbar/css/toolbar.css index a105bfd11..b8bfb5775 100644 --- a/debug_toolbar/static/debug_toolbar/css/toolbar.css +++ b/debug_toolbar/static/debug_toolbar/css/toolbar.css @@ -614,6 +614,12 @@ #djDebug .djdt-highlighted { background-color: lightgrey; } +#djDebug tr.djdt-highlighted.djdt-profile-row { + background-color: #ffc; +} +#djDebug tr.djdt-highlighted.djdt-profile-row:nth-child(2n + 1) { + background-color: #dd9; +} @keyframes djdt-flash-new { from { background-color: green; diff --git a/debug_toolbar/templates/debug_toolbar/panels/profiling.html b/debug_toolbar/templates/debug_toolbar/panels/profiling.html index 837698889..4c1c3acd3 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/profiling.html +++ b/debug_toolbar/templates/debug_toolbar/panels/profiling.html @@ -12,7 +12,7 @@ {% for call in func_list %} - +
{% if call.has_subfuncs %} diff --git a/docs/changes.rst b/docs/changes.rst index 9ed705069..90d197a4f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -9,6 +9,7 @@ Pending will include more data, but increase render time. * Update Profiling panel to include try to always include user code. This code is more important to developers than dependency code. +* Highlight the project function calls in the profiling panel. 3.6.0 (2022-08-17) From 855ade8667c49e89007a854d860fd8a340cb6360 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sat, 10 Sep 2022 13:52:01 -0500 Subject: [PATCH 4/9] Update docs/panels.rst Co-authored-by: Matthias Kestenholz --- docs/panels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/panels.rst b/docs/panels.rst index 25c12af16..92d840d74 100644 --- a/docs/panels.rst +++ b/docs/panels.rst @@ -130,7 +130,7 @@ Profiling information for the processing of the request. This panel is included but inactive by default. You can activate it by default with the ``DISABLE_PANELS`` configuration option. -The panel will include all function calls made by your project if your using +The panel will include all function calls made by your project if you're using the setting ``settings.BASE_DIR`` to point to your project's root directory. If a function is in a file within that directory and does not include ``"/site-packages/"`` in the path, it will be included. From 9970a3138bed4de50686f817c3fbeaedca25c830 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sat, 10 Sep 2022 13:52:16 -0500 Subject: [PATCH 5/9] Update docs/configuration.rst Co-authored-by: Matthias Kestenholz --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index c017cd1ec..017985a5a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -261,7 +261,7 @@ Panel options * ``PROFILER_THRESHOLD_RATIO`` - Default: ``10`` + Default: ``8`` Panel: profiling From 2be3e56744f216885af20fa5b1ebbd3767933fd1 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sat, 10 Sep 2022 13:52:28 -0500 Subject: [PATCH 6/9] Update docs/configuration.rst Co-authored-by: Matthias Kestenholz --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 017985a5a..666e08fda 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -270,7 +270,7 @@ Panel options render of the profiling panel, but will exclude data. This value is used to determine the threshold of cumulative time to include - the nested functions. The threshold is calculated by the root calls + the nested functions. The threshold is calculated by the root calls' cumulative time divided by this ratio. * ``SHOW_TEMPLATE_CONTEXT`` From 94245516a179d5ee6d924b1fdfa5c7c6dd444233 Mon Sep 17 00:00:00 2001 From: tschilling Date: Sun, 11 Sep 2022 12:47:40 -0500 Subject: [PATCH 7/9] Add setting PROFILER_CAPTURE_PROJECT_CODE. This can be used to disable the attempt to include all project code. This is useful if dependencies are installed within the project. --- debug_toolbar/panels/profiling.py | 8 ++++++-- debug_toolbar/settings.py | 1 + docs/changes.rst | 4 ++++ docs/configuration.rst | 12 ++++++++++++ docs/tips.rst | 2 ++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index 798cfe0bc..ca32b98c2 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -44,7 +44,8 @@ def is_project_func(self): file_name, _, _ = self.func return ( str(settings.BASE_DIR) in file_name - and "/site-packages" not in file_name + and "/site-packages/" not in file_name + and "/dist-packages/" not in file_name ) return None @@ -139,6 +140,7 @@ class ProfilingPanel(Panel): title = _("Profiling") template = "debug_toolbar/panels/profiling.html" + capture_project_code = dt_settings.get_config()["PROFILER_CAPTURE_PROJECT_CODE"] def process_request(self, request): self.profiler = cProfile.Profile() @@ -151,7 +153,9 @@ def add_node(self, func_list, func, max_depth, cum_time): for subfunc in func.subfuncs(): # Always include the user's code if subfunc.stats[3] >= cum_time or ( - subfunc.is_project_func() and subfunc.stats[3] > 0 + self.capture_project_code + and subfunc.is_project_func() + and subfunc.stats[3] > 0 ): func.has_subfuncs = True self.add_node(func_list, subfunc, max_depth, cum_time) diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index c88780dbb..2bad251c1 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -33,6 +33,7 @@ "django.utils.functional", ), "PRETTIFY_SQL": True, + "PROFILER_CAPTURE_PROJECT_CODE": True, "PROFILER_MAX_DEPTH": 10, "PROFILER_THRESHOLD_RATIO": 8, "SHOW_TEMPLATE_CONTEXT": True, diff --git a/docs/changes.rst b/docs/changes.rst index 90d197a4f..bb0befcdb 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -10,6 +10,10 @@ Pending * Update Profiling panel to include try to always include user code. This code is more important to developers than dependency code. * Highlight the project function calls in the profiling panel. +* Added Profiling panel setting ``PROFILER_CAPTURE_PROJECT_CODE`` to allow + users to disable the inclusion of all project code. This will be useful + to project setups that have dependencies installed under + ``settings.BASE_DIR``. 3.6.0 (2022-08-17) diff --git a/docs/configuration.rst b/docs/configuration.rst index 666e08fda..07e0a845c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -250,6 +250,18 @@ Panel options WHERE "auth_user"."username" = '''test_username''' LIMIT 21 +* ``PROFILER_CAPTURE_PROJECT_CODE`` + + Default: ``True`` + + Panel: profiling + + When enabled this setting will include all project function calls in the + panel. Project code is defined as files in the path defined at + ``settings.BASE_DIR``. If you install dependencies under + ``settings.BASE_DIR`` in a directory other than ``sites-packages`` or + ``dist-packages`` you may need to disable this setting. + * ``PROFILER_MAX_DEPTH`` Default: ``10`` diff --git a/docs/tips.rst b/docs/tips.rst index e6957b0c6..d5d160fb3 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -77,6 +77,8 @@ by disabling some configuration options that are enabled by default: - ``ENABLE_STACKTRACES`` for the SQL and cache panels, - ``SHOW_TEMPLATE_CONTEXT`` for the template panel. +- ``PROFILER_CAPTURE_PROJECT_CODE`` and ``PROFILER_THRESHOLD_RATIO`` for the + profiling panel. Also, check ``SKIP_TEMPLATE_PREFIXES`` when you're using template-based form widgets. From 308fb0657e04f77a9f329fdb2c982e63f64b6b1d Mon Sep 17 00:00:00 2001 From: tschilling Date: Sun, 11 Sep 2022 17:29:52 -0500 Subject: [PATCH 8/9] Fix bug with test_cum_time_threshold profiling. --- tests/panels/test_profiling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/panels/test_profiling.py b/tests/panels/test_profiling.py index 3c8eff9ea..2169932b2 100644 --- a/tests/panels/test_profiling.py +++ b/tests/panels/test_profiling.py @@ -33,7 +33,7 @@ def test_insert_content(self): # ensure the panel renders correctly. content = self.panel.content self.assertIn("regular_view", content) - self.assertIn("get_template", content) + self.assertIn("render", content) self.assertValidHTML(content) @override_settings(DEBUG_TOOLBAR_CONFIG={"PROFILER_THRESHOLD_RATIO": 1}) @@ -47,7 +47,7 @@ def test_cum_time_threshold(self): # ensure the panel renders but doesn't include our function. content = self.panel.content self.assertIn("regular_view", content) - self.assertNotIn("get_template", content) + self.assertNotIn("render", content) self.assertValidHTML(content) def test_listcomp_escaped(self): From 9e897c5ec871070a9a0859dbd740a16d33c5b853 Mon Sep 17 00:00:00 2001 From: tschilling Date: Mon, 12 Sep 2022 19:34:20 -0500 Subject: [PATCH 9/9] Include dist-packages in profiling panel docs. --- docs/panels.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/panels.rst b/docs/panels.rst index 92d840d74..09891f2e5 100644 --- a/docs/panels.rst +++ b/docs/panels.rst @@ -133,7 +133,8 @@ with the ``DISABLE_PANELS`` configuration option. The panel will include all function calls made by your project if you're using the setting ``settings.BASE_DIR`` to point to your project's root directory. If a function is in a file within that directory and does not include -``"/site-packages/"`` in the path, it will be included. +``"/site-packages/"`` or ``"/dist-packages/"`` in the path, it will be +included. Third-party panels ------------------