From 3bd6a58032069a149e453fcf14e69614d04461db Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Tue, 23 Apr 2024 23:47:07 -0700 Subject: [PATCH 1/5] gh-118218: Reuse return tuple in itertools.pairwise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this change: ``` b1(1) min=152.8us mean=155.5us ± 3.8us (25 repeats, 1000 loops) b2(1) min=149.0us mean=159.1us ± 8.8us (25 repeats, 1000 loops) b3(1) min=232.6us mean=242.5us ± 11.7us (25 repeats, 1000 loops) b1(10) min=279.2us mean=296.9us ± 16.6us (25 repeats, 1000 loops) b2(10) min=249.5us mean=259.2us ± 12.2us (25 repeats, 1000 loops) b3(10) min=386.6us mean=398.8us ± 10.1us (25 repeats, 1000 loops) b1(1000) min=20.3ms mean=20.7ms ± 0.5ms (25 repeats, 1000 loops) b2(1000) min=16.7ms mean=17.1ms ± 0.2ms (25 repeats, 1000 loops) b3(1000) min=26.0ms mean=26.2ms ± 0.3ms (25 repeats, 1000 loops) ``` Without this change: ``` b1(1) min=142.2us mean=143.0us ± 0.9us (25 repeats, 1000 loops) b2(1) min=142.7us mean=143.3us ± 1.0us (25 repeats, 1000 loops) b3(1) min=219.8us mean=227.2us ± 4.4us (25 repeats, 1000 loops) b1(10) min=314.2us mean=323.8us ± 4.1us (25 repeats, 1000 loops) b2(10) min=335.4us mean=341.8us ± 5.1us (25 repeats, 1000 loops) b3(10) min=362.0us mean=386.2us ± 14.9us (25 repeats, 1000 loops) b1(1000) min=26.5ms mean=27.3ms ± 0.3ms (25 repeats, 1000 loops) b2(1000) min=29.8ms mean=30.2ms ± 0.2ms (25 repeats, 1000 loops) b3(1000) min=26.0ms mean=26.5ms ± 0.4ms (25 repeats, 1000 loops) ``` --- Modules/itertoolsmodule.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6ee447ef6a8cd6..ff2f34a3293de3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -270,6 +270,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject* result; } pairwiseobject; /*[clinic input] @@ -301,6 +302,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -311,6 +317,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -321,6 +328,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -355,8 +363,30 @@ pairwise_next(pairwiseobject *po) Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + Py_XSETREF(po->old, new); Py_DECREF(old); return result; From f653f22fd0796c02dc7cdfab0f22ddd714a86381 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 07:45:12 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst diff --git a/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst new file mode 100644 index 00000000000000..76f7c263d0034a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst @@ -0,0 +1 @@ +Speed up :func:`itertools.pairwise` by up to 1.8x From ea0cfd6759e0a10cc557984ff40446c7fc30e6c0 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 24 Apr 2024 00:48:59 -0700 Subject: [PATCH 3/5] add bpo-42536 style test for gc tracking --- Lib/test/test_itertools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 95e67911db6a7f..e243da309f03d8 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1821,6 +1821,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject From 68b5d27ac61dbf666d7f2079f39cd0700d198c81 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sun, 28 Apr 2024 13:03:53 -0700 Subject: [PATCH 4/5] nit --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ff2f34a3293de3..21ce3ecfad0354 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -270,7 +270,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; - PyObject* result; + PyObject *result; } pairwiseobject; /*[clinic input] From 1bfddd6e793169679a219cf7ccdd3db9dec0b10f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 30 Apr 2024 11:40:03 -0700 Subject: [PATCH 5/5] Update Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst Co-authored-by: Serhiy Storchaka --- .../next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst index 76f7c263d0034a..6d3c54cb485655 100644 --- a/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst +++ b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst @@ -1 +1 @@ -Speed up :func:`itertools.pairwise` by up to 1.8x +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x.