diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py new file mode 100644 index 00000000000000..4eae063eb517ec --- /dev/null +++ b/Lib/test/test_free_threading/test_itertools.py @@ -0,0 +1,42 @@ +import unittest +from threading import Thread + +from test.support import threading_helper + +from itertools import zip_longest + +class PairwiseThreading(unittest.TestCase): + @staticmethod + def work(enum): + while True: + try: + next(enum) + except StopIteration: + break + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_zip_longest(self): + number_of_threads = 8 + number_of_iterations = 40 + n = 200 + enum = zip_longest(range(n), range(2*n)) + for _ in range(number_of_iterations): + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread( + target=self.work, + args=[ + enum, + ], + ) + ) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-08-57-14.gh-issue-123471.p0UQBR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-08-57-14.gh-issue-123471.p0UQBR.rst new file mode 100644 index 00000000000000..a6c66e04d47516 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-08-57-14.gh-issue-123471.p0UQBR.rst @@ -0,0 +1 @@ +Make concurrent iterations over the same :func:`itertools.pairwise` iterator safe under free-threading. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 3f736f0cf19968..5d395378445f9b 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -327,37 +327,54 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) static PyObject * pairwise_next(pairwiseobject *po) { +#ifdef Py_GIL_DISABLED + PyObject *it = Py_XNewRef(po->it); +#else PyObject *it = po->it; - PyObject *old = po->old; - PyObject *new, *result; - +#endif if (it == NULL) { return NULL; } + + PyObject *old = Py_XNewRef(po->old); + PyObject *new, *result; + if (old == NULL) { old = (*Py_TYPE(it)->tp_iternext)(it); - Py_XSETREF(po->old, old); if (old == NULL) { Py_CLEAR(po->it); +#ifdef Py_GIL_DISABLED + Py_DECREF(it); +#endif return NULL; } - it = po->it; - if (it == NULL) { + Py_XSETREF(po->old, Py_NewRef(old)); + if (po->it == NULL) { + // gh-109786: special case for re-entrant calls to pairwise next. the actual behavior is not + // important and this does not avoid any bugs (or does it?) + // the reason for having it is to make the behaviour equal to the python implementation Py_CLEAR(po->old); + Py_DECREF(old); +#ifdef Py_GIL_DISABLED + Py_DECREF(it); +#endif return NULL; } } - Py_INCREF(old); + new = (*Py_TYPE(it)->tp_iternext)(it); if (new == NULL) { Py_CLEAR(po->it); Py_CLEAR(po->old); +#ifdef Py_GIL_DISABLED + Py_DECREF(it); +#endif Py_DECREF(old); return NULL; } result = po->result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); PyObject *last_old = PyTuple_GET_ITEM(result, 0); PyObject *last_new = PyTuple_GET_ITEM(result, 1); @@ -380,7 +397,10 @@ pairwise_next(pairwiseobject *po) } Py_XSETREF(po->old, new); - Py_DECREF(old); + Py_DECREF(old); // instead of the decref here we could borrow the reference above +#ifdef Py_GIL_DISABLED + Py_DECREF(it); +#endif return result; }