gh-105201: Add PyIter_NextItem()

Doc/c-api/iter.rst

@@ -10,7 +10,8 @@ There are two functions specifically for working with iterators.
 .. c:function:: int PyIter_Check(PyObject *o)
 
    Return non-zero if the object *o* can be safely passed to
-   :c:func:`PyIter_Next`, and ``0`` otherwise.  This function always succeeds.
+   :c:func:`PyIter_NextItem` and ``0`` otherwise.
+   This function always succeeds.
 
 .. c:function:: int PyAIter_Check(PyObject *o)
 
@@ -19,40 +20,41 @@ There are two functions specifically for working with iterators.
 
    .. versionadded:: 3.10
 
-.. c:function:: PyObject* PyIter_Next(PyObject *o)
-
-   Return the next value from the iterator *o*.  The object must be an iterator
-   according to :c:func:`PyIter_Check` (it is up to the caller to check this).
-   If there are no remaining values, returns ``NULL`` with no exception set.
-   If an error occurs while retrieving the item, returns ``NULL`` and passes
-   along the exception.
+.. c:function:: int PyIter_NextItem(PyObject *iter, PyObject **item)
 
-To write a loop which iterates over an iterator, the C code should look
-something like this::
+   Return ``1`` and set *item* to a :term:`strong reference` of the
+   next value of the iterator *iter* on success.
+   Return ``0`` and set *item* to ``NULL`` if there are no remaining values.
+   Return ``-1``, set *item* to ``NULL`` and set an exception on error.
 
-   PyObject *iterator = PyObject_GetIter(obj);
-   PyObject *item;
+   Iterate over *iter* using the following pattern::
 
-   if (iterator == NULL) {
-       /* propagate error */
-   }
+      PyObject *iter = PyObject_GetIter(obj);
+      if (iter == NULL) {
+          goto error;
+      }
 
-   while ((item = PyIter_Next(iterator))) {
-       /* do something with item */
-       ...
-       /* release reference when done */
-       Py_DECREF(item);
-   }
+      PyObject *item = NULL;
+      while (PyIter_NextItem(iter, &item)) {
+          if (item == NULL) {
+              goto error;
+          }
+          do_something(item);
+          Py_DECREF(item);
+      }
+      Py_DECREF(iter);
 
-   Py_DECREF(iterator);
+.. c:function:: PyObject* PyIter_Next(PyObject *o)
 
-   if (PyErr_Occurred()) {
-       /* propagate error */
-   }
-   else {
-       /* continue doing useful work */
-   }
+   This is an older version of :c:func:`!PyIter_NextItem`,
+   which is retained for backwards compatibility.
+   Prefer :c:func:`PyIter_NextItem`.
 
+   Return the next value from the iterator *o*.  The object must be an iterator
+   according to :c:func:`PyIter_Check` (it is up to the caller to check this).
+   If there are no remaining values, returns ``NULL`` with no exception set.
+   If an error occurs while retrieving the item, returns ``NULL`` and passes
+   along the exception.
 
 .. c:type:: PySendResult
 

Doc/data/refcounts.dat

@@ -1136,6 +1136,10 @@ PyAIter_Check:PyObject*:o:0:
 PyIter_Next:PyObject*::+1:
 PyIter_Next:PyObject*:o:0:
 
+PyIter_NextIter:int:::
+PyIter_NextIter:PyObject*:iter:0:
+PyIter_NextIter:PyObject**:item:+1:
+
 PyIter_Send:int:::
 PyIter_Send:PyObject*:iter:0:
 PyIter_Send:PyObject*:arg:0:

Doc/whatsnew/3.14.rst

@@ -397,6 +397,10 @@ New Features
 
   (Contributed by Victor Stinner in :gh:`119182`.)
 
+* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`,
+  which has an ambiguous return value.
+  (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.)
+
 Porting to Python 3.14
 ----------------------
 

Include/abstract.h

@@ -397,6 +397,12 @@ PyAPI_FUNC(int) PyIter_Check(PyObject *);
    This function always succeeds. */
 PyAPI_FUNC(int) PyAIter_Check(PyObject *);
 
+/* Return 1 and set 'item' to the next item of iter on success.
+ * Return 0 and set 'item' to NULL when there are no remaining values.
+ * Return -1, set 'item' to NULL and set an exception on error.
+ */
+PyAPI_FUNC(int) PyIter_NextItem(PyObject *iter, PyObject **item);
+
 /* Takes an iterator object and calls its tp_iternext slot,
    returning the next value.
 

Lib/test/test_capi/test_misc.py

@@ -401,6 +401,41 @@ def check_negative_refcount(self, code):
                          br'_Py_NegativeRefcount: Assertion failed: '
                          br'object has negative ref count')
 
+    def run_iter_api_test(self, next_func):
+        inputs = [ (), (1,2,3),
+                   [], [1,2,3]]
+
+        for inp in inputs:
+            items = []
+            it = iter(inp)
+            while (item := next_func(it)) is not None:
+                items.append(item)
+            self.assertEqual(items, list(inp))
+
+        class Broken:
+            def __init__(self):
+                self.count = 0
+
+            def __next__(self):
+                if self.count < 3:
+                    self.count += 1
+                    return self.count
+                else:
+                    raise TypeError('bad type')
+
+        it = Broken()
+        self.assertEqual(next_func(it), 1)
+        self.assertEqual(next_func(it), 2)
+        self.assertEqual(next_func(it), 3)
+        with self.assertRaisesRegex(TypeError, 'bad type'):
+            next_func(it)
+
+    def test_iter_next(self):
+        self.run_iter_api_test(_testcapi.PyIter_Next)
+
+    def test_iter_nextitem(self):
+        self.run_iter_api_test(_testcapi.PyIter_NextItem)
+
     @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'),
                          'need _testcapi.negative_refcount()')
     def test_negative_refcount(self):

Misc/NEWS.d/next/C_API/2024-07-27-00-28-35.gh-issue-105201.0-xUWq.rst

@@ -0,0 +1,2 @@
+Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, which has an
+ambiguous return value. Patch by Irit Katriel and Erlend Aasland.

Misc/stable_abi.toml

@@ -2508,3 +2508,5 @@
 
 [function.Py_TYPE]
     added = '3.14'
+[function.PyIter_NextItem]
+    added = '3.15'

Modules/_testcapimodule.c

@@ -281,6 +281,44 @@ test_dict_iteration(PyObject* self, PyObject *Py_UNUSED(ignored))
     Py_RETURN_NONE;
 }
 
+static PyObject *
+pyiter_next(PyObject *self, PyObject *args)
+{
+    PyObject *iter;
+    if (!PyArg_ParseTuple(args, "O:pyiter_next", &iter)) {
+        return NULL;
+    }
+    assert(PyIter_Check(iter) || PyAIter_Check(iter));
+    PyObject *item = PyIter_Next(iter);
+    if (item == NULL && !PyErr_Occurred()) {
+        Py_RETURN_NONE;
+    }
+    return item;
+}
+
+static PyObject *
+pyiter_nextitem(PyObject *self, PyObject *args)
+{
+    PyObject *iter;
+    if (!PyArg_ParseTuple(args, "O:pyiter_nextitem", &iter)) {
+        return NULL;
+    }
+    assert(PyIter_Check(iter) || PyAIter_Check(iter));
+    PyObject *item;
+    int rc = PyIter_NextItem(iter, &item);
+    if (rc < 0) {
+        assert(PyErr_Occurred());
+        assert(item == NULL);
+        return NULL;
+    }
+    assert(!PyErr_Occurred());
+    if (item == NULL) {
+        Py_RETURN_NONE;
+    }
+    return item;
+}
+
+
 /* Issue #4701: Check that PyObject_Hash implicitly calls
  *   PyType_Ready if it hasn't already been called
  */
@@ -3483,6 +3521,9 @@ static PyMethodDef TestMethods[] = {
     {"test_weakref_capi", test_weakref_capi, METH_NOARGS},
     {"function_set_warning", function_set_warning, METH_NOARGS},
     {"test_critical_sections", test_critical_sections, METH_NOARGS},
+
+    {"PyIter_Next", pyiter_next, METH_VARARGS},
+    {"PyIter_NextItem", pyiter_nextitem, METH_VARARGS},
     {NULL, NULL} /* sentinel */
 };
 

Objects/abstract.c

@@ -2881,7 +2881,47 @@ PyAIter_Check(PyObject *obj)
             tp->tp_as_async->am_anext != &_PyObject_NextNotImplemented);
 }
 
+static int
+iternext(PyObject *iter, PyObject **item)
+{
+    *item = (*Py_TYPE(iter)->tp_iternext)(iter);
+    if (*item == NULL) {
+        PyThreadState *tstate = _PyThreadState_GET();
+        /* When the iterator is exhausted it must return NULL;
+         * a StopIteration exception may or may not be set. */
+        if (!_PyErr_Occurred(tstate)) {
+            return 0;
+        }
+        if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
+            _PyErr_Clear(tstate);
+            return 0;
+        }
+        return -1;
+    }
+    return 1;
+}
+
+/* Return 1 and set 'item' to the next item of iter on success.
+ * Return 0 and set 'item' to NULL when there are no remaining values.
+ * Return -1, set 'item' to NULL and set an exception on error.
+ */
+int
+PyIter_NextItem(PyObject *iter, PyObject **item)
+{
+    assert(iter != NULL);
+    assert(item != NULL);
+
+    if (!PyIter_Check(iter) && !PyAIter_Check(iter)) {
+        PyErr_Format(PyExc_TypeError, "expected an iterator, not '%T'", iter);
+        return -1;
+    }
+
+    return iternext(iter, item);
+}
+
 /* Return next item.
+ * Deprecated; use PyIter_NextItem() instead.
+ *
  * If an error occurs, return NULL.  PyErr_Occurred() will be true.
  * If the iteration terminates normally, return NULL and clear the
  * PyExc_StopIteration exception (if it was set).  PyErr_Occurred()
@@ -2891,17 +2931,9 @@ PyAIter_Check(PyObject *obj)
 PyObject *
 PyIter_Next(PyObject *iter)
 {
-    PyObject *result;
-    result = (*Py_TYPE(iter)->tp_iternext)(iter);
-    if (result == NULL) {
-        PyThreadState *tstate = _PyThreadState_GET();
-        if (_PyErr_Occurred(tstate)
-            && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
-        {
-            _PyErr_Clear(tstate);
-        }
-    }
-    return result;
+    PyObject *item;
+    (void)iternext(iter, &item);
+    return item;
 }
 
 PySendResult