Change return types of py.None(), py.NotImplemented() and py.Ellipsis() to typed singletons

Author: davidhewitt

guide/src/class/protocols.md

@@ -103,7 +103,7 @@ given signatures should be interpreted as follows:
             match op {
                 CompareOp::Eq => (self.0 == other.0).into_py(py),
                 CompareOp::Ne => (self.0 != other.0).into_py(py),
-                _ => py.NotImplemented(),
+                _ => py.NotImplemented().into(),
             }
         }
     }

guide/src/migration.md

@@ -3,6 +3,42 @@
 This guide can help you upgrade code through breaking changes from one PyO3 version to the next.
 For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
+## from 0.20.* to 0.21
+
+### `py.None()`, `py.NotImplemented()` and `py.Ellipsis()` now return typed singletons
+
+Previously `py.None()`, `py.NotImplemented()` and `py.Ellipsis()` would return `PyObject`. This had a few downsides:
+ - `PyObject` does not carry static type information
+ - `PyObject` takes ownership of a reference to the singletons, adding refcounting performance overhead
+ - `PyObject` is not gil-bound, meaning follow up method calls might again need `py`, causing repetition
+
+To avoid these downsides, these methods now return typed gil-bound references to the singletons, e.g. `py.None()` returns `&PyNone`. These typed singletons all implement `Into<PyObject>`, so migration is straightforward.
+
+Before:
+
+```rust,compile_fail
+# use pyo3::prelude::*;
+Python::with_gil(|py| {
+    let a: PyObject = py.None();
+
+    let b: &PyAny = py.None().as_ref(py);  // or into_ref(py)
+});
+```
+
+After:
+
+```rust
+# use pyo3::prelude::*;
+Python::with_gil(|py| {
+    // For uses needing a PyObject, add `.into()`
+    let a: PyObject = py.None().into();
+
+    // For uses needing &PyAny, remove `.as_ref(py)`
+    let b: &PyAny = py.None();
+});
+```
+
+
 ## from 0.19.* to 0.20
 
 ### Drop support for older technologies
@@ -158,7 +194,7 @@ fn raise_err() -> anyhow::Result<()> {
     Err(PyValueError::new_err("original error message").into())
 }
 
-fn main() {
+# fn main() {
     Python::with_gil(|py| {
         let rs_func = wrap_pyfunction!(raise_err, py).unwrap();
         pyo3::py_run!(
@@ -936,14 +972,14 @@ ensure that the Python GIL was held by the current thread). Technically, this wa
 To migrate, just pass a `py` argument to any calls to these methods.
 
 Before:
-```rust,compile_fail
+```rust,ignore
 # pyo3::Python::with_gil(|py| {
 py.None().get_refcnt();
 # })
 ```
 
 After:
-```rust
+```rust,compile_fail
 # pyo3::Python::with_gil(|py| {
 py.None().get_refcnt(py);
 # })

newsfragments/3578.changed.md

@@ -0,0 +1 @@
+Change return type of `py.None()`, `py.NotImplemented()`, and `py.Ellipsis()` from `PyObject` to typed singletons (`&PyNone`, `&PyNotImplemented` and `PyEllipsis` respectively).

pyo3-benches/benches/bench_gil.rs

@@ -16,7 +16,7 @@ fn bench_clean_acquire_gil(b: &mut Bencher<'_>) {
 }
 
 fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) {
-    let obj = Python::with_gil(|py| py.None());
+    let obj: PyObject = Python::with_gil(|py| py.None().into());
     b.iter_batched(
         || {
             // Clone and drop an object so that the GILPool has work to do.

pyo3-benches/benches/bench_pyobject.rs

@@ -6,7 +6,7 @@ fn drop_many_objects(b: &mut Bencher<'_>) {
     Python::with_gil(|py| {
         b.iter(|| {
             for _ in 0..1000 {
-                std::mem::drop(py.None());
+                std::mem::drop(PyObject::from(py.None()));
             }
         });
     });

pyo3-macros-backend/src/pyclass.rs

@@ -587,7 +587,7 @@ fn impl_enum(
                             return Ok((self_val == other.__pyo3__int__()).to_object(py));
                         }
 
-                        return Ok(py.NotImplemented());
+                        return Ok(::std::convert::Into::into(py.NotImplemented()));
                     }
                     _pyo3::basic::CompareOp::Ne => {
                         let self_val = self.__pyo3__int__();
@@ -598,9 +598,9 @@ fn impl_enum(
                             return Ok((self_val != other.__pyo3__int__()).to_object(py));
                         }
 
-                        return Ok(py.NotImplemented());
+                        return Ok(::std::convert::Into::into(py.NotImplemented()));
                     }
-                    _ => Ok(py.NotImplemented()),
+                    _ => Ok(::std::convert::Into::into(py.NotImplemented())),
                 }
             }
         };

pytests/src/awaitable.rs

@@ -36,7 +36,7 @@ impl IterAwaitable {
                 Ok(v) => Ok(IterNextOutput::Return(v)),
                 Err(err) => Err(err),
             },
-            _ => Ok(IterNextOutput::Yield(py.None())),
+            _ => Ok(IterNextOutput::Yield(py.None().into())),
         }
     }
 }

src/conversion.rs

@@ -141,7 +141,7 @@ pub trait ToPyObject {
 ///         match self {
 ///             Self::Integer(val) => val.into_py(py),
 ///             Self::String(val) => val.into_py(py),
-///             Self::None => py.None(),
+///             Self::None => py.None().into(),
 ///         }
 ///     }
 /// }
@@ -251,7 +251,7 @@ where
 {
     fn to_object(&self, py: Python<'_>) -> PyObject {
         self.as_ref()
-            .map_or_else(|| py.None(), |val| val.to_object(py))
+            .map_or_else(|| py.None().into(), |val| val.to_object(py))
     }
 }
 
@@ -260,7 +260,7 @@ where
     T: IntoPy<PyObject>,
 {
     fn into_py(self, py: Python<'_>) -> PyObject {
-        self.map_or_else(|| py.None(), |val| val.into_py(py))
+        self.map_or_else(|| py.None().into(), |val| val.into_py(py))
     }
 }
 
@@ -622,13 +622,13 @@ mod tests {
             assert_eq!(option.as_ptr(), std::ptr::null_mut());
 
             let none = py.None();
-            option = Some(none.clone());
+            option = Some(none.into());
 
-            let ref_cnt = none.get_refcnt(py);
+            let ref_cnt = none.get_refcnt();
             assert_eq!(option.as_ptr(), none.as_ptr());
 
             // Ensure ref count not changed by as_ptr call
-            assert_eq!(none.get_refcnt(py), ref_cnt);
+            assert_eq!(none.get_refcnt(), ref_cnt);
         });
     }
 }

src/err/mod.rs

@@ -181,7 +181,7 @@ impl PyErr {
         } else {
             // Assume obj is Type[Exception]; let later normalization handle if this
             // is not the case
-            PyErrState::lazy(obj, obj.py().None())
+            PyErrState::lazy(obj, Option::<PyObject>::None)
         };
 
         PyErr::from_state(state)

src/ffi/tests.rs

@@ -314,15 +314,15 @@ fn test_inc_dec_ref_immortal() {
     Python::with_gil(|py| {
         let obj = py.None();
 
-        let ref_count = obj.get_refcnt(py);
+        let ref_count = obj.get_refcnt();
         let ptr = obj.as_ptr();
 
         unsafe { Py_INCREF(ptr) };
 
-        assert_eq!(obj.get_refcnt(py), ref_count);
+        assert_eq!(obj.get_refcnt(), ref_count);
 
         unsafe { Py_DECREF(ptr) };
 
-        assert_eq!(obj.get_refcnt(py), ref_count);
+        assert_eq!(obj.get_refcnt(), ref_count);
     })
 }