[smart_holder] Conversions between Python's native (stdlib) enum types and C++ enums.

.github/workflows/python312.yml

@@ -0,0 +1,142 @@
+name: Python312
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+concurrency:
+  group: python312-${{ github.ref }}
+  cancel-in-progress: false
+
+env:
+  PYTEST_TIMEOUT: 300
+
+jobs:
+  standard:
+    name: "🐍 3.12 latest • ubuntu-latest • x64"
+    runs-on: ubuntu-latest
+
+    # Hint: Select the 'python dev' label in the PR web view.
+    # if: "contains(github.event.pull_request.labels.*.name, 'python dev')"
+
+    steps:
+    - name: Show env
+      run: env
+
+    - uses: actions/checkout@v3
+
+    - name: Setup Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.12-dev"
+
+    - name: Setup Boost
+      run: sudo apt-get install libboost-dev
+
+    - name: Update CMake
+      uses: jwlawson/[email protected]
+
+    - name: Run pip installs
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --prefer-binary -r tests/requirements.txt
+    #   python -m pip install --prefer-binary numpy  # SLOW
+    #   python -m pip install --prefer-binary scipy  # FAILED ~Nov 2022
+
+    - name: Show platform info
+      run: python -m platform
+
+    - name: Show CMake version
+      run: cmake --version
+
+    # FIRST BUILD
+    - name: Configure C++11
+      run: >
+        cmake -S . -B build11
+        -DCMAKE_VERBOSE_MAKEFILE=ON
+        -DPYBIND11_WERROR=ON
+        -DDOWNLOAD_CATCH=ON
+        -DDOWNLOAD_EIGEN=ON
+        -DCMAKE_CXX_STANDARD=11
+        -DCMAKE_BUILD_TYPE=Debug
+
+    - name: Build C++11
+      run: cmake --build build11 -j 2
+
+    - name: Python tests C++11
+      run: cmake --build build11 --target pytest -j 2
+
+    # python-3.12.0-alpha.1-linux-22.04-x64.tar.gz  PASSED
+    # python-3.12.0-alpha.2-linux-22.04-x64.tar.gz  FAILED
+    # python-3.12.0-alpha.3-linux-22.04-x64.tar.gz  FAILED
+    #     free(): invalid pointer
+    #     Custom PyConfig
+    #     /home/runner/work/pybind11/pybind11/tests/test_embed/test_interpreter.cpp:175
+    #     /home/runner/work/pybind11/pybind11/tests/test_embed/test_interpreter.cpp:179: FAILED:
+    # gdb traceback points here:
+    #     https://github.com/python/cpython/blame/54289f85b2af1ecf046089ddf535dda1bdf6af24/Python/import.c#L524
+    # - name: C++ tests C++11
+    #   run: cmake --build build11 --target cpptest -j 2
+
+    - name: Interface test C++11
+      run: cmake --build build11 --target test_cmake_build
+
+    - name: Clean directory
+      run: git clean -fdx
+
+    # SECOND BUILD
+    - name: Configure C++17
+      run: >
+        cmake -S . -B build17
+        -DCMAKE_VERBOSE_MAKEFILE=ON
+        -DPYBIND11_WERROR=ON
+        -DDOWNLOAD_CATCH=ON
+        -DDOWNLOAD_EIGEN=ON
+        -DCMAKE_CXX_STANDARD=17
+        -DCMAKE_BUILD_TYPE=Debug
+
+    - name: Build C++17
+      run: cmake --build build17 -j 2
+
+    - name: Python tests C++17
+      run: cmake --build build17 --target pytest
+
+    # - name: C++ tests C++17
+    #   run: cmake --build build17 --target cpptest
+
+    - name: Interface test C++17
+      run: cmake --build build17 --target test_cmake_build
+
+    - name: Clean directory
+      run: git clean -fdx
+
+    # THIRD BUILD
+    - name: Configure C++17 max DPYBIND11_INTERNALS_VERSION
+      run: >
+        cmake -S . -B build17max
+        -DCMAKE_VERBOSE_MAKEFILE=ON
+        -DPYBIND11_WERROR=ON
+        -DDOWNLOAD_CATCH=ON
+        -DDOWNLOAD_EIGEN=ON
+        -DCMAKE_CXX_STANDARD=17
+        -DCMAKE_BUILD_TYPE=Debug
+        -DPYBIND11_INTERNALS_VERSION=10000000
+
+    - name: Build C++17 max DPYBIND11_INTERNALS_VERSION
+      run: cmake --build build17max -j 2
+
+    - name: Python tests C++17 max DPYBIND11_INTERNALS_VERSION
+      run: cmake --build build17max --target pytest
+
+    # - name: C++ tests C++17 max DPYBIND11_INTERNALS_VERSION
+    #   run: cmake --build build17max --target cpptest
+
+    - name: Interface test C++17 max DPYBIND11_INTERNALS_VERSION
+      run: cmake --build build17max --target test_cmake_build
+
+    # Ensure the setup_helpers module can build packages using setuptools
+    - name: Setuptools helpers test
+      run: pytest tests/extra_setuptools
+
+    - name: Clean directory
+      run: git clean -fdx

CMakeLists.txt

@@ -111,17 +111,21 @@ cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF
 
 # NB: when adding a header don't forget to also add it to setup.py
 set(PYBIND11_HEADERS
+    include/pybind11/detail/abi_platform_id.h
     include/pybind11/detail/class.h
     include/pybind11/detail/common.h
+    include/pybind11/detail/cross_extension_shared_state.h
     include/pybind11/detail/descr.h
     include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
+    include/pybind11/detail/native_enum_data.h
     include/pybind11/detail/smart_holder_poc.h
     include/pybind11/detail/smart_holder_sfinae_hooks_only.h
     include/pybind11/detail/smart_holder_type_casters.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/type_caster_odr_guard.h
+    include/pybind11/detail/type_map.h
     include/pybind11/detail/typeid.h
     include/pybind11/attr.h
     include/pybind11/buffer_info.h
@@ -138,6 +142,7 @@ set(PYBIND11_HEADERS
     include/pybind11/gil.h
     include/pybind11/iostream.h
     include/pybind11/functional.h
+    include/pybind11/native_enum.h
     include/pybind11/numpy.h
     include/pybind11/operators.h
     include/pybind11/pybind11.h

include/pybind11/cast.h

@@ -12,6 +12,7 @@
 
 #include "detail/common.h"
 #include "detail/descr.h"
+#include "detail/native_enum_data.h"
 #include "detail/smart_holder_sfinae_hooks_only.h"
 #include "detail/type_caster_base.h"
 #include "detail/type_caster_odr_guard.h"
@@ -63,12 +64,6 @@ using make_caster_for_intrinsic = type_caster<type>;
 template <typename type>
 using make_caster = make_caster_for_intrinsic<intrinsic_t<type>>;
 
-template <typename T>
-struct type_uses_smart_holder_type_caster {
-    static constexpr bool value
-        = std::is_base_of<smart_holder_type_caster_base_tag, make_caster<T>>::value;
-};
-
 // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T
 template <typename T>
 typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) {
@@ -81,6 +76,126 @@ cast_op(make_caster<T> &&caster) {
         template cast_op_type<typename std::add_rvalue_reference<T>::type>();
 }
 
+template <typename EnumType>
+class type_caster_enum_type {
+private:
+    using Underlying = typename std::underlying_type<EnumType>::type;
+
+public:
+    static constexpr auto name = const_name<EnumType>();
+
+    template <typename SrcType>
+    static handle cast(SrcType &&src, return_value_policy, handle parent) {
+        auto const &natives = cross_extension_shared_states::native_enum_type_map::get();
+        auto found = natives.find(std::type_index(typeid(EnumType)));
+        if (found != natives.end()) {
+            return handle(found->second)(static_cast<Underlying>(src)).release();
+        }
+        return type_caster_for_class_<EnumType>::cast(
+            std::forward<SrcType>(src),
+            // Fixes https://github.com/pybind/pybind11/pull/3643#issuecomment-1022987818:
+            return_value_policy::copy,
+            parent);
+    }
+
+    bool load(handle src, bool convert) {
+        auto const &natives = cross_extension_shared_states::native_enum_type_map::get();
+        auto found = natives.find(std::type_index(typeid(EnumType)));
+        if (found != natives.end()) {
+            if (!isinstance(src, found->second)) {
+                return false;
+            }
+            type_caster<Underlying> underlying_caster;
+            if (!underlying_caster.load(src.attr("value"), convert)) {
+                pybind11_fail("native_enum internal consistency failure.");
+            }
+            value = static_cast<EnumType>(static_cast<Underlying>(underlying_caster));
+            return true;
+        }
+        if (!pybind11_enum_) {
+            pybind11_enum_.reset(new type_caster_for_class_<EnumType>());
+        }
+        return pybind11_enum_->load(src, convert);
+    }
+
+    template <typename T>
+    using cast_op_type = detail::cast_op_type<T>;
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator EnumType *() {
+        if (!pybind11_enum_) {
+            return &value;
+        }
+        return pybind11_enum_->operator EnumType *();
+    }
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator EnumType &() {
+        if (!pybind11_enum_) {
+            return value;
+        }
+        return pybind11_enum_->operator EnumType &();
+    }
+
+private:
+    std::unique_ptr<type_caster_for_class_<EnumType>> pybind11_enum_;
+    EnumType value;
+};
+
+template <typename EnumType, typename SFINAE = void>
+struct type_caster_enum_type_enabled : std::true_type {};
+
+template <typename T>
+struct type_uses_type_caster_enum_type {
+    static constexpr bool value
+        = std::is_enum<T>::value && type_caster_enum_type_enabled<T>::value;
+};
+
+template <typename EnumType>
+class type_caster<EnumType, detail::enable_if_t<type_uses_type_caster_enum_type<EnumType>::value>>
+    : public type_caster_enum_type<EnumType> {};
+
+template <typename T>
+struct type_uses_smart_holder_type_caster {
+    static constexpr bool value
+        = std::is_base_of<smart_holder_type_caster_base_tag, make_caster<T>>::value
+#ifdef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT
+          || type_uses_type_caster_enum_type<T>::value
+#endif
+        ;
+};
+
+template <typename T, typename SFINAE = void>
+struct type_caster_classh_enum_aware : type_caster<T> {};
+
+#ifdef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT
+template <typename EnumType>
+struct type_caster_classh_enum_aware<
+    EnumType,
+    detail::enable_if_t<type_uses_type_caster_enum_type<EnumType>::value>>
+    : type_caster_for_class_<EnumType> {};
+#endif
+
+template <typename T, detail::enable_if_t<std::is_enum<T>::value, int> = 0>
+bool isinstance_native_enum_impl(handle obj, const std::type_info &tp) {
+    auto const &natives = cross_extension_shared_states::native_enum_type_map::get();
+    auto found = natives.find(tp);
+    if (found == natives.end()) {
+        return false;
+    }
+    return isinstance(obj, found->second);
+}
+
+template <typename T, detail::enable_if_t<!std::is_enum<T>::value, int> = 0>
+bool isinstance_native_enum_impl(handle, const std::type_info &) {
+    return false;
+}
+
+template <typename T>
+bool isinstance_native_enum(handle obj, const std::type_info &tp) {
+    return isinstance_native_enum_impl<intrinsic_t<T>>(obj, tp);
+}
+
 template <typename type>
 class type_caster<std::reference_wrapper<type>> {
 private:
@@ -1024,11 +1139,11 @@ using move_never = none_of<move_always<T>, move_if_unreferenced<T>>;
 // non-reference/pointer `type`s and reference/pointers from a type_caster_generic are safe;
 // everything else returns a reference/pointer to a local variable.
 template <typename type>
-using cast_is_temporary_value_reference
-    = bool_constant<(std::is_reference<type>::value || std::is_pointer<type>::value)
-                    && !std::is_base_of<type_caster_generic, make_caster<type>>::value
-                    && !type_uses_smart_holder_type_caster<intrinsic_t<type>>::value
-                    && !std::is_same<intrinsic_t<type>, void>::value>;
+using cast_is_temporary_value_reference = bool_constant<
+    (std::is_reference<type>::value || std::is_pointer<type>::value)
+    && !std::is_base_of<type_caster_generic, make_caster<type>>::value
+    && !std::is_base_of<smart_holder_type_caster_base_tag, make_caster<type>>::value
+    && !std::is_same<intrinsic_t<type>, void>::value>;
 
 // When a value returned from a C++ function is being cast back to Python, we almost always want to
 // force `policy = move`, regardless of the return value policy the function/method was declared
@@ -1086,8 +1201,18 @@ PYBIND11_NAMESPACE_END(detail)
 template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0>
 T cast(const handle &handle) {
     using namespace detail;
-    static_assert(!cast_is_temporary_value_reference<T>::value,
+    constexpr bool is_enum_cast = type_uses_type_caster_enum_type<intrinsic_t<T>>::value;
+    static_assert(!cast_is_temporary_value_reference<T>::value || is_enum_cast,
                   "Unable to cast type to reference: value is local to type caster");
+#ifndef NDEBUG
+    if (is_enum_cast && cast_is_temporary_value_reference<T>::value) {
+        if (cross_extension_shared_states::native_enum_type_map::get().count(
+                std::type_index(typeid(intrinsic_t<T>)))
+            != 0) {
+            pybind11_fail("Unable to cast native enum type to reference");
+        }
+    }
+#endif
     return cast_op<T>(load_type<T>(handle));
 }