Merge branches 'eigen-numpy-referencing', 'unicode-fixes-and-docs' and 'args-kwargs-posargs-mixing'

Author: jagerman

docs/advanced/cast/eigen.rst

@@ -223,6 +223,51 @@ consideration is required: the created numpy array will have the required
 stride that allows numpy to properly interpret the array, whatever its storage
 order.
 
+Failing rather than copying
+===========================
+
+The default behaviour when binding ``Eigen::Ref<const MatrixType>`` eigen
+references is to copy matrix values when passed a numpy array that does not
+conform to the element type of ``MatrixType`` or does not have a compatible
+stride layout.  If you want to explicitly avoid copying in such a case,
+pybind11 provides a special ``py::EigenNoCopyRef<const Matrix>`` class that
+works exactly like ``Eigen::Ref<const Matrix>``, but lets pybind11 know that it
+should fail to accept incompatible data rather than performing a conversion.
+
+In order to use this in binding code, it is typically required to provide an
+intermediate lambda function that accepts the special class then simply calls
+the actual ``Eigen::Ref<const Matrix>``-accepting function.  The following example
+shows how to do this for both a function and instance method:
+
+.. code-block:: cpp
+
+    // The method and function to be bound:
+    class MyClass {
+        // ...
+        double some_method(const Eigen::Ref<const MatrixXd> &matrix) { /* ... */ }
+    };
+    float some_function(const Eigen::Ref<const Matrix4f> &matrix) { /* ... */ }
+
+    // The associated binding code:
+    py::class_<MyClass>(m, "MyClass")
+        // ... other class definitions
+        .def("some_method", [](MyClass &cl, py::EigenNoCopyRef<const MatrixXd> &matrix) {
+            cl.some_method(matrix);
+        });
+    m.def("some_function", [](py::EigenNoCopyRef<const Matrix4f> &matrix) {
+        return some_function(matrix);
+    });
+
+With the above binding code, attempting to call the the ``some_method(m)``
+method on a ``MyClass`` object, or attempting to call ``some_function(m)`` will
+raise a ``RuntimeError`` rather than making a temporary copy of the array.
+This wrapper class may only be used with non-mutable references (generally,
+references with a ``const`` matrix type): mutable references such as
+``Eigen::Ref<Eigen::MatrixXd>`` are never copied.
+
+pybind11 also provides a ``py::EigenNoCopyDRef`` which is the no-copying
+version of ``py::EigenDRef``.
+
 Vectors versus column/row matrices
 ==================================
 

docs/advanced/cast/overview.rst

@@ -94,14 +94,26 @@ as arguments and return values, refer to the section on binding :ref:`classes`.
 +------------------------------------+---------------------------+-------------------------------+
 | ``char``                           | Character literal         | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
+| ``char16_t``                       | UTF-16 character literal  | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
+| ``char32_t``                       | UTF-32 character literal  | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
 | ``wchar_t``                        | Wide character literal    | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
 | ``const char *``                   | UTF-8 string literal      | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
+| ``const char16_t *``               | UTF-16 string literal     | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
+| ``const char32_t *``               | UTF-32 string literal     | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
 | ``const wchar_t *``                | Wide string literal       | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
 | ``std::string``                    | STL dynamic UTF-8 string  | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
+| ``std::u16string``                 | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
+| ``std::u32string``                 | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
 | ``std::wstring``                   | STL dynamic wide string   | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
 | ``std::pair<T1, T2>``              | Pair of two custom types  | :file:`pybind11/pybind11.h`   |

include/pybind11/attr.h

@@ -64,7 +64,8 @@ struct undefined_t;
 template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_;
 template <typename... Args> struct init;
 template <typename... Args> struct init_alias;
-inline void keep_alive_impl(size_t Nurse, size_t Patient, function_arguments args, handle ret);
+struct function_call;
+inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret);
 
 /// Internal data structure which holds metadata about a keyword argument
 struct argument_record {
@@ -95,7 +96,7 @@ struct function_record {
     std::vector<argument_record> args;
 
     /// Pointer to lambda function which converts arguments and performs the actual call
-    handle (*impl) (function_record *, function_arguments, handle) = nullptr;
+    handle (*impl) (function_call &) = nullptr;
 
     /// Storage for the wrapped function pointer and captured data, if any
     void *data[3] = { };
@@ -204,6 +205,22 @@ struct type_record {
     }
 };
 
+/// Internal data associated with a single function call
+struct function_call {
+    function_call(const function_record &f, handle p) : func{f}, parent(p) {
+        args.reserve(f.nargs);
+    }
+
+    /// The function data:
+    const function_record &func;
+
+    /// Arguments passed to the function:
+    std::vector<handle> args;
+
+    /// The parent, if any
+    handle parent;
+};
+
 /**
  * Partial template specializations to process custom attributes provided to
  * cpp_function_ and class_. These are either used to initialize the respective
@@ -216,8 +233,8 @@ template <typename T> struct process_attribute_default {
     /// Default implementation: do nothing
     static void init(const T &, function_record *) { }
     static void init(const T &, type_record *) { }
-    static void precall(function_arguments) { }
-    static void postcall(function_arguments, handle) { }
+    static void precall(function_call &) { }
+    static void postcall(function_call &, handle) { }
 };
 
 /// Process an attribute specifying the function's name
@@ -345,13 +362,13 @@ struct process_attribute<arithmetic> : process_attribute_default<arithmetic> {};
  */
 template <size_t Nurse, size_t Patient> struct process_attribute<keep_alive<Nurse, Patient>> : public process_attribute_default<keep_alive<Nurse, Patient>> {
     template <size_t N = Nurse, size_t P = Patient, enable_if_t<N != 0 && P != 0, int> = 0>
-    static void precall(function_arguments args) { keep_alive_impl(Nurse, Patient, args, handle()); }
+    static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); }
     template <size_t N = Nurse, size_t P = Patient, enable_if_t<N != 0 && P != 0, int> = 0>
-    static void postcall(function_arguments, handle) { }
+    static void postcall(function_call &, handle) { }
     template <size_t N = Nurse, size_t P = Patient, enable_if_t<N == 0 || P == 0, int> = 0>
-    static void precall(function_arguments) { }
+    static void precall(function_call &) { }
     template <size_t N = Nurse, size_t P = Patient, enable_if_t<N == 0 || P == 0, int> = 0>
-    static void postcall(function_arguments args, handle ret) { keep_alive_impl(Nurse, Patient, args, ret); }
+    static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); }
 };
 
 /// Recursively iterate over variadic template arguments
@@ -364,12 +381,12 @@ template <typename... Args> struct process_attributes {
         int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::init(args, r), 0) ... };
         ignore_unused(unused);
     }
-    static void precall(function_arguments fn_args) {
-        int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(fn_args), 0) ... };
+    static void precall(function_call &call) {
+        int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(call), 0) ... };
         ignore_unused(unused);
     }
-    static void postcall(function_arguments fn_args, handle fn_ret) {
-        int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(fn_args, fn_ret), 0) ... };
+    static void postcall(function_call &call, handle fn_ret) {
+        int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(call, fn_ret), 0) ... };
         ignore_unused(unused);
     }
 };

include/pybind11/cast.h

@@ -462,8 +462,15 @@ template <typename type> class type_caster<std::reference_wrapper<type>> : publi
         template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>
 
 
+template <typename CharT> using is_std_char_type = any_of<
+    std::is_same<CharT, char>, /* std::string */
+    std::is_same<CharT, char16_t>, /* std::u16string */
+    std::is_same<CharT, char32_t>, /* std::u32string */
+    std::is_same<CharT, wchar_t> /* std::wstring */
+>;
+
 template <typename T>
-struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value>> {
+struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value && !is_std_char_type<T>::value>> {
     typedef typename std::conditional<sizeof(T) <= sizeof(long), long, long long>::type _py_type_0;
     typedef typename std::conditional<std::is_signed<T>::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>::type _py_type_1;
     typedef typename std::conditional<std::is_floating_point<T>::value, double, _py_type_1>::type py_type;
@@ -604,49 +611,22 @@ template <> class type_caster<bool> {
     PYBIND11_TYPE_CASTER(bool, _("bool"));
 };
 
-template <> class type_caster<std::string> {
-public:
-    bool load(handle src, bool) {
-        object temp;
-        handle load_src = src;
-        if (!src) {
-            return false;
-        } else if (PyUnicode_Check(load_src.ptr())) {
-            temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(load_src.ptr()));
-            if (!temp) { PyErr_Clear(); return false; }  // UnicodeEncodeError
-            load_src = temp;
-        }
-        char *buffer;
-        ssize_t length;
-        int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(load_src.ptr(), &buffer, &length);
-        if (err == -1) { PyErr_Clear(); return false; }  // TypeError
-        value = std::string(buffer, (size_t) length);
-        success = true;
-        return true;
-    }
+// Helper class for UTF-{8,16,32} strings:
+template <typename CharT, class Traits, class Allocator>
+struct type_caster<std::basic_string<CharT, Traits, Allocator>, enable_if_t<is_std_char_type<CharT>::value>> {
+    static constexpr unsigned int UTF_N =
+        std::is_same<CharT, char>::value     ? 8 :
+        std::is_same<CharT, char16_t>::value ? 16 :
+        std::is_same<CharT, char32_t>::value ? 32 :
+        (sizeof(CharT) == 2 ? 16 : 32); /* std::wstring is UTF-16 on Windows, UTF-32 everywhere else */
 
-    static handle cast(const std::string &src, return_value_policy /* policy */, handle /* parent */) {
-        return PyUnicode_FromStringAndSize(src.c_str(), (ssize_t) src.length());
-    }
+    // C++ only requires char/char16_t/char32_t to be at least 8/16/32 bits, but Python's encoding
+    // assumes exactly 1/2/4 bytes:
+    static_assert(sizeof(CharT) == UTF_N / 8,
+            "Internal error: string type_caster requires 1/2/4-sized character types");
 
-    PYBIND11_TYPE_CASTER(std::string, _(PYBIND11_STRING_NAME));
-protected:
-    bool success = false;
-};
+    using StringType = std::basic_string<CharT, Traits, Allocator>;
 
-template <typename type, typename deleter> class type_caster<std::unique_ptr<type, deleter>> {
-public:
-    static handle cast(std::unique_ptr<type, deleter> &&src, return_value_policy policy, handle parent) {
-        handle result = type_caster_base<type>::cast(src.get(), policy, parent);
-        if (result)
-            src.release();
-        return result;
-    }
-    static PYBIND11_DESCR name() { return type_caster_base<type>::name(); }
-};
-
-template <> class type_caster<std::wstring> {
-public:
     bool load(handle src, bool) {
         object temp;
         handle load_src = src;
@@ -657,78 +637,62 @@ template <> class type_caster<std::wstring> {
             if (!temp) { PyErr_Clear(); return false; }
             load_src = temp;
         }
-        wchar_t *buffer = nullptr;
-        ssize_t length = -1;
-#if PY_MAJOR_VERSION >= 3
-        buffer = PyUnicode_AsWideCharString(load_src.ptr(), &length);
-#else
-        temp = reinterpret_steal<object>(PyUnicode_AsEncodedString(
-            load_src.ptr(), sizeof(wchar_t) == sizeof(short)
-            ? "utf16" : "utf32", nullptr));
-
-        if (temp) {
-            int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), (char **) &buffer, &length);
-            if (err == -1) { buffer = nullptr; }  // TypeError
-            length = length / (ssize_t) sizeof(wchar_t) - 1; ++buffer; // Skip BOM
-        }
-#endif
-        if (!buffer) { PyErr_Clear(); return false; }
-        value = std::wstring(buffer, (size_t) length);
+
+        object utfNbytes = reinterpret_steal<object>(PyUnicode_AsEncodedString(
+            load_src.ptr(),
+            UTF_N == 8 ? "utf8" : UTF_N == 16 ? "utf16" : "utf32",
+            nullptr));
+        if (!utfNbytes) { PyErr_Clear(); return false; }
+
+        const CharT *buffer = reinterpret_cast<const CharT *>(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr()));
+        size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT);
+        if (UTF_N > 8) { buffer++; length--; } // Skip BOM for UTF-16/32
+        value = StringType(buffer, length);
         success = true;
         return true;
     }
 
-    static handle cast(const std::wstring &src, return_value_policy /* policy */, handle /* parent */) {
-        return PyUnicode_FromWideChar(src.c_str(), (ssize_t) src.length());
+    static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) {
+        const char *buffer = reinterpret_cast<const char *>(src.c_str());
+        ssize_t nbytes = ssize_t(src.size() * sizeof(CharT));
+        handle s = PyUnicode_Decode(buffer, nbytes, UTF_N == 8 ? "utf8" : UTF_N == 16 ? "utf16" : "utf32", nullptr);
+        if (!s) throw error_already_set();
+        return s;
     }
 
-    PYBIND11_TYPE_CASTER(std::wstring, _(PYBIND11_STRING_NAME));
+    PYBIND11_TYPE_CASTER(StringType, _(PYBIND11_STRING_NAME));
 protected:
     bool success = false;
 };
 
-template <> class type_caster<char> : public type_caster<std::string> {
+template <typename CharT> struct type_caster<CharT, enable_if_t<is_std_char_type<CharT>::value>>
+: type_caster<std::basic_string<CharT>> {
+    using StringType = std::basic_string<CharT>;
+    using StringCaster = type_caster<StringType>;
+    using StringCaster::success;
+    using StringCaster::value;
 public:
     bool load(handle src, bool convert) {
         if (src.is_none()) return true;
-        return type_caster<std::string>::load(src, convert);
+        return StringCaster::load(src, convert);
     }
 
-    static handle cast(const char *src, return_value_policy /* policy */, handle /* parent */) {
+    static handle cast(const CharT *src, return_value_policy policy, handle parent) {
         if (src == nullptr) return none().inc_ref();
-        return PyUnicode_FromString(src);
-    }
-
-    static handle cast(char src, return_value_policy /* policy */, handle /* parent */) {
-        char str[2] = { src, '\0' };
-        return PyUnicode_DecodeLatin1(str, 1, nullptr);
-    }
-
-    operator char*() { return success ? (char *) value.c_str() : nullptr; }
-    operator char&() { return value[0]; }
-
-    static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); }
-};
-
-template <> class type_caster<wchar_t> : public type_caster<std::wstring> {
-public:
-    bool load(handle src, bool convert) {
-        if (src.is_none()) return true;
-        return type_caster<std::wstring>::load(src, convert);
+        return StringCaster::cast(StringType(src), policy, parent);
     }
 
-    static handle cast(const wchar_t *src, return_value_policy /* policy */, handle /* parent */) {
-        if (src == nullptr) return none().inc_ref();
-        return PyUnicode_FromWideChar(src, (ssize_t) wcslen(src));
-    }
-
-    static handle cast(wchar_t src, return_value_policy /* policy */, handle /* parent */) {
-        wchar_t wstr[2] = { src, L'\0' };
-        return PyUnicode_FromWideChar(wstr, 1);
+    static handle cast(CharT src, return_value_policy policy, handle parent) {
+        if (std::is_same<char, CharT>::value) {
+            handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr);
+            if (!s) throw error_already_set();
+            return s;
+        }
+        return StringCaster::cast(StringType(1, src), policy, parent);
     }
 
-    operator wchar_t*() { return success ? (wchar_t *) value.c_str() : nullptr; }
-    operator wchar_t&() { return value[0]; }
+    operator CharT*() { return success ? (CharT *) value.c_str() : nullptr; }
+    operator CharT&() { return value[0]; }
 
     static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); }
 };
@@ -835,6 +799,17 @@ template <typename... Tuple> class type_caster<std::tuple<Tuple...>> {
     std::tuple<make_caster<Tuple>...> value;
 };
 
+template <typename type, typename deleter> class type_caster<std::unique_ptr<type, deleter>> {
+public:
+    static handle cast(std::unique_ptr<type, deleter> &&src, return_value_policy policy, handle parent) {
+        handle result = type_caster_base<type>::cast(src.get(), policy, parent);
+        if (result)
+            src.release();
+        return result;
+    }
+    static PYBIND11_DESCR name() { return type_caster_base<type>::name(); }
+};
+
 /// Type caster for holder types like std::shared_ptr, etc.
 template <typename type, typename holder_type> class type_caster_holder : public type_caster_base<type> {
 public:
@@ -1231,12 +1206,11 @@ NAMESPACE_BEGIN(detail)
 // forward declaration
 struct function_record;
 
-using function_arguments = const std::vector<handle> &;
-
 /// Helper class which loads arguments for C++ functions called from Python
 template <typename... Args>
 class argument_loader {
     using indices = make_index_sequence<sizeof...(Args)>;
+    using function_arguments = const std::vector<handle> &;
 
     template <typename Arg> using argument_is_args   = std::is_same<intrinsic_t<Arg>, args>;
     template <typename Arg> using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>;