From c801639e5127028a9ef4779fb17c9ae8193026b9 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 3 Jun 2021 12:51:19 +0200 Subject: [PATCH 1/5] Adding possibility to 'cast' or copy to `xt::xarray` etc --- .azure-pipelines/unix-build.yml | 8 ++ docs/source/examples/copy_cast/CMakeLists.txt | 13 ++ docs/source/examples/copy_cast/example.py | 6 + docs/source/examples/copy_cast/main.cpp | 25 ++++ include/xtensor-python/pynative_casters.hpp | 1 - .../xtensor_type_caster_base.hpp | 121 +++++++++++++++--- 6 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 docs/source/examples/copy_cast/CMakeLists.txt create mode 100644 docs/source/examples/copy_cast/example.py create mode 100644 docs/source/examples/copy_cast/main.cpp diff --git a/.azure-pipelines/unix-build.yml b/.azure-pipelines/unix-build.yml index d01651c3..0008e138 100644 --- a/.azure-pipelines/unix-build.yml +++ b/.azure-pipelines/unix-build.yml @@ -42,6 +42,14 @@ steps: displayName: Example - readme 1 workingDirectory: $(Build.SourcesDirectory)/docs/source/examples/readme_example_1 + - script: | + source activate xtensor-python + cmake . -DPYTHON_EXECUTABLE=`which python` + cmake --build . + python example.py + displayName: Example - Copy 'cast' + workingDirectory: $(Build.SourcesDirectory)/docs/source/examples/copy_cast + - script: | source activate xtensor-python cmake . -DPYTHON_EXECUTABLE=`which python` diff --git a/docs/source/examples/copy_cast/CMakeLists.txt b/docs/source/examples/copy_cast/CMakeLists.txt new file mode 100644 index 00000000..e17611c1 --- /dev/null +++ b/docs/source/examples/copy_cast/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.1..3.19) + +project(mymodule) + +find_package(pybind11 CONFIG REQUIRED) +find_package(xtensor REQUIRED) +find_package(xtensor-python REQUIRED) +find_package(Python REQUIRED COMPONENTS NumPy) + +pybind11_add_module(mymodule main.cpp) +target_link_libraries(mymodule PUBLIC pybind11::module xtensor-python Python::NumPy) + +target_compile_definitions(mymodule PRIVATE VERSION_INFO=0.1.0) diff --git a/docs/source/examples/copy_cast/example.py b/docs/source/examples/copy_cast/example.py new file mode 100644 index 00000000..d007e7c1 --- /dev/null +++ b/docs/source/examples/copy_cast/example.py @@ -0,0 +1,6 @@ +import mymodule +import numpy as np + +c = np.array([[1, 2, 3], [4, 5, 6]]) +assert np.isclose(np.sum(np.sin(c)), mymodule.sum_of_sines(c)) +assert np.isclose(np.sum(np.cos(c)), mymodule.sum_of_cosines(c)) diff --git a/docs/source/examples/copy_cast/main.cpp b/docs/source/examples/copy_cast/main.cpp new file mode 100644 index 00000000..cb3e65c8 --- /dev/null +++ b/docs/source/examples/copy_cast/main.cpp @@ -0,0 +1,25 @@ +#include +#include +#include +#define FORCE_IMPORT_ARRAY +#include + +double sum_of_sines(xt::pyarray& m) +{ + auto sines = xt::sin(m); // sines does not actually hold values. + return std::accumulate(sines.begin(), sines.end(), 0.0); +} + +double sum_of_cosines(const xt::xarray& m) +{ + auto cosines = xt::cos(m); // cosines does not actually hold values. + return std::accumulate(cosines.begin(), cosines.end(), 0.0); +} + +PYBIND11_MODULE(mymodule, m) +{ + xt::import_numpy(); + m.doc() = "Test module for xtensor python bindings"; + m.def("sum_of_sines", sum_of_sines, "Sum the sines of the input values"); + m.def("sum_of_cosines", sum_of_cosines, "Sum the cosines of the input values"); +} diff --git a/include/xtensor-python/pynative_casters.hpp b/include/xtensor-python/pynative_casters.hpp index f0f00652..caaad34b 100644 --- a/include/xtensor-python/pynative_casters.hpp +++ b/include/xtensor-python/pynative_casters.hpp @@ -12,7 +12,6 @@ #include "xtensor_type_caster_base.hpp" - namespace pybind11 { namespace detail diff --git a/include/xtensor-python/xtensor_type_caster_base.hpp b/include/xtensor-python/xtensor_type_caster_base.hpp index 3c8a015f..e587865e 100644 --- a/include/xtensor-python/xtensor_type_caster_base.hpp +++ b/include/xtensor-python/xtensor_type_caster_base.hpp @@ -23,6 +23,87 @@ namespace pybind11 { namespace detail { + template + struct xtensor_get_buffer + { + template + static auto get(H src) + { + return array_t::ensure(src); + } + }; + + template + struct xtensor_get_buffer + { + template + static auto get(H src) + { + return array_t::ensure(src); + } + }; + + template + struct xtensor_check_buffer + { + }; + + template + struct xtensor_check_buffer> + { + template + static auto get(H src) + { + auto buf = xtensor_get_buffer::get(src); + return buf; + } + }; + + template + struct xtensor_check_buffer> + { + template + static auto get(H src) + { + auto buf = xtensor_get_buffer::get(src); + if (buf.ndim() != N) { + return false; + } + return buf; + } + }; + + template + struct xtensor_check_buffer> + { + template + static auto get(H /*src*/) + { + return false; + } + }; + + template + struct xtensor_check_buffer> + { + template + static auto get(H /*src*/) + { + return false; + } + }; + + template + struct xtensor_check_buffer> + { + template + static auto get(H /*src*/) + { + return false; + } + }; + + // Casts a strided expression type to numpy array.If given a base, // the numpy array references the src data, otherwise it'll make a copy. // The writeable attributes lets you specify writeable flag for the array. @@ -74,10 +155,6 @@ namespace pybind11 template struct xtensor_type_caster_base { - bool load(handle /*src*/, bool) - { - return false; - } private: @@ -106,6 +183,30 @@ namespace pybind11 public: + PYBIND11_TYPE_CASTER(Type, _("numpy.ndarray[") + npy_format_descriptor::name + _("]")); + + bool load(handle src, bool convert) + { + using T = typename Type::value_type; + + if (!convert && !array_t::check_(src)) { + return false; + } + + auto buf = xtensor_check_buffer::get(src); + + if (!buf) { + return false; + } + + std::vector shape(buf.ndim()); + std::copy(buf.shape(), buf.shape() + buf.ndim(), shape.begin()); + value = Type(shape); + std::copy(buf.data(), buf.data() + buf.size(), value.begin()); + + return true; + } + // Normal returned non-reference, non-const value: static handle cast(Type&& src, return_value_policy /* policy */, handle parent) { @@ -151,18 +252,6 @@ namespace pybind11 { return cast_impl(src, policy, parent); } - -#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3 - static PYBIND11_DESCR name() - { - return _("xt::xtensor"); - } -#else - static constexpr auto name = _("xt::xtensor"); -#endif - - template - using cast_op_type = cast_op_type; }; } } From e21ecb0177268267df8aa16d4639d681ddc86dcd Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Sat, 5 Jun 2021 13:08:01 +0200 Subject: [PATCH 2/5] Adding example --- docs/source/examples.rst | 50 +++++++++++++++++++ docs/source/examples/copy_cast/main.cpp | 6 ++- .../xtensor_type_caster_base.hpp | 24 ++++----- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 4788b8bf..cd7242a5 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -98,3 +98,53 @@ Then we can test the module: we should compile and run the example from the same folder. To install, please consult `this pybind11 / CMake example `_. + + +Fall-back cast +============== + +The previous example showed you how to design your module to be flexible in accepting data. +From C++ we used ``xt::xarray``, +whereas for the Python API we used ``xt::pyarray`` to operate directly on the memory +of a NumPy array from Python (without copying the data). + +Sometimes, you might not have the flexibility to design your module's methods +with template parameters. +This might occur when you want to ``override`` functions +(though it is recommended to use CRTP to still use templates). +In this case we can still bind the module in Python using *xtensor-python*, +however, we have to copy the data from a (NumPy) array. +This means that although the following signatures are quite different when used from C++, +as follows: + +1. *Constant reference*: read from the data, without copying it. + + .. code-block:: cpp + + void foo(const xt::xarray& a); + +2. *Reference*: read from and/or write to the data, without copying it. + + .. code-block:: cpp + + void foo(xt::xarray& a); + +3. *Copy*: copy the data. + + .. code-block:: cpp + + void foo(xt::xarray a); + +The Python will all cases result in a copy to a temporary variable +(though the last signature will lead to a copy to a temporary variable, and another copy to ``a``). +On the one hand, this is more costly than when using ``xt::pyarray`` and ``xt::pyxtensor``, +on the other hand, it means that all changes you make to a reference, are made to the temporary +copy, and are thus lost. + +Still, it might be a convenient way to create Python bindings, using a minimal effort. +Consider this example: + +:download:`main.cpp ` + +.. literalinclude:: examples/copy_cast/main.cpp + :language: cpp diff --git a/docs/source/examples/copy_cast/main.cpp b/docs/source/examples/copy_cast/main.cpp index cb3e65c8..2e126098 100644 --- a/docs/source/examples/copy_cast/main.cpp +++ b/docs/source/examples/copy_cast/main.cpp @@ -4,12 +4,14 @@ #define FORCE_IMPORT_ARRAY #include -double sum_of_sines(xt::pyarray& m) +template +double sum_of_sines(T& m) { auto sines = xt::sin(m); // sines does not actually hold values. return std::accumulate(sines.begin(), sines.end(), 0.0); } +// In the Python API this a reference to a temporary variable double sum_of_cosines(const xt::xarray& m) { auto cosines = xt::cos(m); // cosines does not actually hold values. @@ -20,6 +22,6 @@ PYBIND11_MODULE(mymodule, m) { xt::import_numpy(); m.doc() = "Test module for xtensor python bindings"; - m.def("sum_of_sines", sum_of_sines, "Sum the sines of the input values"); + m.def("sum_of_sines", sum_of_sines>, "Sum the sines of the input values"); m.def("sum_of_cosines", sum_of_cosines, "Sum the cosines of the input values"); } diff --git a/include/xtensor-python/xtensor_type_caster_base.hpp b/include/xtensor-python/xtensor_type_caster_base.hpp index e587865e..68218eea 100644 --- a/include/xtensor-python/xtensor_type_caster_base.hpp +++ b/include/xtensor-python/xtensor_type_caster_base.hpp @@ -26,8 +26,7 @@ namespace pybind11 template struct xtensor_get_buffer { - template - static auto get(H src) + static auto get(handle src) { return array_t::ensure(src); } @@ -36,8 +35,7 @@ namespace pybind11 template struct xtensor_get_buffer { - template - static auto get(H src) + static auto get(handle src) { return array_t::ensure(src); } @@ -51,8 +49,7 @@ namespace pybind11 template struct xtensor_check_buffer> { - template - static auto get(H src) + static auto get(handle src) { auto buf = xtensor_get_buffer::get(src); return buf; @@ -62,8 +59,7 @@ namespace pybind11 template struct xtensor_check_buffer> { - template - static auto get(H src) + static auto get(handle src) { auto buf = xtensor_get_buffer::get(src); if (buf.ndim() != N) { @@ -76,8 +72,7 @@ namespace pybind11 template struct xtensor_check_buffer> { - template - static auto get(H /*src*/) + static auto get(handle /*src*/) { return false; } @@ -86,18 +81,17 @@ namespace pybind11 template struct xtensor_check_buffer> { - template - static auto get(H /*src*/) + static auto get(handle src) { - return false; + auto buf = xtensor_get_buffer::get(src); + return buf; } }; template struct xtensor_check_buffer> { - template - static auto get(H /*src*/) + static auto get(handle /*src*/) { return false; } From 428ed03b5dbc2b9ab0a62ffbf9da5a4ed0705afe Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Sat, 5 Jun 2021 16:08:29 +0200 Subject: [PATCH 3/5] Adding tests & minor bugfix --- .../xtensor_type_caster_base.hpp | 39 ++++++++++++++----- test_python/main.cpp | 32 +++++++++++++++ test_python/test_pyarray.py | 16 ++++++++ 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/include/xtensor-python/xtensor_type_caster_base.hpp b/include/xtensor-python/xtensor_type_caster_base.hpp index 68218eea..8db9e445 100644 --- a/include/xtensor-python/xtensor_type_caster_base.hpp +++ b/include/xtensor-python/xtensor_type_caster_base.hpp @@ -37,7 +37,7 @@ namespace pybind11 { static auto get(handle src) { - return array_t::ensure(src); + return array_t::ensure(src); } }; @@ -51,8 +51,7 @@ namespace pybind11 { static auto get(handle src) { - auto buf = xtensor_get_buffer::get(src); - return buf; + return xtensor_get_buffer::get(src); } }; @@ -61,11 +60,7 @@ namespace pybind11 { static auto get(handle src) { - auto buf = xtensor_get_buffer::get(src); - if (buf.ndim() != N) { - return false; - } - return buf; + return xtensor_get_buffer::get(src); } }; @@ -98,6 +93,27 @@ namespace pybind11 }; + template + struct xtensor_verify + { + template + static bool get(const B& buf) + { + return true; + } + }; + + template + struct xtensor_verify> + { + template + static bool get(const B& buf) + { + return buf.ndim() == N; + } + }; + + // Casts a strided expression type to numpy array.If given a base, // the numpy array references the src data, otherwise it'll make a copy. // The writeable attributes lets you specify writeable flag for the array. @@ -192,11 +208,14 @@ namespace pybind11 if (!buf) { return false; } + if (!xtensor_verify::get(buf)) { + return false; + } std::vector shape(buf.ndim()); std::copy(buf.shape(), buf.shape() + buf.ndim(), shape.begin()); - value = Type(shape); - std::copy(buf.data(), buf.data() + buf.size(), value.begin()); + value = Type::from_shape(shape); + std::copy(buf.data(), buf.data() + buf.size(), value.data()); return true; } diff --git a/test_python/main.cpp b/test_python/main.cpp index 66f4e3ca..7e5963c1 100644 --- a/test_python/main.cpp +++ b/test_python/main.cpp @@ -33,6 +33,33 @@ xt::pyarray example2(xt::pyarray& m) return m + 2; } +xt::xarray example3_xarray(const xt::xarray& m) +{ + return xt::transpose(m) + 2; +} + +xt::xarray example3_xarray_colmajor( + const xt::xarray& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor example3_xtensor3(const xt::xtensor& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor example3_xtensor2(const xt::xtensor& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor example3_xtensor2_colmajor( + const xt::xtensor& m) +{ + return xt::transpose(m) + 2; +} + // Readme Examples double readme_example1(xt::pyarray& m) @@ -249,6 +276,11 @@ PYBIND11_MODULE(xtensor_python_test, m) m.def("example1", example1); m.def("example2", example2); + m.def("example3_xarray", example3_xarray); + m.def("example3_xarray_colmajor", example3_xarray_colmajor); + m.def("example3_xtensor3", example3_xtensor3); + m.def("example3_xtensor2", example3_xtensor2); + m.def("example3_xtensor2_colmajor", example3_xtensor2_colmajor); m.def("complex_overload", no_complex_overload); m.def("complex_overload", complex_overload); diff --git a/test_python/test_pyarray.py b/test_python/test_pyarray.py index 71e9dd33..8760c974 100644 --- a/test_python/test_pyarray.py +++ b/test_python/test_pyarray.py @@ -36,6 +36,22 @@ def test_example2(self): y = xt.example2(x) np.testing.assert_allclose(y, res, 1e-12) + def test_example3(self): + x = np.arange(2 * 3).reshape(2, 3) + xc = np.asfortranarray(x) + y = np.arange(2 * 3 * 4).reshape(2, 3, 4) + v = y[1:, 1:, 0] + z = np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5) + np.testing.assert_array_equal(xt.example3_xarray(x), x.T + 2) + np.testing.assert_array_equal(xt.example3_xarray_colmajor(xc), xc.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor3(y), y.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor2(x), x.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor2(y[1:, 1:, 0]), v.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor2_colmajor(xc), xc.T + 2) + + with self.assertRaises(TypeError): + xt.example3_xtensor3(x) + def test_vectorize(self): x1 = np.array([[0, 1], [2, 3]]) x2 = np.array([0, 1]) From acc30eddda5089dbd7873897c479dbd72c116d46 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 10 Jun 2021 11:51:03 +0200 Subject: [PATCH 4/5] Nitpicking --- .../xtensor_type_caster_base.hpp | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/include/xtensor-python/xtensor_type_caster_base.hpp b/include/xtensor-python/xtensor_type_caster_base.hpp index 8db9e445..672bd644 100644 --- a/include/xtensor-python/xtensor_type_caster_base.hpp +++ b/include/xtensor-python/xtensor_type_caster_base.hpp @@ -24,69 +24,69 @@ namespace pybind11 namespace detail { template - struct xtensor_get_buffer + struct pybind_array_getter_impl { - static auto get(handle src) + static auto run(handle src) { return array_t::ensure(src); } }; template - struct xtensor_get_buffer + struct pybind_array_getter_impl { - static auto get(handle src) + static auto run(handle src) { return array_t::ensure(src); } }; template - struct xtensor_check_buffer + struct pybind_array_getter { }; template - struct xtensor_check_buffer> + struct pybind_array_getter> { - static auto get(handle src) + static auto run(handle src) { - return xtensor_get_buffer::get(src); + return pybind_array_getter_impl::run(src); } }; template - struct xtensor_check_buffer> + struct pybind_array_getter> { - static auto get(handle src) + static auto run(handle src) { - return xtensor_get_buffer::get(src); + return pybind_array_getter_impl::run(src); } }; template - struct xtensor_check_buffer> + struct pybind_array_getter> { - static auto get(handle /*src*/) + static auto run(handle /*src*/) { return false; } }; template - struct xtensor_check_buffer> + struct pybind_array_getter> { - static auto get(handle src) + static auto run(handle src) { - auto buf = xtensor_get_buffer::get(src); + auto buf = pybind_array_getter_impl::run(src); return buf; } }; template - struct xtensor_check_buffer> + struct pybind_array_getter> { - static auto get(handle /*src*/) + static auto run(handle /*src*/) { return false; } @@ -94,20 +94,20 @@ namespace pybind11 template - struct xtensor_verify + struct pybind_array_dim_checker { template - static bool get(const B& buf) + static bool run(const B& buf) { return true; } }; template - struct xtensor_verify> + struct pybind_array_dim_checker> { template - static bool get(const B& buf) + static bool run(const B& buf) { return buf.ndim() == N; } @@ -199,16 +199,19 @@ namespace pybind11 { using T = typename Type::value_type; - if (!convert && !array_t::check_(src)) { + if (!convert && !array_t::check_(src)) + { return false; } - auto buf = xtensor_check_buffer::get(src); + auto buf = pybind_array_getter::run(src); - if (!buf) { + if (!buf) + { return false; } - if (!xtensor_verify::get(buf)) { + if (!pybind_array_dim_checker::run(buf)) + { return false; } From af91def5d4bb9b7966f134adb581c59f6c66af87 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 10 Jun 2021 11:52:58 +0200 Subject: [PATCH 5/5] Fixing CI --- .azure-pipelines/unix-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/unix-build.yml b/.azure-pipelines/unix-build.yml index 0fcf6924..97cb4052 100644 --- a/.azure-pipelines/unix-build.yml +++ b/.azure-pipelines/unix-build.yml @@ -52,13 +52,16 @@ steps: cmake --build . cp ../example.py . python example.py + cd .. displayName: Example - Copy 'cast' workingDirectory: $(Build.SourcesDirectory)/docs/source/examples/copy_cast - script: | source activate xtensor-python - cmake . -DPYTHON_EXECUTABLE=`which python` + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build cmake --build . + cp ../example.py . python example.py cd .. displayName: Example - SFINAE