From 1d86ad45968e3c37345d0ef6121f91dbe82690f5 Mon Sep 17 00:00:00 2001 From: Alessio Eberl Date: Wed, 24 Apr 2024 22:42:35 +0200 Subject: [PATCH 1/3] Use buffer protocol to convert python objects to managed arrays If a python object is converted to a managed array and it exposes the buffer protocol the buffer can be copied directly to the managed array. For now this works only if the requested array element type is blittable. --- src/embed_tests/TestConverter.cs | 70 ++++++++++++++++++++++++++++++++ src/runtime/Converter.cs | 38 +++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index a59b9c97b..748db71df 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Numerics; +using System.Text; using NUnit.Framework; @@ -209,6 +210,75 @@ class PyGetListImpl(test.GetListImpl): List result = inst.GetList(); CollectionAssert.AreEqual(new[] { "testing" }, result); } + + [Test] + public void TestConvertNumpyFloat32ArrayToManaged() + { + var testValue = new float[] { 0, 1, 2, 3 }; + var nparr = np.arange(4, dtype: np.float32); + + object convertedValue; + var converted = Converter.ToManaged(nparr, typeof(float[]), out convertedValue, false); + + Assert.IsTrue(converted); + Assert.AreEqual(testValue, convertedValue); + } + + [Test] + public void TestConvertNumpyFloat64_2DArrayToManaged() + { + var testValue = new double[,] {{ 0, 1, 2, 3,}, { 4, 5, 6, 7 }, { 8, 9, 10, 11 }}; + var shape = new PyTuple(new[] {new PyInt(3), new PyInt(4)}); + var nparr = np.arange(12, dtype: np.float64).reshape(shape); + + object convertedValue; + var converted = Converter.ToManaged(nparr, typeof(double[,]), out convertedValue, false); + + Assert.IsTrue(converted); + Assert.AreEqual(testValue, convertedValue); + } + + [Test] + public void TestConvertBytearrayToManaged() + { + var testValue = Encoding.ASCII.GetBytes("test"); + using var str = PythonEngine.Eval("'test'.encode('ascii')"); + + object convertedValue; + var converted = Converter.ToManaged(str, typeof(byte[]), out convertedValue, false); + + Assert.IsTrue(converted); + Assert.AreEqual(testValue, convertedValue); + } + + [Test] + public void TestConvertCharArrayToManaged() + { + var testValue = new char[] { 't', 'e', 's', 't' }; + using var str = PythonEngine.Eval("'test'.encode('ascii')"); + + object convertedValue; + var converted = Converter.ToManaged(str, typeof(char[]), out convertedValue, false); + + Assert.IsTrue(converted); + Assert.AreEqual(testValue, convertedValue); + } + + dynamic np + { + get + { + try + { + return Py.Import("numpy"); + } + catch (PythonException) + { + Assert.Inconclusive("Numpy or dependency not installed"); + return null; + } + } + } } public interface IGetList diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 50b33e60e..5e755476d 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using System.Security; @@ -880,12 +881,49 @@ private static void SetConversionError(BorrowedReference value, Type target) /// Convert a Python value to a correctly typed managed array instance. /// The Python value must support the Python iterator protocol or and the /// items in the sequence must be convertible to the target array type. + /// If the Python value supports the buffer protocol the underlying buffer + /// will be copied directly. /// private static bool ToArray(BorrowedReference value, Type obType, out object? result, bool setError) { Type elementType = obType.GetElementType(); result = null; + // structs can also be blittable but for the standard usecases this suffices + var isBlittable = elementType.IsPrimitive && elementType != typeof(char) && elementType != typeof(bool); + if (isBlittable && Runtime.PyObject_GetBuffer(value, out var view, (int)PyBUF.CONTIG_RO) == 0) + { + GCHandle ptr; + try + { + var shape = new IntPtr[view.ndim]; + Marshal.Copy(view.shape, shape, 0, view.ndim); + Array arr = Array.CreateInstance(elementType, shape.Select(x => (long)x).ToArray()); + ptr = GCHandle.Alloc(arr, GCHandleType.Pinned); + var addr = ptr.AddrOfPinnedObject(); + unsafe + { + Buffer.MemoryCopy((void*)view.buf, (void*)addr, arr.Length * Marshal.SizeOf(elementType), view.len); + } + + result = arr; + return true; + } + finally + { + if (ptr.IsAllocated) + { + ptr.Free(); + } + + Runtime.PyBuffer_Release(ref view); + } + } + else + { + Exceptions.Clear(); + } + using var IterObject = Runtime.PyObject_GetIter(value); if (IterObject.IsNull()) { From b487188443db70ff2eccadbd3d0ac289c111ba00 Mon Sep 17 00:00:00 2001 From: Alessio Eberl Date: Sat, 4 May 2024 17:35:17 +0200 Subject: [PATCH 2/3] Add tests for python arrays --- src/embed_tests/TestConverter.cs | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index 748db71df..7af678fec 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -264,6 +264,36 @@ public void TestConvertCharArrayToManaged() Assert.AreEqual(testValue, convertedValue); } + [Test] + [TestCaseSource(typeof(Arrays))] + public void TestConvertArrayToManaged(string arrayType, Type t, object expected) + { + object convertedValue; + var arr = array.array(arrayType.ToPython(), expected.ToPython()); + var converted = Converter.ToManaged(arr, t, out convertedValue, false); + + Assert.IsTrue(converted); + Assert.AreEqual(expected, convertedValue); + } + + public class Arrays : System.Collections.IEnumerable + { + public System.Collections.IEnumerator GetEnumerator() + { + yield return new object[] { "b", typeof(byte[]), new byte[] { 0, 1, 2, 3, 4 } }; + yield return new object[] { "B", typeof(byte[]), new byte[] { 0, 1, 2, 3, 4 } }; + yield return new object[] { "u", typeof(char[]), new char[] { 'a', 'b', 'c', 'd', 'e' } }; + yield return new object[] { "h", typeof(short[]), new short[] { -2, -1, 0, 1, 2, 3, 4 } }; + yield return new object[] { "H", typeof(ushort[]), new ushort[] { 0, 1, 2, 3, 4 } }; + yield return new object[] { "i", typeof(int[]), new int[] { -2, -1, 0, 1, 2, 3, 4 } }; + yield return new object[] { "I", typeof(uint[]), new uint[] { 0, 1, 2, 3, 4 } }; + yield return new object[] { "q", typeof(long[]), new long[] { -2, -1, 0, 1, 2, 3, 4 } }; + yield return new object[] { "q", typeof(ulong[]), new ulong[] { 0, 1, 2, 3, 4 } }; + yield return new object[] { "f", typeof(float[]), new float[] { -2, -1, 0, 1, 2, 3, 4 } }; + yield return new object[] { "d", typeof(double[]), new double[] { -2, -1, 0, 1, 2, 3, 4 } }; + } + }; + dynamic np { get @@ -279,6 +309,22 @@ dynamic np } } } + + dynamic array + { + get + { + try + { + return Py.Import("array"); + } + catch (PythonException) + { + Assert.Inconclusive("Could not import array"); + return null; + } + } + } } public interface IGetList From 2e2866dc7b47c52001f81c2baba28d975f6a39da Mon Sep 17 00:00:00 2001 From: Alessio Eberl Date: Sun, 5 May 2024 13:19:14 +0200 Subject: [PATCH 3/3] Remove bytearray to char test Currently not handled by the buffer copying mechanism anyway --- src/embed_tests/TestConverter.cs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index 7af678fec..2fb21f06c 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -251,19 +251,6 @@ public void TestConvertBytearrayToManaged() Assert.AreEqual(testValue, convertedValue); } - [Test] - public void TestConvertCharArrayToManaged() - { - var testValue = new char[] { 't', 'e', 's', 't' }; - using var str = PythonEngine.Eval("'test'.encode('ascii')"); - - object convertedValue; - var converted = Converter.ToManaged(str, typeof(char[]), out convertedValue, false); - - Assert.IsTrue(converted); - Assert.AreEqual(testValue, convertedValue); - } - [Test] [TestCaseSource(typeof(Arrays))] public void TestConvertArrayToManaged(string arrayType, Type t, object expected)