Skip to content

Commit 707ef36

Browse files
koubaakoubaa
authored
Lossless encoders for IList<T>, IEnumerable<T> and ICollection<T> (#1084)
1 parent 1ab9cb1 commit 707ef36

File tree

11 files changed

+720
-19
lines changed

11 files changed

+720
-19
lines changed

‎src/embed_tests/Codecs.cs

+232-18
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,39 @@
11
namespace Python.EmbeddingTest {
22
using System;
33
using System.Collections.Generic;
4-
using System.Text;
4+
using System.Linq;
55
using NUnit.Framework;
66
using Python.Runtime;
77
using Python.Runtime.Codecs;
88

9-
public class Codecs {
9+
public class Codecs
10+
{
1011
[SetUp]
11-
public void SetUp() {
12+
public void SetUp()
13+
{
1214
PythonEngine.Initialize();
1315
}
1416

1517
[TearDown]
16-
public void Dispose() {
18+
public void Dispose()
19+
{
1720
PythonEngine.Shutdown();
1821
}
1922

2023
[Test]
21-
public void ConversionsGeneric() {
22-
ConversionsGeneric<ValueTuple<int, string, object>, ValueTuple>();
24+
public void TupleConversionsGeneric()
25+
{
26+
TupleConversionsGeneric<ValueTuple<int, string, object>, ValueTuple>();
2327
}
2428

25-
static void ConversionsGeneric<T, TTuple>() {
29+
static void TupleConversionsGeneric<T, TTuple>()
30+
{
2631
TupleCodec<TTuple>.Register();
2732
var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
2833
T restored = default;
2934
using (Py.GIL())
30-
using (var scope = Py.CreateScope()) {
35+
using (var scope = Py.CreateScope())
36+
{
3137
void Accept(T value) => restored = value;
3238
var accept = new Action<T>(Accept).ToPython();
3339
scope.Set(nameof(tuple), tuple);
@@ -38,15 +44,18 @@ static void ConversionsGeneric<T, TTuple>() {
3844
}
3945

4046
[Test]
41-
public void ConversionsObject() {
42-
ConversionsObject<ValueTuple<int, string, object>, ValueTuple>();
47+
public void TupleConversionsObject()
48+
{
49+
TupleConversionsObject<ValueTuple<int, string, object>, ValueTuple>();
4350
}
44-
static void ConversionsObject<T, TTuple>() {
51+
static void TupleConversionsObject<T, TTuple>()
52+
{
4553
TupleCodec<TTuple>.Register();
4654
var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
4755
T restored = default;
4856
using (Py.GIL())
49-
using (var scope = Py.CreateScope()) {
57+
using (var scope = Py.CreateScope())
58+
{
5059
void Accept(object value) => restored = (T)value;
5160
var accept = new Action<object>(Accept).ToPython();
5261
scope.Set(nameof(tuple), tuple);
@@ -57,31 +66,236 @@ static void ConversionsObject<T, TTuple>() {
5766
}
5867

5968
[Test]
60-
public void TupleRoundtripObject() {
69+
public void TupleRoundtripObject()
70+
{
6171
TupleRoundtripObject<ValueTuple<int, string, object>, ValueTuple>();
6272
}
63-
static void TupleRoundtripObject<T, TTuple>() {
73+
static void TupleRoundtripObject<T, TTuple>()
74+
{
6475
var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
65-
using (Py.GIL()) {
76+
using (Py.GIL())
77+
{
6678
var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
6779
Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out object restored));
6880
Assert.AreEqual(expected: tuple, actual: restored);
6981
}
7082
}
7183

7284
[Test]
73-
public void TupleRoundtripGeneric() {
85+
public void TupleRoundtripGeneric()
86+
{
7487
TupleRoundtripGeneric<ValueTuple<int, string, object>, ValueTuple>();
7588
}
7689

77-
static void TupleRoundtripGeneric<T, TTuple>() {
90+
static void TupleRoundtripGeneric<T, TTuple>()
91+
{
7892
var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
79-
using (Py.GIL()) {
93+
using (Py.GIL())
94+
{
8095
var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
8196
Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out T restored));
8297
Assert.AreEqual(expected: tuple, actual: restored);
8398
}
8499
}
100+
101+
static PyObject GetPythonIterable()
102+
{
103+
using (Py.GIL())
104+
{
105+
return PythonEngine.Eval("map(lambda x: x, [1,2,3])");
106+
}
107+
}
108+
109+
[Test]
110+
public void ListDecoderTest()
111+
{
112+
var codec = ListDecoder.Instance;
113+
var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
114+
115+
var pyList = new PyList(items.ToArray());
116+
117+
var pyListType = pyList.GetPythonType();
118+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<bool>)));
119+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<int>)));
120+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable)));
121+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable<int>)));
122+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection<float>)));
123+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool)));
124+
125+
//we'd have to copy into a list instance to do this, it would not be lossless.
126+
//lossy converters can be implemented outside of the python.net core library
127+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(List<int>)));
128+
129+
//convert to list of int
130+
IList<int> intList = null;
131+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); });
132+
CollectionAssert.AreEqual(intList, new List<object> { 1, 2, 3 });
133+
134+
//convert to list of string. This will not work.
135+
//The ListWrapper class will throw a python exception when it tries to access any element.
136+
//TryDecode is a lossless conversion so there will be no exception at that point
137+
//interestingly, since the size of the python list can be queried without any conversion,
138+
//the IList will report a Count of 3.
139+
IList<string> stringList = null;
140+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); });
141+
Assert.AreEqual(stringList.Count, 3);
142+
Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; });
143+
144+
//can't convert python iterable to list (this will require a copy which isn't lossless)
145+
var foo = GetPythonIterable();
146+
var fooType = foo.GetPythonType();
147+
Assert.IsFalse(codec.CanDecode(fooType, typeof(IList<int>)));
148+
}
149+
150+
[Test]
151+
public void SequenceDecoderTest()
152+
{
153+
var codec = SequenceDecoder.Instance;
154+
var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
155+
156+
//SequenceConverter can only convert to any ICollection
157+
var pyList = new PyList(items.ToArray());
158+
//it can convert a PyList, since PyList satisfies the python sequence protocol
159+
160+
Assert.IsFalse(codec.CanDecode(pyList, typeof(bool)));
161+
Assert.IsFalse(codec.CanDecode(pyList, typeof(IList<int>)));
162+
Assert.IsFalse(codec.CanDecode(pyList, typeof(System.Collections.IEnumerable)));
163+
Assert.IsFalse(codec.CanDecode(pyList, typeof(IEnumerable<int>)));
164+
165+
Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection<float>)));
166+
Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection<string>)));
167+
Assert.IsTrue(codec.CanDecode(pyList, typeof(ICollection<int>)));
168+
169+
//convert to collection of int
170+
ICollection<int> intCollection = null;
171+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); });
172+
CollectionAssert.AreEqual(intCollection, new List<object> { 1, 2, 3 });
173+
174+
//no python exception should have occurred during the above conversion and check
175+
Runtime.CheckExceptionOccurred();
176+
177+
//convert to collection of string. This will not work.
178+
//The SequenceWrapper class will throw a python exception when it tries to access any element.
179+
//TryDecode is a lossless conversion so there will be no exception at that point
180+
//interestingly, since the size of the python sequence can be queried without any conversion,
181+
//the IList will report a Count of 3.
182+
ICollection<string> stringCollection = null;
183+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); });
184+
Assert.AreEqual(3, stringCollection.Count());
185+
Assert.Throws(typeof(InvalidCastException), () => {
186+
string[] array = new string[3];
187+
stringCollection.CopyTo(array, 0);
188+
});
189+
190+
Runtime.CheckExceptionOccurred();
191+
192+
//can't convert python iterable to collection (this will require a copy which isn't lossless)
193+
//python iterables do not satisfy the python sequence protocol
194+
var foo = GetPythonIterable();
195+
var fooType = foo.GetPythonType();
196+
Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection<int>)));
197+
198+
//python tuples do satisfy the python sequence protocol
199+
var pyTuple = new PyTuple(items.ToArray());
200+
var pyTupleType = pyTuple.GetPythonType();
201+
202+
Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection<float>)));
203+
Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection<int>)));
204+
Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection<string>)));
205+
206+
//convert to collection of int
207+
ICollection<int> intCollection2 = null;
208+
Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); });
209+
CollectionAssert.AreEqual(intCollection2, new List<object> { 1, 2, 3 });
210+
211+
//no python exception should have occurred during the above conversion and check
212+
Runtime.CheckExceptionOccurred();
213+
214+
//convert to collection of string. This will not work.
215+
//The SequenceWrapper class will throw a python exception when it tries to access any element.
216+
//TryDecode is a lossless conversion so there will be no exception at that point
217+
//interestingly, since the size of the python sequence can be queried without any conversion,
218+
//the IList will report a Count of 3.
219+
ICollection<string> stringCollection2 = null;
220+
Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); });
221+
Assert.AreEqual(3, stringCollection2.Count());
222+
Assert.Throws(typeof(InvalidCastException), () => {
223+
string[] array = new string[3];
224+
stringCollection2.CopyTo(array, 0);
225+
});
226+
227+
Runtime.CheckExceptionOccurred();
228+
229+
}
230+
231+
[Test]
232+
public void IterableDecoderTest()
233+
{
234+
var codec = IterableDecoder.Instance;
235+
var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
236+
237+
var pyList = new PyList(items.ToArray());
238+
var pyListType = pyList.GetPythonType();
239+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList<bool>)));
240+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable)));
241+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<int>)));
242+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection<float>)));
243+
Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool)));
244+
245+
//ensure a PyList can be converted to a plain IEnumerable
246+
System.Collections.IEnumerable plainEnumerable1 = null;
247+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); });
248+
CollectionAssert.AreEqual(plainEnumerable1, new List<object> { 1, 2, 3 });
249+
250+
//can convert to any generic ienumerable. If the type is not assignable from the python element
251+
//it will lead to an empty iterable when decoding. TODO - should it throw?
252+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<int>)));
253+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<double>)));
254+
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable<string>)));
255+
256+
IEnumerable<int> intEnumerable = null;
257+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); });
258+
CollectionAssert.AreEqual(intEnumerable, new List<object> { 1, 2, 3 });
259+
260+
Runtime.CheckExceptionOccurred();
261+
262+
IEnumerable<double> doubleEnumerable = null;
263+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); });
264+
CollectionAssert.AreEqual(doubleEnumerable, new List<object> { 1, 2, 3 });
265+
266+
Runtime.CheckExceptionOccurred();
267+
268+
IEnumerable<string> stringEnumerable = null;
269+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); });
270+
271+
Assert.Throws(typeof(InvalidCastException), () => {
272+
foreach (string item in stringEnumerable)
273+
{
274+
var x = item;
275+
}
276+
});
277+
Assert.Throws(typeof(InvalidCastException), () => {
278+
stringEnumerable.Count();
279+
});
280+
281+
Runtime.CheckExceptionOccurred();
282+
283+
//ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable
284+
var foo = GetPythonIterable();
285+
var fooType = foo.GetPythonType();
286+
System.Collections.IEnumerable plainEnumerable2 = null;
287+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); });
288+
CollectionAssert.AreEqual(plainEnumerable2, new List<object> { 1, 2, 3 });
289+
290+
//can convert to any generic ienumerable. If the type is not assignable from the python element
291+
//it will be an exception during TryDecode
292+
Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable<int>)));
293+
Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable<double>)));
294+
Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable<string>)));
295+
296+
Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); });
297+
CollectionAssert.AreEqual(intEnumerable, new List<object> { 1, 2, 3 });
298+
}
85299
}
86300

87301
/// <summary>

‎src/runtime/Codecs/IterableDecoder.cs

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
using System;
2+
using System.Collections.Generic;
3+
4+
namespace Python.Runtime.Codecs
5+
{
6+
public class IterableDecoder : IPyObjectDecoder
7+
{
8+
internal static bool IsIterable(Type targetType)
9+
{
10+
//if it is a plain IEnumerable, we can decode it using sequence protocol.
11+
if (targetType == typeof(System.Collections.IEnumerable))
12+
return true;
13+
14+
if (!targetType.IsGenericType)
15+
return false;
16+
17+
return targetType.GetGenericTypeDefinition() == typeof(IEnumerable<>);
18+
}
19+
20+
internal static bool IsIterable(PyObject objectType)
21+
{
22+
return objectType.HasAttr("__iter__");
23+
}
24+
25+
public bool CanDecode(PyObject objectType, Type targetType)
26+
{
27+
return IsIterable(objectType) && IsIterable(targetType);
28+
}
29+
30+
public bool TryDecode<T>(PyObject pyObj, out T value)
31+
{
32+
//first see if T is a plan IEnumerable
33+
if (typeof(T) == typeof(System.Collections.IEnumerable))
34+
{
35+
object enumerable = new CollectionWrappers.IterableWrapper<object>(pyObj);
36+
value = (T)enumerable;
37+
return true;
38+
}
39+
40+
var elementType = typeof(T).GetGenericArguments()[0];
41+
var collectionType = typeof(CollectionWrappers.IterableWrapper<>).MakeGenericType(elementType);
42+
43+
var instance = Activator.CreateInstance(collectionType, new[] { pyObj });
44+
value = (T)instance;
45+
return true;
46+
}
47+
48+
public static IterableDecoder Instance { get; } = new IterableDecoder();
49+
50+
public static void Register()
51+
{
52+
PyObjectConversions.RegisterDecoder(Instance);
53+
}
54+
}
55+
}

0 commit comments

Comments
 (0)