Introduced PyIterable, PyObject no longer implements IEnumerable<PyOb…

…ject>
pythonnet · lostmsu · Sep 27, 2021 · Sep 23, 2021 · Sep 23, 2021 · Sep 23, 2021
commit f8becef17ef8ae9672b62dce2a29c9fba49112e8
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,7 @@ See [Mixins/collections.py](src/runtime/Mixins/collections.py).
 -   .NET arrays implement Python buffer protocol
 -   Python.NET will correctly resolve .NET methods, that accept `PyList`, `PyInt`,
 and other `PyObject` derived types when called from Python.
+-   `PyIterable` type, that wraps any iterable object in Python
 
 
 ### Changed
@@ -67,6 +68,8 @@ See [Mixins/collections.py](src/runtime/Mixins/collections.py).
 -   BREAKING: When trying to convert Python `int` to `System.Object`, result will
 be of type `PyInt` instead of `System.Int32` due to possible loss of information.
 Python `float` will continue to be converted to `System.Double`.
+-   BREAKING: `PyObject` no longer implements `IEnumerable<PyObject>`.
+Instead, `PyIterable` does that.
 
 ### Fixed
 

diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs
@@ -45,12 +45,12 @@ static void TupleConversionsGeneric<T, TTuple>()
         [Test]
         public void TupleConversionsObject()
         {
-            TupleConversionsObject<ValueTuple<int, string, object>, ValueTuple>();
+            TupleConversionsObject<ValueTuple<double, string, object>, ValueTuple>();
         }
         static void TupleConversionsObject<T, TTuple>()
         {
             TupleCodec<TTuple>.Register();
-            var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
+            var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object());
             T restored = default;
             using (var scope = Py.CreateScope())
             {
@@ -66,11 +66,11 @@ static void TupleConversionsObject<T, TTuple>()
         [Test]
         public void TupleRoundtripObject()
         {
-            TupleRoundtripObject<ValueTuple<int, string, object>, ValueTuple>();
+            TupleRoundtripObject<ValueTuple<double, string, object>, ValueTuple>();
         }
         static void TupleRoundtripObject<T, TTuple>()
         {
-            var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
+            var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object());
             var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
             Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out object restored));
             Assert.AreEqual(expected: tuple, actual: restored);
@@ -231,7 +231,7 @@ public void IterableDecoderTest()
             //ensure a PyList can be converted to a plain IEnumerable
             System.Collections.IEnumerable plainEnumerable1 = null;
             Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); });
-            CollectionAssert.AreEqual(plainEnumerable1, new List<object> { 1, 2, 3 });
+            CollectionAssert.AreEqual(plainEnumerable1.Cast<PyInt>().Select(i => i.ToInt32()), new List<object> { 1, 2, 3 });
 
             //can convert to any generic ienumerable.  If the type is not assignable from the python element
             //it will lead to an empty iterable when decoding.  TODO - should it throw?
@@ -271,7 +271,7 @@ public void IterableDecoderTest()
             var fooType = foo.GetPythonType();
             System.Collections.IEnumerable plainEnumerable2 = null;
             Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); });
-            CollectionAssert.AreEqual(plainEnumerable2, new List<object> { 1, 2, 3 });
+            CollectionAssert.AreEqual(plainEnumerable2.Cast<PyInt>().Select(i => i.ToInt32()), new List<object> { 1, 2, 3 });
 
             //can convert to any generic ienumerable.  If the type is not assignable from the python element
             //it will be an exception during TryDecode

diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs
@@ -1,5 +1,6 @@
 #nullable enable
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Runtime.InteropServices;
 
@@ -68,5 +69,7 @@ internal static string ReadStringResource(this System.Reflection.Assembly assemb
             using var reader = new StreamReader(stream);
             return reader.ReadToEnd();
         }
+
+        public static IEnumerator<T> GetEnumerator<T>(this IEnumerator<T> enumerator) => enumerator;
     }
 }
diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs
@@ -174,7 +174,7 @@ internal static Type CreateDerivedType(string name,
             {
                 Runtime.XIncref(py_dict);
                 using (var dict = new PyDict(py_dict))
-                using (PyObject keys = dict.Keys())
+                using (PyIterable keys = dict.Keys())
                 {
                     foreach (PyObject pyKey in keys)
                     {
@@ -223,7 +223,7 @@ internal static Type CreateDerivedType(string name,
             {
                 Runtime.XIncref(py_dict);
                 using (var dict = new PyDict(py_dict))
-                using (PyObject keys = dict.Keys())
+                using (PyIterable keys = dict.Keys())
                 {
                     foreach (PyObject pyKey in keys)
                     {
@@ -439,19 +439,14 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde
             }
 
             using (PyObject pyReturnType = func.GetAttr("_clr_return_type_"))
-            using (PyObject pyArgTypes = func.GetAttr("_clr_arg_types_"))
+            using (var pyArgTypes = PyIter.GetIter(func.GetAttr("_clr_arg_types_")))
             {
                 var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type;
                 if (returnType == null)
                 {
                     returnType = typeof(void);
                 }
 
-                if (!pyArgTypes.IsIterable())
-                {
-                    throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types");
-                }
-
                 var argTypes = new List<Type>();
                 foreach (PyObject pyArgType in pyArgTypes)
                 {

diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs
@@ -8,7 +8,7 @@ namespace Python.Runtime
     /// PY3: https://docs.python.org/3/c-api/dict.html
     /// for details.
     /// </summary>
-    public class PyDict : PyObject
+    public class PyDict : PyIterable
     {
         /// <summary>
         /// PyDict Constructor
@@ -102,14 +102,14 @@ public bool HasKey(string key)
         /// <remarks>
         /// Returns a sequence containing the keys of the dictionary.
         /// </remarks>
-        public PyObject Keys()
+        public PyIterable Keys()
         {
             using var items = Runtime.PyDict_Keys(Reference);
             if (items.IsNull())
             {
                 throw PythonException.ThrowLastAsClrException();
             }
-            return items.MoveToPyObject();
+            return new PyIterable(items.Steal());
         }
 
 
@@ -119,14 +119,14 @@ public PyObject Keys()
         /// <remarks>
         /// Returns a sequence containing the values of the dictionary.
         /// </remarks>
-        public PyObject Values()
+        public PyIterable Values()
         {
             IntPtr items = Runtime.PyDict_Values(obj);
             if (items == IntPtr.Zero)
             {
                 throw PythonException.ThrowLastAsClrException();
             }
-            return new PyObject(items);
+            return new PyIterable(items);
         }
 
 
@@ -136,22 +136,14 @@ public PyObject Values()
         /// <remarks>
         /// Returns a sequence containing the items of the dictionary.
         /// </remarks>
-        public PyObject Items()
+        public PyIterable Items()
         {
-            var items = Runtime.PyDict_Items(this.Reference);
-            try
-            {
-                if (items.IsNull())
-                {
-                    throw PythonException.ThrowLastAsClrException();
-                }
-
-                return items.MoveToPyObject();
-            }
-            finally
+            using var items = Runtime.PyDict_Items(this.Reference);
+            if (items.IsNull())
             {
-                items.Dispose();
+                throw PythonException.ThrowLastAsClrException();
             }
+            return new PyIterable(items.Steal());
         }
 
 

diff --git a/src/runtime/pyiterable.cs b/src/runtime/pyiterable.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Python.Runtime
+{
+    public class PyIterable : PyObject, IEnumerable<PyObject>
+    {
+        internal PyIterable(IntPtr ptr) : base(ptr)
+        {
+        }
+
+        internal PyIterable(BorrowedReference reference) : base(reference) { }
+        internal PyIterable(in StolenReference reference) : base(reference) { }
+
+        /// <summary>
+        /// Return a new PyIter object for the object. This allows any iterable
+        /// python object to be iterated over in C#. A PythonException will be
+        /// raised if the object is not iterable.
+        /// </summary>
+        public PyIter GetEnumerator()
+        {
+            return PyIter.GetIter(this);
+        }
+        IEnumerator<PyObject> IEnumerable<PyObject>.GetEnumerator() => this.GetEnumerator();
+        IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator();
+    }
+}
diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs
@@ -17,7 +17,7 @@ namespace Python.Runtime
     /// </summary>
     [Serializable]
     [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")]
-    public partial class PyObject : DynamicObject, IEnumerable<PyObject>, IDisposable
+    public partial class PyObject : DynamicObject, IDisposable
     {
 #if TRACE_ALLOC
         /// <summary>
@@ -80,7 +80,7 @@ internal PyObject(BorrowedReference reference)
 #endif
         }
 
-        internal PyObject(StolenReference reference)
+        internal PyObject(in StolenReference reference)
         {
             if (reference == null) throw new ArgumentNullException(nameof(reference));
 
@@ -703,21 +703,6 @@ public PyObject GetIterator()
             return new PyObject(r);
         }
 
-        /// <summary>
-        /// GetEnumerator Method
-        /// </summary>
-        /// <remarks>
-        /// Return a new PyIter object for the object. This allows any iterable
-        /// python object to be iterated over in C#. A PythonException will be
-        /// raised if the object is not iterable.
-        /// </remarks>
-        public IEnumerator<PyObject> GetEnumerator()
-        {
-            return PyIter.GetIter(this);
-        }
-        IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator();
-
-
         /// <summary>
         /// Invoke Method
         /// </summary>

diff --git a/src/runtime/pysequence.cs b/src/runtime/pysequence.cs
@@ -1,5 +1,4 @@
 using System;
-using System.Collections;
 
 namespace Python.Runtime
 {
@@ -10,13 +9,14 @@ namespace Python.Runtime
     /// PY3: https://docs.python.org/3/c-api/sequence.html
     /// for details.
     /// </summary>
-    public class PySequence : PyObject, IEnumerable
+    public class PySequence : PyIterable
     {
-        protected PySequence(IntPtr ptr) : base(ptr)
+        protected internal PySequence(IntPtr ptr) : base(ptr)
         {
         }
 
         internal PySequence(BorrowedReference reference) : base(reference) { }
+        internal PySequence(StolenReference reference) : base(reference) { }
 
 
         /// <summary>
@@ -30,7 +30,6 @@ public static bool IsSequenceType(PyObject value)
             return Runtime.PySequence_Check(value.obj);
         }
 
-
         /// <summary>
         /// GetSlice Method
         /// </summary>

diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs
@@ -387,7 +387,7 @@ public string Format()
             using var traceback = PyModule.Import("traceback");
             var buffer = new StringBuilder();
             using var values = traceback.InvokeMethod("format_exception", copy.Type, copy.Value, copy.Traceback);
-            foreach (PyObject val in values)
+            foreach (PyObject val in PyIter.GetIter(values))
             {
                 buffer.Append(val);
                 val.Dispose();