pythonnet · lostmsu · Sep 23, 2021 · May 24, 2020 · Jun 3, 2020 · Jun 17, 2020
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 -    Improved exception handling:
 -   exceptions can now be converted with codecs
 -   `InnerException` and `__cause__` are propagated properly
+-   .NET collection types now implement standard Python collection interfaces from `collections.abc`.
+See [Mixins/collections.py](src/runtime/Mixins/collections.py).
 -   .NET arrays implement Python buffer protocol
 
 

diff --git a/src/runtime/InteropConfiguration.cs b/src/runtime/InteropConfiguration.cs
@@ -3,6 +3,8 @@ namespace Python.Runtime
     using System;
     using System.Collections.Generic;
 
+    using Python.Runtime.Mixins;
+
     public sealed class InteropConfiguration
     {
         internal readonly PythonBaseTypeProviderGroup pythonBaseTypeProviders
@@ -18,6 +20,7 @@ public static InteropConfiguration MakeDefault()
                 PythonBaseTypeProviders =
                 {
                     DefaultBaseTypeProvider.Instance,
+                    new CollectionMixinsProvider(new Lazy<PyObject>(() => Py.Import("clr._extras.collections"))),
                 },
             };
         }

diff --git a/src/runtime/Mixins/CollectionMixinsProvider.cs b/src/runtime/Mixins/CollectionMixinsProvider.cs
@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Python.Runtime.Mixins
+{
+    class CollectionMixinsProvider : IPythonBaseTypeProvider
+    {
+        readonly Lazy<PyObject> mixinsModule;
+        public CollectionMixinsProvider(Lazy<PyObject> mixinsModule)
+        {
+            this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule));
+        }
+
+        public PyObject Mixins => this.mixinsModule.Value;
+
+        public IEnumerable<PyType> GetBaseTypes(Type type, IList<PyType> existingBases)
+        {
+            if (type is null)
+                throw new ArgumentNullException(nameof(type));
+
+            if (existingBases is null)
+                throw new ArgumentNullException(nameof(existingBases));
+
+            var interfaces = NewInterfaces(type).Select(GetDefinition).ToArray();
+
+            var newBases = new List<PyType>();
+            newBases.AddRange(existingBases);
+
+            // dictionaries
+            if (interfaces.Contains(typeof(IDictionary<,>)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("MutableMappingMixin")));
+            }
+            else if (interfaces.Contains(typeof(IReadOnlyDictionary<,>)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("MappingMixin")));
+            }
+
+            // item collections
+            if (interfaces.Contains(typeof(IList<>))
+                || interfaces.Contains(typeof(System.Collections.IList)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("MutableSequenceMixin")));
+            }
+            else if (interfaces.Contains(typeof(IReadOnlyList<>)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("SequenceMixin")));
+            }
+            else if (interfaces.Contains(typeof(ICollection<>))
+                     || interfaces.Contains(typeof(System.Collections.ICollection)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("CollectionMixin")));
+            }
+            else if (interfaces.Contains(typeof(System.Collections.IEnumerable)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("IterableMixin")));
+            }
+
+            // enumerators
+            if (interfaces.Contains(typeof(System.Collections.IEnumerator)))
+            {
+                newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin")));
+            }
+
+            if (newBases.Count == existingBases.Count)
+            {
+                return existingBases;
+            }
+
+            if (type.IsInterface && type.BaseType is null)
+            {
+                newBases.RemoveAll(@base => @base.Handle == Runtime.PyBaseObjectType);
+            }
+
+            return newBases;
+        }
+
+        static Type[] NewInterfaces(Type type)
+        {
+            var result = type.GetInterfaces();
+            return type.BaseType != null
+                ? result.Except(type.BaseType.GetInterfaces()).ToArray()
+                : result;
+        }
+
+        static Type GetDefinition(Type type)
+            => type.IsGenericType ? type.GetGenericTypeDefinition() : type;
+    }
+}
diff --git a/src/runtime/Mixins/collections.py b/src/runtime/Mixins/collections.py
@@ -0,0 +1,82 @@
+"""
+Implements collections.abc for common .NET types
+https://docs.python.org/3.6/library/collections.abc.html
+"""
+
+import collections.abc as col
+
+class IteratorMixin(col.Iterator):
+    def close(self):
+        self.Dispose()
+
+class IterableMixin(col.Iterable):
+    pass
+
+class SizedMixin(col.Sized):
+    def __len__(self): return self.Count
+
+class ContainerMixin(col.Container):
+    def __contains__(self, item): return self.Contains(item)
+
+try:
+    abc_Collection = col.Collection
+except AttributeError:
+    # Python 3.5- does not have collections.abc.Collection
+    abc_Collection = col.Container
+
+class CollectionMixin(SizedMixin, IterableMixin, ContainerMixin, abc_Collection):
+    pass
+
+class SequenceMixin(CollectionMixin, col.Sequence):
+    pass
+
+class MutableSequenceMixin(SequenceMixin, col.MutableSequence):
+    pass
+
+class MappingMixin(CollectionMixin, col.Mapping):
+    def __contains__(self, item): return self.ContainsKey(item)
+    def keys(self): return self.Keys
+    def items(self): return [(k,self.get(k)) for k in self.Keys]
+    def values(self): return self.Values
+    def __iter__(self): return self.Keys.__iter__()
+    def get(self, key, default=None):
+        existed, item = self.TryGetValue(key, None)
+        return item if existed else default
+
+class MutableMappingMixin(MappingMixin, col.MutableMapping):
+    _UNSET_ = object()
+
+    def __delitem__(self, key):
+        self.Remove(key)
+
+    def clear(self):
+        self.Clear()
+
+    def pop(self, key, default=_UNSET_):
+        existed, item = self.TryGetValue(key, None)
+        if existed:
+            self.Remove(key)
+            return item
+        elif default == self._UNSET_:
+            raise KeyError(key)
+        else:
+            return default
+
+    def setdefault(self, key, value=None):
+        existed, item = self.TryGetValue(key, None)
+        if existed:
+            return item
+        else:
+            self[key] = value
+            return value
+
+    def update(self, items, **kwargs):
+        if isinstance(items, col.Mapping):
+            for key, value in items.items():
+                self[key] = value
+        else:
+            for key, value in items:
+                self[key] = value
+
+        for key, value in kwargs.items():
+            self[key] = value
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -39,13 +39,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="resources\clr.py" />
     <EmbeddedResource Include="resources\clr.py">
       <LogicalName>clr.py</LogicalName>
     </EmbeddedResource>
     <EmbeddedResource Include="resources\interop.py">
       <LogicalName>interop.py</LogicalName>
     </EmbeddedResource>
+    <EmbeddedResource Include="Mixins\*.py" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs
@@ -12,6 +12,9 @@ internal static class Util
         internal const string MinimalPythonVersionRequired =
             "Only Python 3.5 or newer is supported";
 
+        internal const string UseOverloadWithReferenceTypes =
+            "This API is unsafe, and will be removed in the future. Use overloads working with *Reference types";
+
         internal static Int64 ReadCLong(IntPtr tp, int offset)
         {
             // On Windows, a C long is always 32 bits.

diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs
@@ -360,18 +360,40 @@ public static void tp_dealloc(IntPtr ob)
 
         public static int tp_clear(IntPtr ob)
         {
-            ManagedType self = GetManagedObject(ob);
+            if (GetManagedObject(ob) is { } self)
+            {
+                if (self.clearReentryGuard) return 0;
+
+                // workaround for https://bugs.python.org/issue45266
+                self.clearReentryGuard = true;
+
+                try
+                {
+                    return ClearImpl(ob, self);
+                }
+                finally
+                {
+                    self.clearReentryGuard = false;
+                }
+            }
+            else
+            {
+                return ClearImpl(ob, null);
+            }
+        }
 
+        static int ClearImpl(IntPtr ob, ManagedType self)
+        {
             bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType;
             if (!isTypeObject)
             {
-                ClearObjectDict(ob);
-
                 int baseClearResult = BaseUnmanagedClear(ob);
                 if (baseClearResult != 0)
                 {
                     return baseClearResult;
                 }
+
+                ClearObjectDict(ob);
             }
             return 0;
         }

diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs
@@ -12,7 +12,7 @@ internal class CLRObject : ManagedType
 
         internal CLRObject(object ob, IntPtr tp)
         {
-            System.Diagnostics.Debug.Assert(tp != IntPtr.Zero);
+            Debug.Assert(tp != IntPtr.Zero);
             IntPtr py = Runtime.PyType_GenericAlloc(tp, 0);
 
             tpHandle = tp;

diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs
@@ -28,6 +28,8 @@ internal enum TrackTypes
         internal IntPtr pyHandle; // PyObject *
         internal IntPtr tpHandle; // PyType *
 
+        internal bool clearReentryGuard;
+
         internal BorrowedReference ObjectReference
         {
             get
@@ -160,7 +162,7 @@ internal static bool IsInstanceOfManagedType(IntPtr ob)
 
         internal static bool IsManagedType(BorrowedReference type)
         {
-            var flags = (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags);
+            var flags = PyType.GetFlags(type);
             return (flags & TypeFlags.HasClrInstance) != 0;
         }
 

diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs
@@ -66,7 +66,7 @@ private PyScope(IntPtr ptr, PyScopeManager manager) : base(ptr)
             PythonException.ThrowIfIsNull(variables);
 
             int res = Runtime.PyDict_SetItem(
-                VarsRef, PyIdentifier.__builtins__,
+                VarsRef, new BorrowedReference(PyIdentifier.__builtins__),
                 Runtime.PyEval_GetBuiltins()
             );
             PythonException.ThrowIfIsNotZero(res);

diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs
@@ -224,10 +224,8 @@ public static void Initialize(IEnumerable<string> args, bool setSysArgv = true,
             var locals = new PyDict();
             try
             {
-                BorrowedReference module = Runtime.PyImport_AddModule("clr._extras");
+                BorrowedReference module = DefineModule("clr._extras");
                 BorrowedReference module_globals = Runtime.PyModule_GetDict(module);
-                BorrowedReference builtins = Runtime.PyEval_GetBuiltins();
-                Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins);
 
                 Assembly assembly = Assembly.GetExecutingAssembly();
                 // add the contents of clr.py to the module
@@ -236,6 +234,8 @@ public static void Initialize(IEnumerable<string> args, bool setSysArgv = true,
 
                 LoadSubmodule(module_globals, "clr.interop", "interop.py");
 
+                    LoadMixins(module_globals);
+
                 // add the imported module to the clr module, and copy the API functions
                 // and decorators into the main clr module.
                 Runtime.PyDict_SetItemString(clr_dict, "_extras", module);
@@ -281,6 +281,16 @@ static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, s
             Runtime.PyDict_SetItemString(targetModuleDict, memberName, module);
         }
 
+        static void LoadMixins(BorrowedReference targetModuleDict)
+        {
+            foreach (string nested in new[] { "collections" })
+            {
+                LoadSubmodule(targetModuleDict,
+                    fullName: "clr._extras." + nested,
+                    resourceName: typeof(PythonEngine).Namespace + ".Mixins." + nested + ".py");
+            }
+        }
+
         static void OnDomainUnload(object _, EventArgs __)
         {
             Shutdown();
@@ -641,7 +651,7 @@ internal static PyObject RunString(string code, BorrowedReference globals, Borro
                 {
                     globals = tempGlobals = NewReference.DangerousFromPointer(Runtime.PyDict_New());
                     Runtime.PyDict_SetItem(
-                        globals, PyIdentifier.__builtins__,
+                        globals, new BorrowedReference(PyIdentifier.__builtins__),
                         Runtime.PyEval_GetBuiltins()
                     );
                 }

diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs
@@ -103,6 +103,12 @@ internal IntPtr GetSlot(TypeSlotID slot)
             return Exceptions.ErrorCheckIfNull(result);
         }
 
+        internal static TypeFlags GetFlags(BorrowedReference type)
+        {
+            Debug.Assert(TypeOffset.tp_flags > 0);
+            return (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags);
+        }
+
         internal static BorrowedReference GetBase(BorrowedReference type)
         {
             Debug.Assert(IsType(type));

diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
@@ -1689,6 +1689,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer
         /// <summary>
         /// Return 0 on success or -1 on failure.
         /// </summary>
+        [Obsolete]
         internal static int PyDict_SetItem(BorrowedReference dict, IntPtr key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, new BorrowedReference(key), value);
         /// <summary>
         /// Return 0 on success or -1 on failure.
@@ -2052,7 +2053,7 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe
         internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases);
 
         /// <summary>
-        /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error.
+        /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error.
         /// </summary>
 
         internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type);

diff --git a/tests/test_collection_mixins.py b/tests/test_collection_mixins.py
@@ -0,0 +1,16 @@
+import System.Collections.Generic as C
+
+def test_contains():
+    l = C.List[int]()
+    l.Add(42)
+    assert 42 in l
+    assert 43 not in l
+
+def test_dict_items():
+    d = C.Dictionary[int, str]()
+    d[42] = "a"
+    items = d.items()
+    assert len(items) == 1
+    k,v = items[0]
+    assert k == 42
+    assert v == "a"