diff --git a/CHANGELOG.md b/CHANGELOG.md index ce9102a5d..c769796f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Improved exception handling: - exceptions can now be converted with codecs - `InnerException` and `__cause__` are propagated properly +- .NET collection types now implement standard Python collection interfaces from `collections.abc`. +See [Mixins/collections.py](src/runtime/Mixins/collections.py). - .NET arrays implement Python buffer protocol diff --git a/src/runtime/InteropConfiguration.cs b/src/runtime/InteropConfiguration.cs index 6853115fe..78af5037a 100644 --- a/src/runtime/InteropConfiguration.cs +++ b/src/runtime/InteropConfiguration.cs @@ -3,6 +3,8 @@ namespace Python.Runtime using System; using System.Collections.Generic; + using Python.Runtime.Mixins; + public sealed class InteropConfiguration { internal readonly PythonBaseTypeProviderGroup pythonBaseTypeProviders @@ -18,6 +20,7 @@ public static InteropConfiguration MakeDefault() PythonBaseTypeProviders = { DefaultBaseTypeProvider.Instance, + new CollectionMixinsProvider(new Lazy(() => Py.Import("clr._extras.collections"))), }, }; } diff --git a/src/runtime/Mixins/CollectionMixinsProvider.cs b/src/runtime/Mixins/CollectionMixinsProvider.cs new file mode 100644 index 000000000..48ea35f1c --- /dev/null +++ b/src/runtime/Mixins/CollectionMixinsProvider.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Python.Runtime.Mixins +{ + class CollectionMixinsProvider : IPythonBaseTypeProvider + { + readonly Lazy mixinsModule; + public CollectionMixinsProvider(Lazy mixinsModule) + { + this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule)); + } + + public PyObject Mixins => this.mixinsModule.Value; + + public IEnumerable GetBaseTypes(Type type, IList existingBases) + { + if (type is null) + throw new ArgumentNullException(nameof(type)); + + if (existingBases is null) + throw new ArgumentNullException(nameof(existingBases)); + + var interfaces = NewInterfaces(type).Select(GetDefinition).ToArray(); + + var newBases = new List(); + newBases.AddRange(existingBases); + + // dictionaries + if (interfaces.Contains(typeof(IDictionary<,>))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("MutableMappingMixin"))); + } + else if (interfaces.Contains(typeof(IReadOnlyDictionary<,>))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("MappingMixin"))); + } + + // item collections + if (interfaces.Contains(typeof(IList<>)) + || interfaces.Contains(typeof(System.Collections.IList))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("MutableSequenceMixin"))); + } + else if (interfaces.Contains(typeof(IReadOnlyList<>))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("SequenceMixin"))); + } + else if (interfaces.Contains(typeof(ICollection<>)) + || interfaces.Contains(typeof(System.Collections.ICollection))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("CollectionMixin"))); + } + else if (interfaces.Contains(typeof(System.Collections.IEnumerable))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("IterableMixin"))); + } + + // enumerators + if (interfaces.Contains(typeof(System.Collections.IEnumerator))) + { + newBases.Add(new PyType(this.Mixins.GetAttr("IteratorMixin"))); + } + + if (newBases.Count == existingBases.Count) + { + return existingBases; + } + + if (type.IsInterface && type.BaseType is null) + { + newBases.RemoveAll(@base => @base.Handle == Runtime.PyBaseObjectType); + } + + return newBases; + } + + static Type[] NewInterfaces(Type type) + { + var result = type.GetInterfaces(); + return type.BaseType != null + ? result.Except(type.BaseType.GetInterfaces()).ToArray() + : result; + } + + static Type GetDefinition(Type type) + => type.IsGenericType ? type.GetGenericTypeDefinition() : type; + } +} diff --git a/src/runtime/Mixins/collections.py b/src/runtime/Mixins/collections.py new file mode 100644 index 000000000..a82032472 --- /dev/null +++ b/src/runtime/Mixins/collections.py @@ -0,0 +1,82 @@ +""" +Implements collections.abc for common .NET types +https://docs.python.org/3.6/library/collections.abc.html +""" + +import collections.abc as col + +class IteratorMixin(col.Iterator): + def close(self): + self.Dispose() + +class IterableMixin(col.Iterable): + pass + +class SizedMixin(col.Sized): + def __len__(self): return self.Count + +class ContainerMixin(col.Container): + def __contains__(self, item): return self.Contains(item) + +try: + abc_Collection = col.Collection +except AttributeError: + # Python 3.5- does not have collections.abc.Collection + abc_Collection = col.Container + +class CollectionMixin(SizedMixin, IterableMixin, ContainerMixin, abc_Collection): + pass + +class SequenceMixin(CollectionMixin, col.Sequence): + pass + +class MutableSequenceMixin(SequenceMixin, col.MutableSequence): + pass + +class MappingMixin(CollectionMixin, col.Mapping): + def __contains__(self, item): return self.ContainsKey(item) + def keys(self): return self.Keys + def items(self): return [(k,self.get(k)) for k in self.Keys] + def values(self): return self.Values + def __iter__(self): return self.Keys.__iter__() + def get(self, key, default=None): + existed, item = self.TryGetValue(key, None) + return item if existed else default + +class MutableMappingMixin(MappingMixin, col.MutableMapping): + _UNSET_ = object() + + def __delitem__(self, key): + self.Remove(key) + + def clear(self): + self.Clear() + + def pop(self, key, default=_UNSET_): + existed, item = self.TryGetValue(key, None) + if existed: + self.Remove(key) + return item + elif default == self._UNSET_: + raise KeyError(key) + else: + return default + + def setdefault(self, key, value=None): + existed, item = self.TryGetValue(key, None) + if existed: + return item + else: + self[key] = value + return value + + def update(self, items, **kwargs): + if isinstance(items, col.Mapping): + for key, value in items.items(): + self[key] = value + else: + for key, value in items: + self[key] = value + + for key, value in kwargs.items(): + self[key] = value diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 79778aabf..dfea71e81 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -39,13 +39,13 @@ - clr.py interop.py + diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index 193e57520..04bc631bb 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -12,6 +12,9 @@ internal static class Util internal const string MinimalPythonVersionRequired = "Only Python 3.5 or newer is supported"; + internal const string UseOverloadWithReferenceTypes = + "This API is unsafe, and will be removed in the future. Use overloads working with *Reference types"; + internal static Int64 ReadCLong(IntPtr tp, int offset) { // On Windows, a C long is always 32 bits. diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index cf797ff30..570ce3062 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -360,18 +360,40 @@ public static void tp_dealloc(IntPtr ob) public static int tp_clear(IntPtr ob) { - ManagedType self = GetManagedObject(ob); + if (GetManagedObject(ob) is { } self) + { + if (self.clearReentryGuard) return 0; + + // workaround for https://bugs.python.org/issue45266 + self.clearReentryGuard = true; + + try + { + return ClearImpl(ob, self); + } + finally + { + self.clearReentryGuard = false; + } + } + else + { + return ClearImpl(ob, null); + } + } + static int ClearImpl(IntPtr ob, ManagedType self) + { bool isTypeObject = Runtime.PyObject_TYPE(ob) == Runtime.PyCLRMetaType; if (!isTypeObject) { - ClearObjectDict(ob); - int baseClearResult = BaseUnmanagedClear(ob); if (baseClearResult != 0) { return baseClearResult; } + + ClearObjectDict(ob); } return 0; } diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 07b816e05..114cce070 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -12,7 +12,7 @@ internal class CLRObject : ManagedType internal CLRObject(object ob, IntPtr tp) { - System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); + Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); tpHandle = tp; diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index d09088e79..2fe177f93 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -28,6 +28,8 @@ internal enum TrackTypes internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * + internal bool clearReentryGuard; + internal BorrowedReference ObjectReference { get @@ -160,7 +162,7 @@ internal static bool IsInstanceOfManagedType(IntPtr ob) internal static bool IsManagedType(BorrowedReference type) { - var flags = (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags); + var flags = PyType.GetFlags(type); return (flags & TypeFlags.HasClrInstance) != 0; } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8cb40d781..315eb75e5 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -66,7 +66,7 @@ private PyScope(IntPtr ptr, PyScopeManager manager) : base(ptr) PythonException.ThrowIfIsNull(variables); int res = Runtime.PyDict_SetItem( - VarsRef, PyIdentifier.__builtins__, + VarsRef, new BorrowedReference(PyIdentifier.__builtins__), Runtime.PyEval_GetBuiltins() ); PythonException.ThrowIfIsNotZero(res); diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 8da8ea5f7..d7322dcc2 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -224,10 +224,8 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, var locals = new PyDict(); try { - BorrowedReference module = Runtime.PyImport_AddModule("clr._extras"); + BorrowedReference module = DefineModule("clr._extras"); BorrowedReference module_globals = Runtime.PyModule_GetDict(module); - BorrowedReference builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); Assembly assembly = Assembly.GetExecutingAssembly(); // add the contents of clr.py to the module @@ -236,6 +234,8 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, LoadSubmodule(module_globals, "clr.interop", "interop.py"); + LoadMixins(module_globals); + // add the imported module to the clr module, and copy the API functions // and decorators into the main clr module. Runtime.PyDict_SetItemString(clr_dict, "_extras", module); @@ -281,6 +281,16 @@ static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, s Runtime.PyDict_SetItemString(targetModuleDict, memberName, module); } + static void LoadMixins(BorrowedReference targetModuleDict) + { + foreach (string nested in new[] { "collections" }) + { + LoadSubmodule(targetModuleDict, + fullName: "clr._extras." + nested, + resourceName: typeof(PythonEngine).Namespace + ".Mixins." + nested + ".py"); + } + } + static void OnDomainUnload(object _, EventArgs __) { Shutdown(); @@ -641,7 +651,7 @@ internal static PyObject RunString(string code, BorrowedReference globals, Borro { globals = tempGlobals = NewReference.DangerousFromPointer(Runtime.PyDict_New()); Runtime.PyDict_SetItem( - globals, PyIdentifier.__builtins__, + globals, new BorrowedReference(PyIdentifier.__builtins__), Runtime.PyEval_GetBuiltins() ); } diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs index 78cfad3f2..b144d09c3 100644 --- a/src/runtime/pytype.cs +++ b/src/runtime/pytype.cs @@ -103,6 +103,12 @@ internal IntPtr GetSlot(TypeSlotID slot) return Exceptions.ErrorCheckIfNull(result); } + internal static TypeFlags GetFlags(BorrowedReference type) + { + Debug.Assert(TypeOffset.tp_flags > 0); + return (TypeFlags)Util.ReadCLong(type.DangerousGetAddress(), TypeOffset.tp_flags); + } + internal static BorrowedReference GetBase(BorrowedReference type) { Debug.Assert(IsType(type)); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index acdf86c4e..261aacd72 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1689,6 +1689,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer /// /// Return 0 on success or -1 on failure. /// + [Obsolete] internal static int PyDict_SetItem(BorrowedReference dict, IntPtr key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, new BorrowedReference(key), value); /// /// Return 0 on success or -1 on failure. @@ -2052,7 +2053,7 @@ internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedRe internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error. /// internal static int PyType_Ready(IntPtr type) => Delegates.PyType_Ready(type); diff --git a/tests/test_collection_mixins.py b/tests/test_collection_mixins.py new file mode 100644 index 000000000..2f74e93ab --- /dev/null +++ b/tests/test_collection_mixins.py @@ -0,0 +1,16 @@ +import System.Collections.Generic as C + +def test_contains(): + l = C.List[int]() + l.Add(42) + assert 42 in l + assert 43 not in l + +def test_dict_items(): + d = C.Dictionary[int, str]() + d[42] = "a" + items = d.items() + assert len(items) == 1 + k,v = items[0] + assert k == 42 + assert v == "a"