diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 692fd5700..4f162dcd5 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -63,7 +63,7 @@ def unload(): global _RUNTIME if _LOADER_ASSEMBLY is not None: func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Shutdown"] - if func(b"") != 0: + if func(b"full_shutdown") != 0: raise RuntimeError("Failed to call Python.NET shutdown") if _RUNTIME is not None: diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 67a7d3338..099d026fa 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -2,6 +2,7 @@ net472;netcoreapp3.1 + 9.0 ..\pythonnet.snk true diff --git a/src/embed_tests/TestClass.cs b/src/embed_tests/TestClass.cs new file mode 100644 index 000000000..cbcb92483 --- /dev/null +++ b/src/embed_tests/TestClass.cs @@ -0,0 +1,78 @@ +using System; +using System.Runtime.InteropServices; + +using NUnit.Framework; + +using Python.Runtime; + +using PyRuntime = Python.Runtime.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestClass + { + public class MyClass + { + } + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void WeakRefForClrObject() + { + var obj = new MyClass(); + using var scope = Py.CreateScope(); + scope.Set("clr_obj", obj); + scope.Exec(@" +import weakref +ref = weakref.ref(clr_obj) +"); + using PyObject pyobj = scope.Get("clr_obj"); + ValidateAttachedGCHandle(obj, pyobj.Handle); + } + + [Test] + public void WeakRefForSubClass() + { + using (var scope = Py.CreateScope()) + { + scope.Exec(@" +from Python.EmbeddingTest import TestClass +import weakref + +class Sub(TestClass.MyClass): + pass + +obj = Sub() +ref = weakref.ref(obj) +"); + using (PyObject pyobj = scope.Get("obj")) + { + IntPtr op = pyobj.Handle; + IntPtr type = PyRuntime.PyObject_TYPE(op); + IntPtr clrHandle = Marshal.ReadIntPtr(op, ObjectOffset.magic(type)); + var clobj = (CLRObject)GCHandle.FromIntPtr(clrHandle).Target; + Assert.IsTrue(clobj.inst is MyClass); + } + } + } + + private static void ValidateAttachedGCHandle(object obj, IntPtr op) + { + IntPtr type = PyRuntime.PyObject_TYPE(op); + IntPtr clrHandle = Marshal.ReadIntPtr(op, ObjectOffset.magic(type)); + var clobj = (CLRObject)GCHandle.FromIntPtr(clrHandle).Target; + Assert.True(ReferenceEquals(clobj.inst, obj)); + } + } +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 8b96a96da..a29e3a9b2 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,9 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Runtime.InteropServices; -using System.Runtime.Serialization; namespace Python.Runtime { @@ -354,44 +352,39 @@ public static IntPtr tp_repr(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - tp_clear(ob); - Runtime.PyObject_GC_UnTrack(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); + if (Runtime.PyType_SUPPORTS_WEAKREFS(Runtime.PyObject_TYPE(ob))) + { + Runtime.PyObject_ClearWeakRefs(ob); + } + RemoveObjectDict(ob); + Runtime.Py_CLEAR(ref self.tpHandle); + Runtime.PyObject_GC_UnTrack(ob); + Runtime.PyObject_GC_Del(ob); self.FreeGCHandle(); } public static int tp_clear(IntPtr ob) { - ManagedType self = GetManagedObject(ob); - if (!self.IsTypeObject()) - { - ClearObjectDict(ob); - } - self.tpHandle = IntPtr.Zero; + ClearObjectDict(ob); return 0; } protected override void OnSave(InterDomainContext context) { base.OnSave(context); - if (pyHandle != tpHandle) - { - IntPtr dict = GetObjectDict(pyHandle); - Runtime.XIncref(dict); - context.Storage.AddValue("dict", dict); - } + IntPtr dict = GetObjectDict(pyHandle); + Runtime.XIncref(dict); + Runtime.XIncref(tpHandle); + context.Storage.AddValue("dict", dict); } protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); - if (pyHandle != tpHandle) - { - IntPtr dict = context.Storage.GetValue("dict"); - SetObjectDict(pyHandle, dict); - } + IntPtr dict = context.Storage.GetValue("dict"); + SetObjectDict(pyHandle, dict); gcHandle = AllocGCHandle(); - Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gcHandle); } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 1ee06e682..5cad4bd69 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Linq; +using System.Diagnostics; namespace Python.Runtime { @@ -270,7 +271,7 @@ private static void InitClassBase(Type type, ClassBase impl) // Finally, initialize the class __dict__ and return the object. var dict = new BorrowedReference(Marshal.ReadIntPtr(tp, TypeOffset.tp_dict)); - + Debug.Assert(!dict.IsNull); if (impl.dotNetMembers == null) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0a352b381..9c288b145 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -30,10 +30,6 @@ internal CLRObject(object ob, IntPtr tp) tpHandle = tp; pyHandle = py; inst = ob; - - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); } protected CLRObject() @@ -49,7 +45,7 @@ static CLRObject GetInstance(object ob, IntPtr pyType) static CLRObject GetInstance(object ob) { ClassBase cc = ClassManager.GetClass(ob.GetType()); - return GetInstance(ob, cc.tpHandle); + return GetInstance(ob, cc.pyHandle); } internal static NewReference GetInstHandle(object ob, BorrowedReference pyType) @@ -67,7 +63,7 @@ internal static IntPtr GetInstHandle(object ob, IntPtr pyType) internal static IntPtr GetInstHandle(object ob, Type type) { ClassBase cc = ClassManager.GetClass(type); - CLRObject co = GetInstance(ob, cc.tpHandle); + CLRObject co = GetInstance(ob, cc.pyHandle); return co.pyHandle; } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index afd0bc14e..51c7729e3 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -85,6 +85,13 @@ internal static Exception ToException(IntPtr ob) } return Runtime.PyUnicode_FromString(message); } + + public static int tp_init(IntPtr ob, IntPtr args, IntPtr kwds) + { + Exceptions.SetArgsAndCause(ob); + return 0; + } + } /// @@ -180,16 +187,23 @@ internal static void SetArgsAndCause(IntPtr ob) args = Runtime.PyTuple_New(0); } - Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); - + int baseOffset = OriginalObjectOffsets.Size; + Runtime.Py_SETREF(ob, baseOffset + ExceptionOffset.args, args); + if (e.InnerException != null) { - // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. - IntPtr cause = CLRObject.GetInstHandle(e.InnerException); - Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); + IntPtr cause = GetExceptHandle(e.InnerException); + Runtime.Py_SETREF(ob, baseOffset + ExceptionOffset.cause, cause); } } + internal static IntPtr GetExceptHandle(Exception e) + { + IntPtr op = CLRObject.GetInstHandle(e); + SetArgsAndCause(op); + return op; + } + /// /// Shortcut for (pointer == NULL) -> throw PythonException /// @@ -289,7 +303,7 @@ public static void SetError(Exception e) return; } - IntPtr op = CLRObject.GetInstHandle(e); + IntPtr op = GetExceptHandle(e); IntPtr etype = Runtime.PyObject_GetAttr(op, PyIdentifier.__class__); Runtime.PyErr_SetObject(new BorrowedReference(etype), new BorrowedReference(op)); Runtime.XDecref(etype); diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index a5f0f1219..e6c95ecaa 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -55,7 +55,7 @@ void SetupGc () /// public static void FinalizeObject(ManagedType self) { - ClearObjectDict(self.pyHandle); + RemoveObjectDict(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); // Not necessary for decref of `tpHandle`. self.FreeGCHandle(); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 0f56f77d9..970ecb20d 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -68,27 +68,22 @@ public ModulePropertyAttribute() } } - internal static partial class TypeOffset - { - public static int magic() => ManagedDataOffsets.Magic; - } - internal static class ManagedDataOffsets { - public static int Magic { get; internal set; } public static readonly Dictionary NameMapping = new Dictionary(); static class DataOffsets { public static readonly int ob_data = 0; public static readonly int ob_dict = 0; + public static readonly int ob_weaklist = 0; static DataOffsets() { FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fields.Length; i++) { - fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + fields[i].SetValue(null, (i) * IntPtr.Size); } } } @@ -98,7 +93,7 @@ static ManagedDataOffsets() NameMapping = TypeOffset.GetOffsets(); FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - size = fields.Length * IntPtr.Size; + Size = fields.Length * IntPtr.Size; } public static int GetSlotOffset(string name) @@ -106,29 +101,10 @@ public static int GetSlotOffset(string name) return NameMapping[name]; } - private static int BaseOffset(IntPtr type) - { - Debug.Assert(type != IntPtr.Zero); - int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0); - return typeSize; - } - - public static int DataOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_data; - } - - public static int DictOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_dict; - } - public static int ob_data => DataOffsets.ob_data; public static int ob_dict => DataOffsets.ob_dict; - public static int Size { get { return size; } } - - private static readonly int size; + public static int ob_weaklist => DataOffsets.ob_weaklist; + public static int Size { get; private set; } } internal static class OriginalObjectOffsets @@ -163,6 +139,26 @@ static OriginalObjectOffsets() public static int ob_type; } + + internal static class ManagedExceptionOffset + { + public static int ob_data { get; private set; } + public static int ob_dict { get; private set; } + public static int ob_weaklist { get; private set; } + + public static int Size { get; private set; } + + static ManagedExceptionOffset() + { + int baseOffset = OriginalObjectOffsets.Size + ExceptionOffset.Size(); + ob_data = baseOffset + ManagedDataOffsets.ob_data; + ob_dict = baseOffset + ManagedDataOffsets.ob_dict; + ob_weaklist = baseOffset + ManagedDataOffsets.ob_weaklist; + Size = ob_weaklist + IntPtr.Size; + } + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] internal class ObjectOffset { @@ -175,27 +171,17 @@ static ObjectOffset() ob_refcnt = OriginalObjectOffsets.ob_refcnt; ob_type = OriginalObjectOffsets.ob_type; - size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + Size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } public static int magic(IntPtr type) { - return ManagedDataOffsets.DataOffset(type); + return (int)Marshal.ReadIntPtr(type, TypeOffset.Size); } public static int TypeDictOffset(IntPtr type) { - return ManagedDataOffsets.DictOffset(type); - } - - public static int Size(IntPtr pyType) - { - if (IsException(pyType)) - { - return ExceptionOffset.Size(); - } - - return size; + return (int)Marshal.ReadIntPtr(type, TypeOffset.tp_dictoffset); } #if PYTHON_WITH_PYDEBUG @@ -204,16 +190,8 @@ public static int Size(IntPtr pyType) #endif public static int ob_refcnt; public static int ob_type; - private static readonly int size; - private static bool IsException(IntPtr pyObjectPtr) - { - var pyObject = new BorrowedReference(pyObjectPtr); - var type = Runtime.PyObject_TYPE(pyObject); - return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) - || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - } + public static int Size { get; private set; } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -225,10 +203,9 @@ static ExceptionOffset() FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); + fi[i].SetValue(null, i * IntPtr.Size); } - - size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + size = fi.Length * IntPtr.Size; } public static int Size() { return size; } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 09e8a3be2..12aa7104f 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -26,7 +26,6 @@ internal enum TrackTypes internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * - internal BorrowedReference ObjectReference => new BorrowedReference(pyHandle); private static readonly Dictionary _managedObjs = new Dictionary(); @@ -84,29 +83,28 @@ internal static ManagedType GetManagedObject(BorrowedReference ob) /// internal static ManagedType GetManagedObject(IntPtr ob) { - if (ob != IntPtr.Zero) + if (ob == IntPtr.Zero) { - IntPtr tp = Runtime.PyObject_TYPE(ob); - if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) - { - tp = ob; - } - - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - IntPtr op = tp == ob - ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); - if (op == IntPtr.Zero) - { - return null; - } - var gc = (GCHandle)op; - return (ManagedType)gc.Target; - } + return null; + } + IntPtr tp = Runtime.PyObject_TYPE(ob); + var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + if ((flags & TypeFlags.Managed) == 0) + { + return null; + } + int offset = ObjectOffset.magic(tp); + if (offset == 0) + { + return null; } - return null; + IntPtr op = Marshal.ReadIntPtr(ob, offset); + if (op == IntPtr.Zero) + { + return null; + } + var gc = (GCHandle)op; + return (ManagedType)gc.Target; } /// @@ -114,18 +112,12 @@ internal static ManagedType GetManagedObject(IntPtr ob) /// internal static ManagedType GetManagedObjectType(IntPtr ob) { - if (ob != IntPtr.Zero) + if (ob == IntPtr.Zero) { - IntPtr tp = Runtime.PyObject_TYPE(ob); - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - var gc = (GCHandle)tp; - return (ManagedType)gc.Target; - } + return null; } - return null; + IntPtr tp = Runtime.PyObject_TYPE(ob); + return GetManagedObject(tp); } @@ -147,11 +139,6 @@ internal static bool IsManagedType(IntPtr ob) if (ob != IntPtr.Zero) { IntPtr tp = Runtime.PyObject_TYPE(ob); - if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) - { - tp = ob; - } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { @@ -161,11 +148,6 @@ internal static bool IsManagedType(IntPtr ob) return false; } - public bool IsTypeObject() - { - return pyHandle == tpHandle; - } - internal static IDictionary GetManagedObjects() { return _managedObjs; @@ -243,6 +225,16 @@ protected virtual void OnSave(InterDomainContext context) { } protected virtual void OnLoad(InterDomainContext context) { } protected static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + Runtime.PyDict_Clear(dict); + } + + protected static void RemoveObjectDict(IntPtr ob) { IntPtr dict = GetObjectDict(ob); if (dict == IntPtr.Zero) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 36b406c7b..3f4717190 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -154,38 +155,17 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) flags |= TypeFlags.Subclass; flags |= TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - - TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); - - // Hmm - the standard subtype_traverse, clear look at ob_size to - // do things, so to allow gc to work correctly we need to move - // our hidden handle out of ob_size. Then, in theory we can - // comment this out and still not crash. - TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); - TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); - - // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); - - return type; - } - - - public static IntPtr tp_alloc(IntPtr mt, int n) - { - IntPtr type = Runtime.PyType_GenericAlloc(mt, n); + unsafe + { + var baseTypeEx = ClrMetaTypeEx.FromType(base_type); + var typeEx = ClrMetaTypeEx.FromType(type); + typeEx->ClrHandleOffset = (IntPtr)(OriginalObjectOffsets.Size + ManagedDataOffsets.ob_data); + typeEx->ClrHandle = baseTypeEx->ClrHandle; + } return type; } - - public static void tp_free(IntPtr tp) - { - Runtime.PyObject_GC_Del(tp); - } - - /// /// Metatype __call__ implementation. This is needed to ensure correct /// initialization (__init__ support), because the tp_call we inherit @@ -288,7 +268,7 @@ public static void tp_dealloc(IntPtr tp) var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { - IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); + IntPtr gc = Marshal.ReadIntPtr(tp, ObjectOffset.magic(Runtime.PyObject_TYPE(tp))); ((GCHandle)gc).Free(); } @@ -304,6 +284,12 @@ public static void tp_dealloc(IntPtr tp) NativeCall.Void_Call_1(op, tp); } + public static int tp_clear(IntPtr ob) + { + ClearObjectDict(ob); + return 0; + } + private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) { var cb = GetManagedObject(tp) as ClassBase; @@ -360,4 +346,18 @@ public static IntPtr __subclasscheck__(IntPtr tp, IntPtr args) return DoInstanceCheck(tp, args, true); } } + + + [StructLayout(LayoutKind.Sequential)] + struct ClrMetaTypeEx + { + public nint ClrHandleOffset; + public IntPtr ClrHandle; + + public static unsafe ClrMetaTypeEx* FromType(IntPtr type) + { + return (ClrMetaTypeEx*)(type + TypeOffset.Size); + } + } + } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 37c01f5c5..630f286da 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -217,7 +217,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); self.ClearMembers(); - ClearObjectDict(ob); + RemoveObjectDict(ob); self.Dealloc(); } diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index 3264531de..21890d57e 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -29,8 +29,6 @@ internal static void Initialize(Version version, BorrowedReference pyType) } var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); TypeOffset.Use(typeOffsets); - - ManagedDataOffsets.Magic = Marshal.ReadInt32(pyType.DangerousGetAddress(), TypeOffset.tp_basicsize); } } } diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 485c041f8..438e96fe8 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -30,6 +30,7 @@ interface ITypeOffsets int nb_inplace_add { get; } int nb_inplace_subtract { get; } int ob_size { get; } + int ob_refcnt { get; } int ob_type { get; } int qualname { get; } int sq_contains { get; } @@ -49,6 +50,7 @@ interface ITypeOffsets int tp_descr_set { get; } int tp_dict { get; } int tp_dictoffset { get; } + int tp_init { get; } int tp_flags { get; } int tp_free { get; } int tp_getattro { get; } @@ -63,6 +65,7 @@ interface ITypeOffsets int tp_new { get; } int tp_repr { get; } int tp_richcompare { get; } + int tp_weaklistoffset { get; } int tp_setattro { get; } int tp_str { get; } int tp_traverse { get; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 9f5ed671b..62bd90b4a 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -8,6 +8,7 @@ namespace Python.Runtime using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; + using System.Runtime.InteropServices; using Python.Runtime.Native; @@ -37,6 +38,7 @@ static partial class TypeOffset internal static int nb_inplace_add { get; private set; } internal static int nb_inplace_subtract { get; private set; } internal static int ob_size { get; private set; } + internal static int ob_refcnt { get; private set; } internal static int ob_type { get; private set; } internal static int qualname { get; private set; } internal static int sq_contains { get; private set; } @@ -56,6 +58,7 @@ static partial class TypeOffset internal static int tp_descr_set { get; private set; } internal static int tp_dict { get; private set; } internal static int tp_dictoffset { get; private set; } + internal static int tp_init { get; private set; } internal static int tp_flags { get; private set; } internal static int tp_free { get; private set; } internal static int tp_getattro { get; private set; } @@ -70,10 +73,13 @@ static partial class TypeOffset internal static int tp_new { get; private set; } internal static int tp_repr { get; private set; } internal static int tp_richcompare { get; private set; } + internal static int tp_weaklistoffset { get; private set; } internal static int tp_setattro { get; private set; } internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } + internal static int Size; + internal static void Use(ITypeOffsets offsets) { if (offsets is null) throw new ArgumentNullException(nameof(offsets)); @@ -88,7 +94,7 @@ internal static void Use(ITypeOffsets offsets) int value = (int)sourceProperty.GetValue(offsets, null); offsetProperty.SetValue(obj: null, value: value, index: null); } - + Size = (int)Marshal.ReadIntPtr(Runtime.PyTypeType, tp_basicsize); ValidateUnusedTypeOffsetProperties(offsetProperties); ValidateRequiredOffsetsPresent(offsetProperties); } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3c7f13ec6..aaffa29fe 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -119,7 +119,9 @@ public static PyObject FromManagedObject(object ob) Runtime.XIncref(Runtime.PyNone); return new PyObject(Runtime.PyNone); } - IntPtr op = CLRObject.GetInstHandle(ob); + IntPtr op = typeof(Exception).IsAssignableFrom(ob.GetType()) ? + Exceptions.GetExceptHandle((Exception)ob) + : CLRObject.GetInstHandle(ob); return new PyObject(op); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec7f5e446..cd581d381 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -348,8 +348,8 @@ internal static void Shutdown(ShutdownMode mode) ClearClrModules(); RemoveClrRootModule(); - MoveClrInstancesOnwershipToPython(); ClassManager.DisposePythonWrappersForClrTypes(); + MoveClrInstancesOnwershipToPython(); TypeManager.RemoveTypes(); MetaType.Release(); @@ -503,15 +503,19 @@ private static void PyDictTryDelItem(BorrowedReference dict, string key) private static void MoveClrInstancesOnwershipToPython() { var objs = ManagedType.GetManagedObjects(); - var copyObjs = objs.ToArray(); - foreach (var entry in copyObjs) + var copyObjs = new KeyValuePair[objs.Count]; { - ManagedType obj = entry.Key; - if (!objs.ContainsKey(obj)) + int i = 0; + foreach (var entry in objs) { - System.Diagnostics.Debug.Assert(obj.gcHandle == default); - continue; + ManagedType obj = entry.Key; + XIncref(obj.pyHandle); + copyObjs[i++] = entry; } + } + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; if (entry.Value == ManagedType.TrackTypes.Extension) { obj.CallTypeClear(); @@ -522,11 +526,21 @@ private static void MoveClrInstancesOnwershipToPython() PyObject_GC_Track(obj.pyHandle); } } - if (obj.gcHandle.IsAllocated) + } + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; + if (!objs.ContainsKey(obj)) { - obj.gcHandle.Free(); + System.Diagnostics.Debug.Assert(obj.gcHandle == default); + continue; + } + if (obj.RefCount > 1) + { + obj.FreeGCHandle(); + Marshal.WriteIntPtr(obj.pyHandle, ObjectOffset.magic(obj.tpHandle), IntPtr.Zero); } - obj.gcHandle = default; + XDecref(obj.pyHandle); } ManagedType.ClearTrackedObjects(); } @@ -2018,25 +2032,27 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) internal static IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name) => Delegates.PyObject_GenericGetAttr(obj, name); - + internal static bool PyType_SUPPORTS_WEAKREFS(IntPtr type) + { + return Marshal.ReadIntPtr(type, TypeOffset.tp_weaklistoffset) != IntPtr.Zero; + } internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - internal static BorrowedReference* _PyObject_GetDictPtr(BorrowedReference obj) => Delegates._PyObject_GetDictPtr(obj); - internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); + internal static IntPtr _PyObject_GC_Calloc(IntPtr basicsize) => Delegates._PyObject_GC_Calloc(basicsize); internal static void PyObject_GC_Track(IntPtr tp) => Delegates.PyObject_GC_Track(tp); - internal static void PyObject_GC_UnTrack(IntPtr tp) => Delegates.PyObject_GC_UnTrack(tp); - internal static void _PyObject_Dump(IntPtr ob) => Delegates._PyObject_Dump(ob); + internal static void PyObject_ClearWeakRefs(IntPtr obj) => Delegates.PyObject_ClearWeakRefs(obj); + //==================================================================== // Python memory API //==================================================================== @@ -2173,6 +2189,16 @@ internal static void Py_CLEAR(ref IntPtr ob) ob = IntPtr.Zero; } + internal static unsafe void Py_SETREF(IntPtr ob, int offset, IntPtr target) + { + var p = (void**)(ob + offset); + if (*p != null) + { + XDecref((IntPtr)(*p)); + } + *p = (void*)target; + } + //==================================================================== // Python Capsules API //==================================================================== @@ -2475,9 +2501,11 @@ static Delegates() PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); _PyObject_GetDictPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GetDictPtr), GetUnmanagedDll(_PythonDll)); PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); + _PyObject_GC_Calloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GC_Calloc), GetUnmanagedDll(_PythonDll)); PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); _PyObject_Dump = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_Dump), GetUnmanagedDll(_PythonDll)); + PyObject_ClearWeakRefs = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_ClearWeakRefs), GetUnmanagedDll(_PythonDll)); PyMem_Malloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Malloc), GetUnmanagedDll(_PythonDll)); PyMem_Realloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Realloc), GetUnmanagedDll(_PythonDll)); PyMem_Free = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Free), GetUnmanagedDll(_PythonDll)); @@ -2740,9 +2768,11 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } internal static delegate* unmanaged[Cdecl] _PyObject_GetDictPtr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } + internal static delegate* unmanaged[Cdecl] _PyObject_GC_Calloc { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } internal static delegate* unmanaged[Cdecl] _PyObject_Dump { get; } + internal static delegate* unmanaged[Cdecl] PyObject_ClearWeakRefs { get; } internal static delegate* unmanaged[Cdecl] PyMem_Malloc { get; } internal static delegate* unmanaged[Cdecl] PyMem_Realloc { get; } internal static delegate* unmanaged[Cdecl] PyMem_Free { get; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e0564b243..1975c2421 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -45,10 +45,10 @@ internal static void Initialize() internal static void RemoveTypes() { - foreach (var tpHandle in cache.Values) + foreach (var entry in _slotsHolders) { - SlotsHolder holder; - if (_slotsHolders.TryGetValue(tpHandle, out holder)) + IntPtr tpHandle = entry.Key; + SlotsHolder holder = entry.Value; { // If refcount > 1, it needs to reset the managed slot, // otherwise it can dealloc without any trick. @@ -160,13 +160,13 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) internal static IntPtr CreateType(Type impl) { IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); - int ob_size = ObjectOffset.Size(type); + int ob_size = ObjectOffset.Size; // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); + var offset = OriginalObjectOffsets.Size + ManagedDataOffsets.ob_dict; + Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)offset); SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); @@ -186,6 +186,11 @@ internal static IntPtr CreateType(Type impl) mod.Dispose(); InitMethods(type, impl); + unsafe + { + var typeEx = ClrMetaTypeEx.FromType(type); + typeEx->ClrHandleOffset = (IntPtr)OriginalObjectOffsets.Size + ManagedDataOffsets.ob_data; + } // The type has been modified after PyType_Ready has been called // Refresh the type @@ -210,17 +215,25 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) } IntPtr base_ = IntPtr.Zero; - int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - + int baseOffset = OriginalObjectOffsets.Size; + int ob_size, tp_dictoffset, tp_weaklistoffset, magicOffset; // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* // subclass BaseException (or better Exception). if (typeof(Exception).IsAssignableFrom(clrType)) { - ob_size = ObjectOffset.Size(Exceptions.Exception); + tp_dictoffset = ManagedExceptionOffset.ob_dict; + tp_weaklistoffset = 0; + ob_size = ManagedExceptionOffset.Size; + magicOffset = ManagedExceptionOffset.ob_data; + } + else + { + tp_dictoffset = baseOffset + ManagedDataOffsets.ob_dict; + tp_weaklistoffset = baseOffset + ManagedDataOffsets.ob_weaklist; + ob_size = baseOffset + ManagedDataOffsets.Size; + magicOffset = baseOffset + ManagedDataOffsets.ob_data; } - - int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; if (clrType == typeof(Exception)) { @@ -240,6 +253,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + Marshal.WriteIntPtr(type, TypeOffset.tp_weaklistoffset, (IntPtr)tp_weaklistoffset); // we want to do this after the slot stuff above in case the class itself implements a slot method SlotsHolder slotsHolder = CreateSolotsHolder(type); @@ -311,10 +325,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); - Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); + unsafe + { + var typePtr = ClrMetaTypeEx.FromType(type); + typePtr->ClrHandle = (IntPtr)gc; + typePtr->ClrHandleOffset = (IntPtr)magicOffset; + } // Set the handle attributes on the implementing instance. - impl.tpHandle = type; + impl.tpHandle = Runtime.PyCLRMetaType; + Runtime.XIncref(type); impl.pyHandle = type; //DebugUtil.DumpType(type); @@ -448,14 +468,19 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) // the standard type slots, and has to subclass PyType_Type for // certain functions in the C runtime to work correctly with it. - IntPtr type = AllocateTypeObject("CLR Metatype", metatype: Runtime.PyTypeType); - IntPtr py_type = Runtime.PyTypeType; - Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); + var heapTypeSize = (int)Marshal.ReadIntPtr(py_type, TypeOffset.tp_basicsize); + Debug.Assert(heapTypeSize == TypeOffset.Size); + int metaSize = heapTypeSize + Marshal.SizeOf(typeof(ClrMetaTypeEx)); + + IntPtr type = Runtime._PyObject_GC_Calloc(new IntPtr(metaSize)); Runtime.XIncref(py_type); + Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); + Marshal.WriteIntPtr(type, TypeOffset.ob_refcnt, (IntPtr)1); + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)metaSize); + Marshal.WriteIntPtr(type, heapTypeSize, (IntPtr)(heapTypeSize + IntPtr.Size)); - int size = TypeOffset.magic() + IntPtr.Size; - Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + SetupHeapType(type, "CLR Metatype"); const int flags = TypeFlags.Default | TypeFlags.Managed @@ -591,39 +616,13 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) return type; } - /// /// Utility method to allocate a type object & do basic initialization. /// internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { - IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); - // Clr type would not use __slots__, - // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), - // thus set the ob_size to 0 for avoiding slots iterations. - Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); - - // Cheat a little: we'll set tp_name to the internal char * of - // the Python version of the type name - otherwise we'd have to - // allocate the tp_name and would have no way to free it. - IntPtr temp = Runtime.PyUnicode_FromString(name); - IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); - Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); - Marshal.WriteIntPtr(type, TypeOffset.name, temp); - - Runtime.XIncref(temp); - Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); - temp = type + TypeOffset.nb_add; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); - - temp = type + TypeOffset.sq_length; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); - - temp = type + TypeOffset.mp_length; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); - - temp = type + TypeOffset.bf_getbuffer; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); + IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyCLRMetaType, 0); + SetupHeapType(type, name); return type; } @@ -780,6 +779,36 @@ private static SlotsHolder CreateSolotsHolder(IntPtr type) _slotsHolders.Add(type, holder); return holder; } + + private static void SetupHeapType(IntPtr type, string name) + { + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); + + // Cheat a little: we'll set tp_name to the internal char * of + // the Python version of the type name - otherwise we'd have to + // allocate the tp_name and would have no way to free it. + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); + Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); + Marshal.WriteIntPtr(type, TypeOffset.name, temp); + + Runtime.XIncref(temp); + Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); + temp = type + TypeOffset.nb_add; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); + + temp = type + TypeOffset.sq_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); + + temp = type + TypeOffset.mp_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); + + temp = type + TypeOffset.bf_getbuffer; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); + } } @@ -788,10 +817,10 @@ class SlotsHolder public delegate void Resetor(IntPtr type, int offset); private readonly IntPtr _type; - private Dictionary _slots = new Dictionary(); - private List _keepalive = new List(); - private Dictionary _customResetors = new Dictionary(); - private List _deallocators = new List(); + private Dictionary _slots; + private List _keepalive; + private Dictionary _customResetors; + private List _deallocators; private bool _alreadyReset = false; /// @@ -805,21 +834,25 @@ public SlotsHolder(IntPtr type) public void Set(int offset, ThunkInfo thunk) { + if (_slots == null) _slots = new Dictionary(); _slots[offset] = thunk; } public void Set(int offset, Resetor resetor) { + if (_customResetors == null) _customResetors = new Dictionary(); _customResetors[offset] = resetor; } public void AddDealloctor(Action deallocate) { + if (_deallocators == null) _deallocators = new List(); _deallocators.Add(deallocate); } public void KeeapAlive(ThunkInfo thunk) { + if (_keepalive == null) _keepalive = new List(); _keepalive.Add(thunk); } @@ -830,47 +863,15 @@ public void ResetSlots() return; } _alreadyReset = true; -#if DEBUG - IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); - string typeName = Marshal.PtrToStringAnsi(tp_name); -#endif - foreach (var offset in _slots.Keys) - { - IntPtr ptr = GetDefaultSlot(offset); -#if DEBUG - //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); -#endif - Marshal.WriteIntPtr(_type, offset, ptr); - } - - foreach (var action in _deallocators) - { - action(); - } + ResetDefaultSlots(); + InvokeDeallocActions(); + InvokeCustomResetors(); - foreach (var pair in _customResetors) + if (_keepalive != null) { - int offset = pair.Key; - var resetor = pair.Value; - resetor?.Invoke(_type, offset); - } - - _customResetors.Clear(); - _slots.Clear(); - _keepalive.Clear(); - _deallocators.Clear(); - - // Custom reset - IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); - if (handlePtr != IntPtr.Zero) - { - GCHandle handle = GCHandle.FromIntPtr(handlePtr); - if (handle.IsAllocated) - { - handle.Free(); - } - Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); + _keepalive.Clear(); } + ReleaseGCHandle(); } public static IntPtr GetDefaultSlot(int offset) @@ -915,6 +916,65 @@ public static IntPtr GetDefaultSlot(int offset) return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); } + + private void InvokeCustomResetors() + { + if (_customResetors == null) return; + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(_type, offset); + } + _customResetors.Clear(); + } + + private void InvokeDeallocActions() + { + if (_deallocators == null) return; + foreach (var action in _deallocators) + { + action(); + } + _deallocators.Clear(); + } + + private void ResetDefaultSlots() + { + if (_slots == null) return; +#if DEBUG + IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + foreach (var offset in _slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Marshal.WriteIntPtr(_type, offset, ptr); + } + _slots.Clear(); + } + + private void ReleaseGCHandle() + { + if (!ManagedType.IsManagedType(_type)) + { + return; + } + int offset = ObjectOffset.magic(Runtime.PyObject_TYPE(_type)); + IntPtr handlePtr = Marshal.ReadIntPtr(_type, offset); + if (handlePtr != IntPtr.Zero) + { + GCHandle handle = GCHandle.FromIntPtr(handlePtr); + if (handle.IsAllocated) + { + handle.Free(); + } + Marshal.WriteIntPtr(_type, offset, IntPtr.Zero); + } + } }