From d1928dcfde6cca317a33abadde2da27fd09adc60 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 14 Jun 2019 02:35:50 +0800 Subject: [PATCH 001/134] Drop LoadLibrary dependency --- src/runtime/runtime.cs | 107 ++--------------------------------------- 1 file changed, 5 insertions(+), 102 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7623200e0..0359dac1a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,97 +7,6 @@ namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); - } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); - } -#endif -#else - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } -#endif - - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } - /// /// Encapsulates the low-level Python C API. Note that it is /// the responsibility of the caller to have acquired the GIL @@ -395,18 +304,12 @@ internal static void Initialize(bool initSigs = false) IntPtr dllLocal = IntPtr.Zero; - if (_PythonDll != "__Internal") - { - dllLocal = NativeMethods.LoadLibrary(_PythonDll); - } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); + var zipimport = PyImport_ImportModule("zipimport"); + var ZipImportError = PyObject_GetAttrString(zipimport, "ZipImportError"); + _PyObject_NextNotImplemented = Marshal.ReadIntPtr(ZipImportError, TypeOffset.tp_iternext); + XDecref(ZipImportError); + XDecref(zipimport); -#if !(MONO_LINUX || MONO_OSX) - if (dllLocal != IntPtr.Zero) - { - NativeMethods.FreeLibrary(dllLocal); - } -#endif // Initialize data about the platform we're running on. We need // this for the type manager and potentially other details. Must // happen after caching the python types, above. From f000e0846407a5f0a8c30b612b3c874981330feb Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 14 Jun 2019 02:51:24 +0800 Subject: [PATCH 002/134] Explain for getting _PyObject_NextNotImplemented --- src/runtime/runtime.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 0359dac1a..7f0349da4 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -304,6 +304,10 @@ internal static void Initialize(bool initSigs = false) IntPtr dllLocal = IntPtr.Zero; + // Since `_PyObject_NextNotImplemented` would set to a heap class + // for tp_iternext which doesn't implement __next__. + // Thus we need a heap class to get it, the ZipImportError is a + // heap class and it's in builtins, so we can use it as a trick. var zipimport = PyImport_ImportModule("zipimport"); var ZipImportError = PyObject_GetAttrString(zipimport, "ZipImportError"); _PyObject_NextNotImplemented = Marshal.ReadIntPtr(ZipImportError, TypeOffset.tp_iternext); From f8824005d489f7d5f3d7bfb95f59af3a2dd03cfe Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 14 Jun 2019 03:18:58 +0800 Subject: [PATCH 003/134] Update changelog and authors --- AUTHORS.md | 1 + CHANGELOG.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index ba954b47d..4f4c9862e 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -52,6 +52,7 @@ - William Sardar ([@williamsardar])(https://github.com/williamsardar) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) +- ([@amos402]https://github.com/amos402) - ([@bltribble](https://github.com/bltribble)) - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5531bf47..d8430da0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - Refactored MethodBinder.Bind in preparation to make it extensible (#829) - Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- Remove `LoadLibrary` call. ([#880][p880]) ### Fixed @@ -57,7 +58,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed conversion of 'float' and 'double' values ([#486][i486]) - Fixed 'clrmethod' for python 2 ([#492][i492]) - Fixed double calling of constructor when deriving from .NET class ([#495][i495]) -- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) - Fixed `LockRecursionException` when loading assemblies ([#627][i627]) - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) From 4e19a4ff82e9a50155f898366c506bfa337c022c Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 18 Sep 2019 00:36:13 +0800 Subject: [PATCH 004/134] Add soft shutdown --- src/embed_tests/TestDomainReload.cs | 3 +- src/runtime/classbase.cs | 42 +++- src/runtime/classmanager.cs | 49 ++++- src/runtime/clrobject.cs | 3 +- src/runtime/constructorbinding.cs | 22 ++ src/runtime/extensiontype.cs | 21 +- src/runtime/fieldobject.cs | 10 + src/runtime/genericutil.cs | 5 - src/runtime/importhook.cs | 25 ++- src/runtime/interop.cs | 58 +++-- src/runtime/managedtype.cs | 105 ++++++++++ src/runtime/methodbinding.cs | 18 +- src/runtime/methodobject.cs | 24 ++- src/runtime/methodwrapper.cs | 26 ++- src/runtime/moduleobject.cs | 39 ++++ src/runtime/pythonengine.cs | 134 ++++++------ src/runtime/pythonexception.cs | 17 ++ src/runtime/runtime.cs | 314 +++++++++++++++++++++++++--- src/runtime/runtime_state.cs | 242 +++++++++++++++++++++ src/runtime/typemanager.cs | 74 +++++-- 20 files changed, 1079 insertions(+), 152 deletions(-) create mode 100644 src/runtime/runtime_state.cs diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index b162d4eb0..6084c4062 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -57,7 +57,7 @@ public static void DomainReloadAndGC() RunAssemblyAndUnload(pythonRunner1, "test1"); // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); + //Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); // This caused a crash because objects allocated in pythonRunner1 // still existed in memory, but the code to do python GC on those @@ -83,6 +83,7 @@ public static void RunPython() { AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); + //PythonEngine.Initialize(softShutdown: true); using (Py.GIL()) { try { var pyScript = string.Format(""import clr\n"" diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..af00c58fe 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Diagnostics; using System.Runtime.InteropServices; namespace Python.Runtime @@ -253,15 +254,48 @@ public static IntPtr tp_str(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); - if (dict != IntPtr.Zero) + if (self.pyHandle != self.tpHandle) { - Runtime.XDecref(dict); + ClearObjectDict(ob); } Runtime.PyObject_GC_UnTrack(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); Runtime.XDecref(self.tpHandle); - self.gcHandle.Free(); + self.FreeGCHandle(); + } + + public static int tp_clear(IntPtr ob) + { + ManagedType self = GetManagedObject(ob); + if (self.pyHandle != self.tpHandle) + { + ClearObjectDict(ob); + } + Runtime.XDecref(self.tpHandle); + self.tpHandle = IntPtr.Zero; + self.FreeGCHandle(); + return 0; + } + + private static IntPtr GetObjectDict(IntPtr ob) + { + return Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + } + + private static void SetObjectDict(IntPtr ob, IntPtr value) + { + Marshal.WriteIntPtr(ob, ObjectOffset.DictOffset(ob), value); + } + + private static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + SetObjectDict(ob, IntPtr.Zero); + Runtime.XDecref(dict); } } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0b084a49d..49b732910 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -4,6 +4,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Security; +using System.Linq; namespace Python.Runtime { @@ -26,7 +27,6 @@ private ClassManager() static ClassManager() { - cache = new Dictionary(128); // SEE: https://msdn.microsoft.com/en-us/library/96b1ayy4(v=vs.100).aspx // ""All delegates inherit from MulticastDelegate, which inherits from Delegate."" // Was Delegate, which caused a null MethodInfo returned from GetMethode("Invoke") @@ -39,6 +39,49 @@ public static void Reset() cache = new Dictionary(128); } + public static IList GetManagedTypes() + { + return cache.Values.ToArray(); // Make a copy. + } + + internal static void RemoveClasses() + { + var visited = new HashSet(); + var visitedHandle = GCHandle.Alloc(visited); + var visitedPtr = (IntPtr)visitedHandle; + try + { + foreach (var cls in cache.Values) + { + cls.TypeTraverse(OnVisit, visitedPtr); + // XXX: Force release some resouces. + cls.TypeClear(); + //Runtime.XDecref(cls.pyHandle); + } + } + finally + { + visitedHandle.Free(); + } + } + + private static int OnVisit(IntPtr ob, IntPtr arg) + { + var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; + if (visited.Contains(ob)) + { + return 0; + } + visited.Add(ob); + var clrObj = ManagedType.GetManagedObject(ob); + if (clrObj != null) + { + clrObj.TypeTraverse(OnVisit, arg); + clrObj.TypeClear(); + } + return 0; + } + /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. @@ -134,7 +177,6 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr tp = TypeManager.GetTypeHandle(impl, type); - impl.tpHandle = tp; // Finally, initialize the class __dict__ and return the object. IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); @@ -146,6 +188,9 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + // info.members are already useless + Runtime.XDecref(item.pyHandle); + item.pyHandle = IntPtr.Zero; } // If class has constructors, generate an __doc__ attribute. diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..29c57b28f 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -22,11 +22,10 @@ internal CLRObject(object ob, IntPtr tp) } } - GCHandle gc = GCHandle.Alloc(this); + GCHandle gc = AllocGCHandle(); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; - gcHandle = gc; inst = ob; // Fix the BaseException args (and __cause__ in case of Python 3) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 3908628b9..a9f6c961b 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -147,6 +147,17 @@ public static IntPtr tp_repr(IntPtr ob) Runtime.XDecref(self.pyTypeHndl); ExtensionType.FinalizeObject(self); } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ConstructorBinding)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; + } } /// @@ -233,5 +244,16 @@ public static IntPtr tp_repr(IntPtr ob) Runtime.XDecref(self.pyTypeHndl); FinalizeObject(self); } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (BoundContructor)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; + } } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..5e8ee6b59 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -28,7 +28,7 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - GCHandle gc = GCHandle.Alloc(this); + GCHandle gc = AllocGCHandle(); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); // We have to support gc because the type machinery makes it very @@ -40,7 +40,6 @@ public ExtensionType() tpHandle = tp; pyHandle = py; - gcHandle = gc; } @@ -51,7 +50,7 @@ public static void FinalizeObject(ManagedType self) { Runtime.PyObject_GC_Del(self.pyHandle); Runtime.XDecref(self.tpHandle); - self.gcHandle.Free(); + self.FreeGCHandle(); } @@ -91,5 +90,21 @@ public static void tp_dealloc(IntPtr ob) ManagedType self = GetManagedObject(ob); FinalizeObject(self); } + + public static int tp_clear(IntPtr ob) + { + ManagedType self = GetManagedObject(ob); + Runtime.XDecref(self.tpHandle); + self.FreeGCHandle(); + return 0; + } + + //public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + //{ + // ManagedType self = GetManagedObject(ob); + // int res = PyVisit(self.tpHandle, visit, arg); + // if (res != 0) return res; + // return 0; + //} } } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 7c9a466d5..c4675d723 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -55,6 +55,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return IntPtr.Zero; + } result = info.GetValue(co.inst); return Converter.ToPython(result, info.FieldType); } @@ -115,6 +120,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) if (!is_static) { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return -1; + } info.SetValue(co.inst, newval); } else diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..df78d9899 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -16,11 +16,6 @@ private GenericUtil() { } - static GenericUtil() - { - mapping = new Dictionary>>(); - } - public static void Reset() { mapping = new Dictionary>>(); diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..ef322648a 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -72,12 +72,29 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) { - Runtime.XDecref(py_clr_module); - Runtime.XDecref(root.pyHandle); - Runtime.XDecref(py_import); + return; } + IntPtr dict = Runtime.PyImport_GetModuleDict(); + IntPtr mod = Runtime.IsPython3 ? + Runtime.PyImport_ImportModule("builtins") + : Runtime.PyDict_GetItemString(dict, "__builtin__"); + Runtime.PyObject_SetAttrString(mod, "__import__", py_import); + + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; + + Runtime.XDecref(root.pyHandle); + root = null; + + hook.Dispose(); + hook = null; + + Runtime.XDecref(py_import); + py_import = IntPtr.Zero; + + CLRModule.Reset(); } /// diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..702bb7982 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; +using System.Collections.Generic; namespace Python.Runtime { @@ -334,7 +335,7 @@ internal class TypeFlags internal class Interop { - private static ArrayList keepAlive; + private static Dictionary, Tuple> keepAlive; private static Hashtable pmap; static Interop() @@ -351,8 +352,8 @@ static Interop() p[item.Name] = item; } - keepAlive = new ArrayList(); - Marshal.AllocHGlobal(IntPtr.Size); + keepAlive = new Dictionary, Tuple>(); + //Marshal.AllocHGlobal(IntPtr.Size); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -450,6 +451,23 @@ internal static Type GetPrototype(string name) } internal static IntPtr GetThunk(MethodInfo method, string funcType = null) + { + var key = new KeyValuePair(method, funcType); + Tuple thunkPair; + if (keepAlive.TryGetValue(key, out thunkPair)) + { + return thunkPair.Item2; + } + thunkPair = GetThunkImpl(method, funcType); + if (thunkPair == null) + { + return IntPtr.Zero; + } + keepAlive[key] = thunkPair; + return thunkPair.Item2; + } + + private static Tuple GetThunkImpl(MethodInfo method, string funcType) { Type dt; if (funcType != null) @@ -457,18 +475,17 @@ internal static IntPtr GetThunk(MethodInfo method, string funcType = null) else dt = GetPrototype(method.Name); - if (dt != null) + if (dt == null) { - IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); - Delegate d = Delegate.CreateDelegate(dt, method); - Thunk cb = new Thunk(d); - Marshal.StructureToPtr(cb, tmp, false); - IntPtr fp = Marshal.ReadIntPtr(tmp, 0); - Marshal.FreeHGlobal(tmp); - keepAlive.Add(d); - return fp; + return null; } - return IntPtr.Zero; + IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); + Delegate d = Delegate.CreateDelegate(dt, method); + Thunk cb = new Thunk(d); + Marshal.StructureToPtr(cb, tmp, false); + IntPtr fp = Marshal.ReadIntPtr(tmp, 0); + Marshal.FreeHGlobal(tmp); + return new Tuple(d, fp); } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -522,4 +539,19 @@ public Thunk(Delegate d) fn = d; } } + + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + struct PyGC_Node + { + public IntPtr gc_next; + public IntPtr gc_prev; + public IntPtr gc_refs; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + struct PyGC_Head + { + public PyGC_Node gc; + } } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 3191da949..6f0e8b88b 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; namespace Python.Runtime @@ -14,6 +16,48 @@ internal abstract class ManagedType internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * +#if NPY_TRACK_OBJECT + private static readonly HashSet _managedObjs = new HashSet(); +#endif + + internal void DecrRefCount() + { + Runtime.XDecref(pyHandle); + } + + internal long RefCount + { + get + { + var gs = Runtime.PyGILState_Ensure(); + try + { + return Runtime.Refcount(pyHandle); + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + } + + internal GCHandle AllocGCHandle() + { + gcHandle = GCHandle.Alloc(this); +#if NPY_TRACK_OBJECT + _managedObjs.Add(this); +#endif + return gcHandle; + } + + internal void FreeGCHandle() + { + gcHandle.Free(); +#if NPY_TRACK_OBJECT + _managedObjs.Remove(this); +#endif + gcHandle = new GCHandle(); + } /// /// Given a Python object, return the associated managed object or null. @@ -75,5 +119,66 @@ internal static bool IsManagedType(IntPtr ob) } return false; } + +#if NPY_TRACK_OBJECT + internal static void Reset() + { + _managedObjs.Clear(); + } + + internal static ICollection GetManagedObjects() + { + return _managedObjs; + } +#endif + + internal static int PyVisit(IntPtr ob, IntPtr visit, IntPtr arg) + { + if (ob == IntPtr.Zero) + { + return 0; + } + var visitFunc = (Interop.ObjObjFunc)Marshal.GetDelegateForFunctionPointer(visit, typeof(Interop.ObjObjFunc)); + return visitFunc(ob, arg); + } + + /// + /// Wrapper for calling tp_clear + /// + internal void TypeClear() + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var clearPtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_clear); + if (clearPtr == IntPtr.Zero) + { + return; + } + var clearFunc = (Interop.InquiryFunc)Marshal.GetDelegateForFunctionPointer(clearPtr, typeof(Interop.InquiryFunc)); + // TODO: Handle errors base on return value + clearFunc(pyHandle); + } + + /// + /// Wrapper for calling tp_traverse + /// + internal void TypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var traversePtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_traverse); + if (traversePtr == IntPtr.Zero) + { + return; + } + var traverseFunc = (Interop.ObjObjArgFunc)Marshal.GetDelegateForFunctionPointer(traversePtr, typeof(Interop.ObjObjArgFunc)); + var visiPtr = Marshal.GetFunctionPointerForDelegate(visitproc); + // TODO: Handle errors base on return value + traverseFunc(pyHandle, visiPtr, arg); + } } } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..b917d9bb9 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -36,6 +36,14 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } + private void ClearMembers() + { + Runtime.XDecref(target); + target = IntPtr.Zero; + Runtime.XDecref(targetType); + targetType = IntPtr.Zero; + } + /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -237,9 +245,15 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - Runtime.XDecref(self.targetType); + self.ClearMembers(); FinalizeObject(self); } + + public new static int tp_clear(IntPtr ob) + { + var self = (MethodBinding)GetManagedObject(ob); + self.ClearMembers(); + return ExtensionType.tp_clear(ob); + } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..2531bc285 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -109,6 +109,17 @@ internal bool IsStatic() return is_static; } + private void ClearMembers() + { + Runtime.XDecref(doc); + doc = IntPtr.Zero; + if (unbound != null) + { + Runtime.XDecref(unbound.pyHandle); + unbound = null; + } + } + /// /// Descriptor __getattribute__ implementation. /// @@ -196,12 +207,15 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); - Runtime.XDecref(self.doc); - if (self.unbound != null) - { - Runtime.XDecref(self.unbound.pyHandle); - } + self.ClearMembers(); ExtensionType.FinalizeObject(self); } + + public new static int tp_clear(IntPtr ob) + { + var self = (MethodObject)GetManagedObject(ob); + self.ClearMembers(); + return ExtensionType.tp_clear(ob); + } } } diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index 2f3ce3ef2..ebcea0709 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -8,10 +8,11 @@ namespace Python.Runtime /// currently used mainly to implement special cases like the CLR /// import hook. /// - internal class MethodWrapper + internal class MethodWrapper : IDisposable { public IntPtr mdef; public IntPtr ptr; + private bool _disposed = false; public MethodWrapper(Type type, string name, string funcType = null) { @@ -27,9 +28,32 @@ public MethodWrapper(Type type, string name, string funcType = null) ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } + ~MethodWrapper() + { + Dispose(); + } + public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); } + + public void Dispose() + { + if (_disposed) + { + return; + } + using (Py.GIL()) + { + _disposed = true; + Runtime.XDecref(ptr); + if (mdef != IntPtr.Zero) + { + Runtime.PyMem_Free(mdef); + mdef = IntPtr.Zero; + } + } + } } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 7a45c6c81..23488f676 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -300,6 +300,45 @@ public static IntPtr tp_repr(IntPtr ob) var self = (ModuleObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + public new static void tp_dealloc(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + Runtime.XDecref(self.dict); + self.dict = IntPtr.Zero; + foreach (var attr in self.cache.Values) + { + Runtime.XDecref(attr.pyHandle); + } + self.cache.Clear(); + ExtensionType.tp_dealloc(ob); + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ModuleObject)GetManagedObject(ob); + int res = PyVisit(self.dict, visit, arg); + if (res != 0) return res; + foreach (var attr in self.cache.Values) + { + res = PyVisit(attr.pyHandle, visit, arg); + if (res != 0) return res; + } + return 0; + } + + public new static int tp_clear(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + Runtime.XDecref(self.dict); + self.dict = IntPtr.Zero; + foreach (var attr in self.cache.Values) + { + Runtime.XDecref(attr.pyHandle); + } + self.cache.Clear(); + return ExtensionType.tp_clear(ob); + } } /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..cf4482c85 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,6 +12,8 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { + public static bool SoftShutdown { get; private set; } + private static DelegateManager delegateManager; private static bool initialized; private static IntPtr _pythonHome = IntPtr.Zero; @@ -140,9 +142,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, bool softShutdown = false) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, softShutdown); } /// @@ -155,35 +157,40 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, bool softShutdown = false) { - if (!initialized) + if (initialized) { - // Creating the delegateManager MUST happen before Runtime.Initialize - // is called. If it happens afterwards, DelegateManager's CodeGenerator - // throws an exception in its ctor. This exception is eaten somehow - // during an initial "import clr", and the world ends shortly thereafter. - // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). - delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); - initialized = true; - Exceptions.Clear(); - - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - - if (setSysArgv) - { - Py.SetArgv(args); - } + return; + } + // Creating the delegateManager MUST happen before Runtime.Initialize + // is called. If it happens afterwards, DelegateManager's CodeGenerator + // throws an exception in its ctor. This exception is eaten somehow + // during an initial "import clr", and the world ends shortly thereafter. + // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). + delegateManager = new DelegateManager(); + Runtime.Initialize(initSigs, softShutdown); + initialized = true; + SoftShutdown = softShutdown; + Exceptions.Clear(); + + // Make sure we clean up properly on app domain unload. + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + + // Remember to shut down the runtime. + AddShutdownHandler(() => Runtime.Shutdown(softShutdown)); + + // The global scope gets used implicitly quite early on, remember + // to clear it out when we shut down. + AddShutdownHandler(PyScopeManager.Global.Clear); + + if (setSysArgv) + { + Py.SetArgv(args); + } + if (!softShutdown) + { // register the atexit callback (this doesn't use Py_AtExit as the C atexit // callbacks are called after python is fully finalized but the python ones // are called while the python engine is still running). @@ -191,46 +198,51 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, "import atexit, clr\n" + "atexit.register(clr._AtExit)\n"; PythonEngine.Exec(code); + } - // Load the clr.py resource into the clr module - IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); - IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + // Load the clr.py resource into the clr module + IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); + IntPtr clr_dict = Runtime.PyModule_GetDict(clr); - var locals = new PyDict(); - try + var locals = new PyDict(); + try + { + IntPtr module = Runtime.PyImport_AddModule("clr._extras"); + IntPtr module_globals = Runtime.PyModule_GetDict(module); + IntPtr builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + + Assembly assembly = Assembly.GetExecutingAssembly(); + Stream stream = assembly.GetManifestResourceStream("clr.py"); + if (stream == null) { - IntPtr module = Runtime.PyImport_AddModule("clr._extras"); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); - - Assembly assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("clr.py")) - using (var reader = new StreamReader(stream)) - { - // add the contents of clr.py to the module - string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Handle); - } + stream = File.OpenRead(@"I:\repos\pythonnet\src\runtime\resources\clr.py"); + } + using (stream) + using (var reader = new StreamReader(stream)) + { + // add the contents of clr.py to the module + string clr_py = reader.ReadToEnd(); + Exec(clr_py, module_globals, locals.Handle); + } - // add the imported module to the clr module, and copy the API functions - // and decorators into the main clr module. - Runtime.PyDict_SetItemString(clr_dict, "_extras", module); - foreach (PyObject key in locals.Keys()) + // add the imported module to the clr module, and copy the API functions + // and decorators into the main clr module. + Runtime.PyDict_SetItemString(clr_dict, "_extras", module); + foreach (PyObject key in locals.Keys()) + { + if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { - if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) - { - PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); - value.Dispose(); - } - key.Dispose(); + PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + value.Dispose(); } + key.Dispose(); } - finally - { - locals.Dispose(); - } + } + finally + { + locals.Dispose(); } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 295a63b3d..8a6a24799 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; namespace Python.Runtime { @@ -190,5 +191,21 @@ public static bool Matches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } + + public static void ThrowIfIsNull(IntPtr ob) + { + if (ob == IntPtr.Zero) + { + throw new PythonException(); + } + } + + public static void ThrowIfIsNotZero(int value) + { + if (value != 0) + { + throw new PythonException(); + } + } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 75f11492f..fdccdf5ab 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -165,22 +165,27 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static PyReferenceCollection _typeRefs; + /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + internal static void Initialize(bool initSigs = false, bool softShutdown = false) { if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); + if (PyEval_ThreadsInitialized() == 0) + { + PyEval_InitThreads(); + } + if (softShutdown) + { + RuntimeState.Save(); + } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } - if (PyEval_ThreadsInitialized() == 0) - { - PyEval_InitThreads(); - } - IsFinalizing = false; CLRModule.Reset(); @@ -202,19 +207,22 @@ internal static void Initialize(bool initSigs = false) dict = PyImport_GetModuleDict(); op = PyDict_GetItemString(dict, "__builtin__"); } - PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); - PyBaseObjectType = PyObject_GetAttrString(op, "object"); - PyNone = PyObject_GetAttrString(op, "None"); - PyTrue = PyObject_GetAttrString(op, "True"); - PyFalse = PyObject_GetAttrString(op, "False"); + _typeRefs = new PyReferenceCollection(); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(op, "NotImplemented")); - PyBoolType = PyObject_Type(PyTrue); - PyNoneType = PyObject_Type(PyNone); - PyTypeType = PyObject_Type(PyNoneType); + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(op, "object")); + + SetPyMember(ref PyNone, PyObject_GetAttrString(op, "None")); + SetPyMember(ref PyTrue, PyObject_GetAttrString(op, "True")); + SetPyMember(ref PyFalse, PyObject_GetAttrString(op, "False")); + + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); op = PyObject_GetAttrString(dict, "keys"); - PyMethodType = PyObject_Type(op); + SetPyMember(ref PyMethodType, PyObject_Type(op)); XDecref(op); // For some arcane reason, builtins.__dict__.__setitem__ is *not* @@ -222,49 +230,53 @@ internal static void Initialize(bool initSigs = false) // // object.__init__ seems safe, though. op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - PyWrapperDescriptorType = PyObject_Type(op); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); XDecref(op); + op = PyDict_GetItemString(dict, "KeyError"); + XIncref(op); + SetPyMember(ref PyExc_KeyError, op); + #if PYTHON3 XDecref(dict); #endif op = PyString_FromString("string"); - PyStringType = PyObject_Type(op); + SetPyMember(ref PyStringType, PyObject_Type(op)); XDecref(op); op = PyUnicode_FromString("unicode"); - PyUnicodeType = PyObject_Type(op); + SetPyMember(ref PyUnicodeType, PyObject_Type(op)); XDecref(op); #if PYTHON3 op = PyBytes_FromString("bytes"); - PyBytesType = PyObject_Type(op); + SetPyMember(ref PyBytesType, PyObject_Type(op)); XDecref(op); #endif op = PyTuple_New(0); - PyTupleType = PyObject_Type(op); + SetPyMember(ref PyTupleType, PyObject_Type(op)); XDecref(op); op = PyList_New(0); - PyListType = PyObject_Type(op); + SetPyMember(ref PyListType, PyObject_Type(op)); XDecref(op); op = PyDict_New(); - PyDictType = PyObject_Type(op); + SetPyMember(ref PyDictType, PyObject_Type(op)); XDecref(op); op = PyInt_FromInt32(0); - PyIntType = PyObject_Type(op); + SetPyMember(ref PyIntType, PyObject_Type(op)); XDecref(op); op = PyLong_FromLong(0); - PyLongType = PyObject_Type(op); + SetPyMember(ref PyLongType, PyObject_Type(op)); XDecref(op); op = PyFloat_FromDouble(0); - PyFloatType = PyObject_Type(op); + SetPyMember(ref PyFloatType, PyObject_Type(op)); XDecref(op); #if PYTHON3 @@ -275,10 +287,10 @@ internal static void Initialize(bool initSigs = false) IntPtr d = PyDict_New(); IntPtr c = PyClass_New(IntPtr.Zero, d, s); - PyClassType = PyObject_Type(c); + SetPyObject(ref PyClassType, PyObject_Type(c)); IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - PyInstanceType = PyObject_Type(i); + SetPyObject(ref PyInstanceType, PyObject_Type(i)); XDecref(s); XDecref(i); @@ -310,7 +322,7 @@ internal static void Initialize(bool initSigs = false) // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); - PyCLRMetaType = MetaType.Initialize(); + SetPyMember(ref PyCLRMetaType, MetaType.Initialize()); Exceptions.Initialize(); ImportHook.Initialize(); @@ -371,12 +383,65 @@ private static void InitializePlatformData() Machine = MType; } - internal static void Shutdown() + internal static void Shutdown(bool soft) { + if (Py_IsInitialized() == 0) + { + return; + } AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); + + ClearClrModules(); + ClassManager.RemoveClasses(); + TypeManager.RemoveTypes(); + RemoveClrRootModule(); + + { + //var intialModules = PySys_GetObject("initial_modules"); + //var modules = PyImport_GetModuleDict(); + //foreach (var name in RuntimeState.GetModuleNames()) + //{ + // if (PySet_Contains(intialModules, name) == 1) + // { + // continue; + // } + // var module = PyDict_GetItem(modules, name); + // IntPtr clr_dict = Runtime._PyObject_GetDictPtr(module); // PyObject** + // clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); + // PyDict_Clear(clr_dict); + // PyDict_DelItem(modules, name); + //} + } + if (soft) + { + PyGC_Collect(); + RemoveClrObjects(); + RuntimeState.Restore(); + //var objs = ManagedType.GetManagedObjects(); + //var subIterp = PyThreadState_Swap(_originIterp); + //var iterp = PyThreadState_Get(); + //Py_EndInterpreter(iterp); + //PyGILState_Ensure(); + ResetPyMembers(); +// Runtime.PyRun_SimpleString(@" +//import gc, pprint, objgraph +//print() +//with open('R:/objsx.log', 'w', encoding='utf8') as f: +// pprint.pprint(objgraph.show_growth(), f) +// #pprint.pprint(objgraph.typestats(), f) + +//#lst = gc.get_objects() +//#with open('R:/objs.log', 'w', encoding='utf8') as f: +// # for obj in lst: +// # pprint.pprint(obj, f) +//#del lst +//"); + return; + } + ResetPyMembers(); Py_Finalize(); } @@ -390,6 +455,69 @@ internal static int AtExit() return 0; } + private static void SetPyMember(ref IntPtr obj, IntPtr value) + { + PythonException.ThrowIfIsNull(value); + obj = value; + // XXX: Is it safe for all platforms? + unsafe + { + fixed (void* p = &obj) + { + _typeRefs.Add((IntPtr)p); + } + } + } + + private static void ResetPyMembers() + { + _typeRefs.Release(); + _typeRefs = null; + } + + private static void ClearClrModules() + { + var modules = PyImport_GetModuleDict(); + var items = PyDict_Items(modules); + long length = PyList_Size(items); + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item, 0); + var module = PyTuple_GetItem(item, 1); + var clrModule = ManagedType.GetManagedObject(module); + if (clrModule != null) + { + PyDict_DelItem(modules, name); + } + } + XDecref(items); + } + + private static void RemoveClrRootModule() + { + var modules = PyImport_GetModuleDict(); + PyDictTryDelItem(modules, "CLR"); + PyDictTryDelItem(modules, "clr"); + PyDictTryDelItem(modules, "clr._extra"); + } + + private static void PyDictTryDelItem(IntPtr dict, string key) + { + if (PyDict_DelItemString(dict, key) == 0) + { + return; + } + if (!PythonException.Matches(PyExc_KeyError)) + { + throw new PythonException(); + } + } + + private static void RemoveClrObjects() + { + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; internal static IntPtr Py_eval_input = (IntPtr)258; @@ -432,6 +560,8 @@ internal static int AtExit() internal static IntPtr PyNone; internal static IntPtr Error; + internal static IntPtr PyExc_KeyError; + /// /// Check if any Python Exceptions occurred. /// If any exist throw new PythonException. @@ -1068,6 +1198,19 @@ internal static bool PyFloat_Check(IntPtr ob) return PyObject_TYPE(ob) == PyFloatType; } + /// + /// Return value: New reference. + /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_FromVoidPtr(IntPtr p); + + /// + /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_AsVoidPtr(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyFloat_FromDouble(double value); @@ -1464,9 +1607,16 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDictProxy_New(IntPtr dict); + /// + /// Return value: Borrowed reference. + /// Return NULL if the key key is not present, but without setting an exception. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); + /// + /// Return value: Borrowed reference. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, string key); @@ -1512,6 +1662,21 @@ internal static long PyDict_Size(IntPtr pointer) internal static extern IntPtr _PyDict_Size(IntPtr pointer); + /// + /// Return value: New reference. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PySet_New(IntPtr iterable); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Add(IntPtr set, IntPtr key); + + /// + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); + //==================================================================== // Python list API //==================================================================== @@ -1532,6 +1697,10 @@ internal static IntPtr PyList_New(long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); + /// + /// Return value: Borrowed reference. + /// If index is out of bounds, return NULL and set an IndexError exception. + /// internal static IntPtr PyList_GetItem(IntPtr pointer, long index) { return PyList_GetItem(pointer, new IntPtr(index)); @@ -1711,6 +1880,10 @@ int updatepath ); #endif + /// + /// Return value: Borrowed reference. + /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySys_GetObject(string name); @@ -1750,6 +1923,9 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + /// + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1777,6 +1953,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyObject_GC_UnTrack(IntPtr tp); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _PyObject_Dump(IntPtr ob); //==================================================================== // Python memory API @@ -1842,6 +2020,54 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + //==================================================================== + // Python GC API + //==================================================================== + + internal const int _PyGC_REFS_SHIFT = 1; + internal const long _PyGC_REFS_UNTRACKED = -2; + internal const long _PyGC_REFS_REACHABLE = -3; + internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; + + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyGC_Collect(); + + internal static IntPtr _Py_AS_GC(IntPtr ob) + { + // XXX: PyGC_Head has a force alignment depend on platform. + // See PyGC_Head in objimpl.h for more details. + return Is32Bit ? ob - 16 : ob - 24; + } + + internal static IntPtr _Py_FROM_GC(IntPtr gc) + { + return Is32Bit ? gc + 16 : gc + 24; + } + + internal static IntPtr _PyGCHead_REFS(IntPtr gc) + { + unsafe + { + var pGC = (PyGC_Head*)gc; + var refs = pGC->gc.gc_refs; + if (Is32Bit) + { + return new IntPtr(refs.ToInt32() >> _PyGC_REFS_SHIFT); + } + return new IntPtr(refs.ToInt64() >> _PyGC_REFS_SHIFT); + } + } + + internal static IntPtr _PyGC_REFS(IntPtr ob) + { + return _PyGCHead_REFS(_Py_AS_GC(ob)); + } + + internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) + { + return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; + } //==================================================================== // Miscellaneous @@ -1859,4 +2085,34 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_MakePendingCalls(); } + + + class PyReferenceCollection + { + public List Objects { get; private set; } + + public PyReferenceCollection() + { + Objects = new List(); + } + + public void Add(IntPtr obj) + { + Objects.Add(obj); + } + + public void Release() + { + foreach (var objRef in Objects) + { + unsafe + { + var p = (void**)objRef; + Runtime.XDecref((IntPtr)(*p)); + *p = null; + } + } + Objects.Clear(); + } + } } diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs new file mode 100644 index 000000000..69291453e --- /dev/null +++ b/src/runtime/runtime_state.cs @@ -0,0 +1,242 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Linq; +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + class RuntimeState + { + public static void Save() + { + if (PySys_GetObject("dummy_gc") != IntPtr.Zero) + { + throw new Exception("Runtime State set already"); + } + + var modules = PySet_New(IntPtr.Zero); + foreach (var name in GetModuleNames()) + { + int res = PySet_Add(modules, name); + PythonException.ThrowIfIsNotZero(res); + } + + var objs = PySet_New(IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + AddObjPtrToSet(objs, obj); + } + + var dummyGCHead = PyMem_Malloc(Marshal.SizeOf(typeof(PyGC_Head))); + unsafe + { + var head = (PyGC_Head*)dummyGCHead; + head->gc.gc_next = dummyGCHead; + head->gc.gc_prev = dummyGCHead; + head->gc.gc_refs = IntPtr.Zero; + } + { + var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); + int res = PySys_SetObject("dummy_gc", pyDummyGC); + PythonException.ThrowIfIsNotZero(res); + XDecref(pyDummyGC); + + AddObjPtrToSet(objs, modules); + try + { + res = PySys_SetObject("initial_modules", modules); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(modules); + } + + AddObjPtrToSet(objs, objs); + try + { + res = PySys_SetObject("initial_objs", objs); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(objs); + } + + } + } + + public static void Restore() + { + var dummyGCAddr = PySys_GetObject("dummy_gc"); + if (dummyGCAddr == IntPtr.Zero) + { + throw new Exception("Runtime state have not set"); + } + var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); + //foreach (var abandomedNode in IterGCNodes(dummyGC)) + //{ + // if (abandomedNode == IntPtr.Zero) + // { + // continue; + // } + // //PyObject_GC_Del(_Py_FROM_GC(abandomedNode)); + //} + + //unsafe + //{ + // var head = (PyGC_Head*)dummyGC; + // head->gc.gc_next = dummyGC; + // head->gc.gc_prev = dummyGC; + // head->gc.gc_refs = IntPtr.Zero; + //} + + ResotreModules(dummyGC); + RestoreObjects(dummyGC); + foreach (var obj in IterObjects(dummyGC)) + { + //if (ManagedType.IsManagedType(obj)) + { + //var clrobj = ManagedType.GetManagedObject(obj); + //Console.WriteLine(clrobj); + } + } + //Console.WriteLine($"abndom {IterGCNodes(dummyGC).Count()}"); + } + + private static void ResotreModules(IntPtr dummyGC) + { + var intialModules = PySys_GetObject("initial_modules"); + Debug.Assert(intialModules != null); + var modules = PyImport_GetModuleDict(); + foreach (var name in GetModuleNames()) + { + if (PySet_Contains(intialModules, name) == 1) + { + continue; + } + var module = PyDict_GetItem(modules, name); + if (_PyObject_GC_IS_TRACKED(module)) + { + ExchangeGCChain(module, dummyGC); + } + PyDict_DelItem(modules, name); + } + } + + private static void RestoreObjects(IntPtr dummyGC) + { + var intialObjs = PySys_GetObject("initial_objs"); + Debug.Assert(intialObjs != null); + foreach (var obj in PyGCGetObjects()) + { + var p = PyLong_FromVoidPtr(obj); + try + { + if (PySet_Contains(intialObjs, p) == 1) + { + continue; + } + } + finally + { + XDecref(p); + } + Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); + ExchangeGCChain(obj, dummyGC); + } + } + + public static IEnumerable PyGCGetObjects() + { + var gc = PyImport_ImportModule("gc"); + var get_objects = PyObject_GetAttrString(gc, "get_objects"); + var objs = PyObject_CallObject(get_objects, IntPtr.Zero); + var length = PyList_Size(objs); + for (long i = 0; i < length; i++) + { + var obj = PyList_GetItem(objs, i); + yield return obj; + } + XDecref(objs); + XDecref(gc); + } + + public static IEnumerable GetModuleNames() + { + var modules = PyImport_GetModuleDict(); + var names = PyDict_Keys(modules); + var length = PyList_Size(names); + for (int i = 0; i < length; i++) + { + var name = PyList_GetItem(names, i); + yield return name; + } + } + + private static void AddObjPtrToSet(IntPtr set, IntPtr obj) + { + var p = PyLong_FromVoidPtr(obj); + XIncref(obj); + try + { + int res = PySet_Add(set, p); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(p); + } + } + /// + /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. + /// + private static void ExchangeGCChain(IntPtr obj, IntPtr gc) + { + var head = _Py_AS_GC(obj); + if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) + { + throw new Exception("GC object untracked"); + } + unsafe + { + var g = (PyGC_Head*)head; + var newGCGen = (PyGC_Head*)gc; + + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = g->gc.gc_next; + ((PyGC_Head*)g->gc.gc_next)->gc.gc_prev = g->gc.gc_prev; + + g->gc.gc_next = gc; + g->gc.gc_prev = newGCGen->gc.gc_prev; + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = head; + newGCGen->gc.gc_prev = head; + } + } + + private static IEnumerable IterGCNodes(IntPtr gc) + { + var node = GetNextGCNode(gc); + while (node != gc) + { + var next = GetNextGCNode(node); + yield return node; + node = next; + } + } + + private static IEnumerable IterObjects(IntPtr gc) + { + foreach (var node in IterGCNodes(gc)) + { + yield return _Py_FROM_GC(node); + } + } + + private static unsafe IntPtr GetNextGCNode(IntPtr node) + { + return ((PyGC_Head*)node)->gc.gc_next; + } + } +} diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 9a98e9ebb..f82260d37 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -4,6 +4,7 @@ using System.Reflection; using System.Runtime.InteropServices; using Python.Runtime.Platform; +using System.Linq; namespace Python.Runtime { @@ -20,7 +21,6 @@ internal class TypeManager static TypeManager() { tbFlags = BindingFlags.Public | BindingFlags.Static; - cache = new Dictionary(128); } public static void Reset() @@ -28,7 +28,22 @@ public static void Reset() cache = new Dictionary(128); } + public static IList GetManagedTypes() + { + return cache.Values.ToArray(); + } + + internal static void RemoveTypes() + { + foreach (var tpHandle in cache.Values) + { + Runtime.XDecref(tpHandle); + } + cache.Clear(); + } + /// + /// Return value: Borrowed reference. /// Given a managed Type derived from ExtensionType, get the handle to /// a Python type object that delegates its implementation to the Type /// object. These Python type instances are used to implement internal @@ -94,11 +109,15 @@ internal static IntPtr CreateType(Type impl) TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); InitMethods(type, impl); @@ -172,20 +191,23 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; IntPtr mod = Runtime.PyString_FromString(mn); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); // Hide the gchandle of the implementation in a magic type slot. - GCHandle gc = GCHandle.Alloc(impl); + GCHandle gc = impl.AllocGCHandle(); Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); // Set the handle attributes on the implementing instance. - impl.tpHandle = Runtime.PyCLRMetaType; - impl.gcHandle = gc; + impl.tpHandle = type; impl.pyHandle = type; //DebugUtil.DumpType(type); @@ -309,17 +331,17 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); + // Slots will inherit from TypeType, it's not neccesary for setting them. // Copy gc and other type slots from the base Python metatype. + //CopySlot(py_type, type, TypeOffset.tp_basicsize); + //CopySlot(py_type, type, TypeOffset.tp_itemsize); - CopySlot(py_type, type, TypeOffset.tp_basicsize); - CopySlot(py_type, type, TypeOffset.tp_itemsize); - - CopySlot(py_type, type, TypeOffset.tp_dictoffset); - CopySlot(py_type, type, TypeOffset.tp_weaklistoffset); + //CopySlot(py_type, type, TypeOffset.tp_dictoffset); + //CopySlot(py_type, type, TypeOffset.tp_weaklistoffset); - CopySlot(py_type, type, TypeOffset.tp_traverse); - CopySlot(py_type, type, TypeOffset.tp_clear); - CopySlot(py_type, type, TypeOffset.tp_is_gc); + //CopySlot(py_type, type, TypeOffset.tp_traverse); + //CopySlot(py_type, type, TypeOffset.tp_clear); + //CopySlot(py_type, type, TypeOffset.tp_is_gc); // Override type slots with those of the managed implementation. @@ -352,7 +374,10 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -394,7 +419,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) InitializeSlots(type, impl); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -738,9 +766,10 @@ internal static void InitializeSlots(IntPtr type, Type impl) ret0 = NativeCodePage + native.Return0; } - InitializeSlot(type, ret0, "tp_traverse"); - InitializeSlot(type, ret0, "tp_clear"); - InitializeSlot(type, ret1, "tp_is_gc"); + bool canOverride = !PythonEngine.SoftShutdown; + InitializeSlot(type, ret0, "tp_traverse", canOverride); + InitializeSlot(type, ret0, "tp_clear", canOverride); + // InitializeSlot(type, ret1, "tp_is_gc"); } static int Return1(IntPtr _) => 1; @@ -757,12 +786,17 @@ internal static void InitializeSlots(IntPtr type, Type impl) /// Type being initialized. /// Function pointer. /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) + /// Can override the slot when it existed + static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) { Type typeOffset = typeof(TypeOffset); FieldInfo fi = typeOffset.GetField(name); var offset = (int)fi.GetValue(typeOffset); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } Marshal.WriteIntPtr(type, offset, slot); } From 657452e7d4b207612862d53ab647b8afd7906db7 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 19 Sep 2019 23:48:56 +0800 Subject: [PATCH 005/134] * Fix refcnt error * Keep delegate of Thunk in tp_dict * Remove useless code --- src/runtime/Python.Runtime.csproj | 19 ++-- src/runtime/classmanager.cs | 5 +- src/runtime/extensiontype.cs | 4 +- src/runtime/importhook.cs | 3 +- src/runtime/interop.cs | 38 ++++---- src/runtime/methodwrapper.cs | 27 +++--- src/runtime/pythonengine.cs | 7 +- src/runtime/runtime.cs | 122 ++++++++++-------------- src/runtime/runtime_state.cs | 26 ------ src/runtime/typemanager.cs | 149 +++++++++++++++++++++++++----- 10 files changed, 221 insertions(+), 179 deletions(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index ac6b59150..1f3a0e392 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -34,7 +34,7 @@ pdbonly - PYTHON3;PYTHON37;UCS4 + PYTHON3;PYTHON37;UCS4 true pdbonly @@ -46,7 +46,7 @@ true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG + PYTHON3;PYTHON37;UCS4;TRACE;DEBUG false full @@ -56,7 +56,7 @@ pdbonly - PYTHON3;PYTHON37;UCS2 + PYTHON3;PYTHON37;UCS2 true pdbonly @@ -68,7 +68,7 @@ true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG + PYTHON3;PYTHON37;UCS2;TRACE;DEBUG false full @@ -137,11 +137,12 @@ + - - + + @@ -151,7 +152,7 @@ - + @@ -170,4 +171,4 @@ - + \ No newline at end of file diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 49b732910..3343371f1 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -56,7 +56,6 @@ internal static void RemoveClasses() cls.TypeTraverse(OnVisit, visitedPtr); // XXX: Force release some resouces. cls.TypeClear(); - //Runtime.XDecref(cls.pyHandle); } } finally @@ -189,8 +188,7 @@ private static void InitClassBase(Type type, ClassBase impl) var name = (string)iter.Key; Runtime.PyDict_SetItemString(dict, name, item.pyHandle); // info.members are already useless - Runtime.XDecref(item.pyHandle); - item.pyHandle = IntPtr.Zero; + item.DecrRefCount(); } // If class has constructors, generate an __doc__ attribute. @@ -225,6 +223,7 @@ private static void InitClassBase(Type type, ClassBase impl) // TODO: deprecate __overloads__ soon... Runtime.PyDict_SetItemString(dict, "__overloads__", ctors.pyHandle); Runtime.PyDict_SetItemString(dict, "Overloads", ctors.pyHandle); + ctors.DecrRefCount(); } // don't generate the docstring if one was already set from a DocStringAttribute. diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 5e8ee6b59..634b1d1a5 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -38,6 +38,7 @@ public ExtensionType() Runtime.PyObject_GC_UnTrack(py); + // Steals a ref to tpHandle. tpHandle = tp; pyHandle = py; } @@ -49,7 +50,8 @@ public ExtensionType() public static void FinalizeObject(ManagedType self) { Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); + // Not necessary decref for `tpHandle`. + // Runtime.XDecref(self.tpHandle); self.FreeGCHandle(); } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index ef322648a..22443c9ad 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -43,7 +43,6 @@ internal static void Initialize() py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); Runtime.PyObject_SetAttrString(mod, "__import__", hook.ptr); - Runtime.XDecref(hook.ptr); root = new CLRModule(); @@ -88,7 +87,7 @@ internal static void Shutdown() Runtime.XDecref(root.pyHandle); root = null; - hook.Dispose(); + hook.Release(); hook = null; Runtime.XDecref(py_import); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 702bb7982..7b03c58a2 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -335,7 +335,6 @@ internal class TypeFlags internal class Interop { - private static Dictionary, Tuple> keepAlive; private static Hashtable pmap; static Interop() @@ -352,8 +351,6 @@ static Interop() p[item.Name] = item; } - keepAlive = new Dictionary, Tuple>(); - //Marshal.AllocHGlobal(IntPtr.Size); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -450,24 +447,12 @@ internal static Type GetPrototype(string name) return pmap[name] as Type; } - internal static IntPtr GetThunk(MethodInfo method, string funcType = null) + internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { - var key = new KeyValuePair(method, funcType); - Tuple thunkPair; - if (keepAlive.TryGetValue(key, out thunkPair)) - { - return thunkPair.Item2; - } - thunkPair = GetThunkImpl(method, funcType); - if (thunkPair == null) - { - return IntPtr.Zero; - } - keepAlive[key] = thunkPair; - return thunkPair.Item2; + return GetThunkImpl(method, funcType); } - private static Tuple GetThunkImpl(MethodInfo method, string funcType) + private static ThunkInfo GetThunkImpl(MethodInfo method, string funcType) { Type dt; if (funcType != null) @@ -477,7 +462,7 @@ private static Tuple GetThunkImpl(MethodInfo method, string fu if (dt == null) { - return null; + return ThunkInfo.Empty; } IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); Delegate d = Delegate.CreateDelegate(dt, method); @@ -485,7 +470,7 @@ private static Tuple GetThunkImpl(MethodInfo method, string fu Marshal.StructureToPtr(cb, tmp, false); IntPtr fp = Marshal.ReadIntPtr(tmp, 0); Marshal.FreeHGlobal(tmp); - return new Tuple(d, fp); + return new ThunkInfo(d, fp); } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -540,6 +525,19 @@ public Thunk(Delegate d) } } + internal struct ThunkInfo + { + public Delegate Target; + public IntPtr Address; + + public static readonly ThunkInfo Empty = new ThunkInfo(null, IntPtr.Zero); + + public ThunkInfo(Delegate target, IntPtr addr) + { + Target = target; + Address = addr; + } + } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] struct PyGC_Node diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index ebcea0709..214a48dc6 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -8,51 +8,46 @@ namespace Python.Runtime /// currently used mainly to implement special cases like the CLR /// import hook. /// - internal class MethodWrapper : IDisposable + internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + public ThunkInfo _thunk; private bool _disposed = false; public MethodWrapper(Type type, string name, string funcType = null) { // Turn the managed method into a function pointer - IntPtr fp = Interop.GetThunk(type.GetMethod(name), funcType); + _thunk = Interop.GetThunk(type.GetMethod(name), funcType); // Allocate and initialize a PyMethodDef structure to represent // the managed method, then create a PyCFunction. mdef = Runtime.PyMem_Malloc(4 * IntPtr.Size); - TypeManager.WriteMethodDef(mdef, name, fp, 0x0003); + TypeManager.WriteMethodDef(mdef, name, _thunk.Address, 0x0003); ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } - ~MethodWrapper() - { - Dispose(); - } public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); } - public void Dispose() + public void Release() { if (_disposed) { return; } - using (Py.GIL()) + _disposed = true; + bool freeDef = Runtime.Refcount(ptr) == 1; + Runtime.XDecref(ptr); + if (freeDef && mdef != IntPtr.Zero) { - _disposed = true; - Runtime.XDecref(ptr); - if (mdef != IntPtr.Zero) - { - Runtime.PyMem_Free(mdef); - mdef = IntPtr.Zero; - } + Runtime.PyMem_Free(mdef); + mdef = IntPtr.Zero; } } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index cf4482c85..f427d4a44 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -213,12 +213,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); Assembly assembly = Assembly.GetExecutingAssembly(); - Stream stream = assembly.GetManifestResourceStream("clr.py"); - if (stream == null) - { - stream = File.OpenRead(@"I:\repos\pythonnet\src\runtime\resources\clr.py"); - } - using (stream) + using (Stream stream = assembly.GetManifestResourceStream("clr.py")) using (var reader = new StreamReader(stream)) { // add the contents of clr.py to the module diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index fdccdf5ab..2b8460e14 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -196,50 +196,39 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false TypeManager.Reset(); IntPtr op; - IntPtr dict; - if (IsPython3) - { - op = PyImport_ImportModule("builtins"); - dict = PyObject_GetAttrString(op, "__dict__"); - } - else // Python2 + _typeRefs = new PyReferenceCollection(); { - dict = PyImport_GetModuleDict(); - op = PyDict_GetItemString(dict, "__builtin__"); - } + var builtins = IsPython3 ? PyImport_ImportModule("builtins") + : PyImport_ImportModule("__builtin__"); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented")); - _typeRefs = new PyReferenceCollection(); - SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(op, "NotImplemented")); + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object")); - SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(op, "object")); + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None")); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True")); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False")); - SetPyMember(ref PyNone, PyObject_GetAttrString(op, "None")); - SetPyMember(ref PyTrue, PyObject_GetAttrString(op, "True")); - SetPyMember(ref PyFalse, PyObject_GetAttrString(op, "False")); + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); - SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); - SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); - SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); + op = PyObject_GetAttrString(builtins, "len"); + SetPyMember(ref PyMethodType, PyObject_Type(op)); + XDecref(op); - op = PyObject_GetAttrString(dict, "keys"); - SetPyMember(ref PyMethodType, PyObject_Type(op)); - XDecref(op); + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); + XDecref(op); - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); - XDecref(op); + op = PyObject_GetAttrString(builtins, "KeyError"); + SetPyMember(ref PyExc_KeyError, op); - op = PyDict_GetItemString(dict, "KeyError"); - XIncref(op); - SetPyMember(ref PyExc_KeyError, op); - -#if PYTHON3 - XDecref(dict); -#endif + XDecref(builtins); + } op = PyString_FromString("string"); SetPyMember(ref PyStringType, PyObject_Type(op)); @@ -287,10 +276,10 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false IntPtr d = PyDict_New(); IntPtr c = PyClass_New(IntPtr.Zero, d, s); - SetPyObject(ref PyClassType, PyObject_Type(c)); + SetPyMember(ref PyClassType, PyObject_Type(c)); IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - SetPyObject(ref PyInstanceType, PyObject_Type(i)); + SetPyMember(ref PyInstanceType, PyObject_Type(i)); XDecref(s); XDecref(i); @@ -322,7 +311,9 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); - SetPyMember(ref PyCLRMetaType, MetaType.Initialize()); + var metaType = MetaType.Initialize(); + XIncref(metaType); + SetPyMember(ref PyCLRMetaType, metaType); Exceptions.Initialize(); ImportHook.Initialize(); @@ -398,47 +389,12 @@ internal static void Shutdown(bool soft) ClassManager.RemoveClasses(); TypeManager.RemoveTypes(); RemoveClrRootModule(); - - { - //var intialModules = PySys_GetObject("initial_modules"); - //var modules = PyImport_GetModuleDict(); - //foreach (var name in RuntimeState.GetModuleNames()) - //{ - // if (PySet_Contains(intialModules, name) == 1) - // { - // continue; - // } - // var module = PyDict_GetItem(modules, name); - // IntPtr clr_dict = Runtime._PyObject_GetDictPtr(module); // PyObject** - // clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); - // PyDict_Clear(clr_dict); - // PyDict_DelItem(modules, name); - //} - } if (soft) { PyGC_Collect(); RemoveClrObjects(); RuntimeState.Restore(); - //var objs = ManagedType.GetManagedObjects(); - //var subIterp = PyThreadState_Swap(_originIterp); - //var iterp = PyThreadState_Get(); - //Py_EndInterpreter(iterp); - //PyGILState_Ensure(); ResetPyMembers(); -// Runtime.PyRun_SimpleString(@" -//import gc, pprint, objgraph -//print() -//with open('R:/objsx.log', 'w', encoding='utf8') as f: -// pprint.pprint(objgraph.show_growth(), f) -// #pprint.pprint(objgraph.typestats(), f) - -//#lst = gc.get_objects() -//#with open('R:/objs.log', 'w', encoding='utf8') as f: -// # for obj in lst: -// # pprint.pprint(obj, f) -//#del lst -//"); return; } ResetPyMembers(); @@ -512,6 +468,7 @@ private static void PyDictTryDelItem(IntPtr dict, string key) { throw new PythonException(); } + PyErr_Clear(); } private static void RemoveClrObjects() @@ -1620,9 +1577,15 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, string key); + /// + /// Return 0 on success or -1 on failure. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItem(IntPtr pointer, IntPtr key, IntPtr value); + /// + /// Return 0 on success or -1 on failure. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItemString(IntPtr pointer, string key, IntPtr value); @@ -2069,6 +2032,17 @@ internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; } + //==================================================================== + // Python Capsules API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_GetPointer(IntPtr capsule, string name); + + //==================================================================== // Miscellaneous //==================================================================== diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 69291453e..44729bd62 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -76,34 +76,8 @@ public static void Restore() throw new Exception("Runtime state have not set"); } var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); - //foreach (var abandomedNode in IterGCNodes(dummyGC)) - //{ - // if (abandomedNode == IntPtr.Zero) - // { - // continue; - // } - // //PyObject_GC_Del(_Py_FROM_GC(abandomedNode)); - //} - - //unsafe - //{ - // var head = (PyGC_Head*)dummyGC; - // head->gc.gc_next = dummyGC; - // head->gc.gc_prev = dummyGC; - // head->gc.gc_refs = IntPtr.Zero; - //} - ResotreModules(dummyGC); RestoreObjects(dummyGC); - foreach (var obj in IterObjects(dummyGC)) - { - //if (ManagedType.IsManagedType(obj)) - { - //var clrobj = ManagedType.GetManagedObject(obj); - //Console.WriteLine(clrobj); - } - } - //Console.WriteLine($"abndom {IterGCNodes(dummyGC).Count()}"); } private static void ResotreModules(IntPtr dummyGC) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index f82260d37..97bfc89ad 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -103,7 +103,8 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = new SlotsHolder(); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; @@ -119,8 +120,11 @@ internal static IntPtr CreateType(Type impl) Runtime.PyDict_SetItemString(dict, "__module__", mod); Runtime.XDecref(mod); - InitMethods(type, impl); + IntPtr capsule = slotsHolder.ToCapsule(); + Runtime.PyDict_SetItemString(dict, "_slotsHodler", capsule); + Runtime.XDecref(capsule); + InitMethods(type, impl); return type; } @@ -172,7 +176,8 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); - InitializeSlots(type, impl.GetType()); + SlotsHolder slotsHolder = new SlotsHolder(); + InitializeSlots(type, impl.GetType(), slotsHolder); if (base_ != IntPtr.Zero) { @@ -202,6 +207,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.PyDict_SetItemString(dict, "__module__", mod); Runtime.XDecref(mod); + IntPtr capsule = slotsHolder.ToCapsule(); + Runtime.PyDict_SetItemString(dict, "_slotsHodler", capsule); + Runtime.XDecref(capsule); + // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); @@ -343,9 +352,9 @@ internal static IntPtr CreateMetaType(Type impl) //CopySlot(py_type, type, TypeOffset.tp_clear); //CopySlot(py_type, type, TypeOffset.tp_is_gc); + SlotsHolder slotsHolder = new SlotsHolder(); // Override type slots with those of the managed implementation. - - InitializeSlots(type, impl); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default; flags |= TypeFlags.Managed; @@ -357,16 +366,20 @@ internal static IntPtr CreateMetaType(Type impl) // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); IntPtr mdefStart = mdef; + ThunkInfo thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); + slotsHolder.Add(mdef, thunk.Target); mdef = WriteMethodDef( mdef, "__instancecheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc") + thunk.Address ); + thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); + slotsHolder.Add(mdef, thunk.Target); mdef = WriteMethodDef( mdef, "__subclasscheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc") + thunk.Address ); // FIXME: mdef is not used @@ -383,6 +396,10 @@ internal static IntPtr CreateMetaType(Type impl) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + IntPtr capsule = slotsHolder.ToCapsule(); + Runtime.PyDict_SetItemString(dict, "_slotsHodler", capsule); + Runtime.XDecref(capsule); + //DebugUtil.DumpType(type); return type; @@ -417,7 +434,8 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = new SlotsHolder(); + InitializeSlots(type, impl, slotsHolder); if (Runtime.PyType_Ready(type) != 0) { @@ -428,6 +446,9 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(tp_dict, "__module__", mod); + IntPtr capsule = slotsHolder.ToCapsule(); + Runtime.PyDict_SetItemString(tp_dict, "_slotsHodler", capsule); + Runtime.XDecref(capsule); return type; } @@ -705,7 +726,7 @@ internal static void InitializeNativeCodePage() /// provides the implementation for the type, connect the type slots of /// the Python object to the managed methods of the implementing Type. /// - internal static void InitializeSlots(IntPtr type, Type impl) + internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHolder = null) { // We work from the most-derived class up; make sure to get // the most-derived slot and not to override it with a base @@ -733,7 +754,7 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); seen.Add(name); } @@ -741,6 +762,7 @@ internal static void InitializeSlots(IntPtr type, Type impl) impl = impl.BaseType; } + bool canOverride = !PythonEngine.SoftShutdown; var native = NativeCode.Active; // The garbage collection related slots always have to return 1 or 0 @@ -750,10 +772,6 @@ internal static void InitializeSlots(IntPtr type, Type impl) // tp_is_gc (returns 1) // These have to be defined, though, so by default we fill these with // static C# functions from this class. - - var ret0 = Interop.GetThunk(((Func)Return0).Method); - var ret1 = Interop.GetThunk(((Func)Return1).Method); - if (native != null) { // If we want to support domain reload, the C# implementation @@ -762,14 +780,35 @@ internal static void InitializeSlots(IntPtr type, Type impl) // load them into a separate code page that is leaked // intentionally. InitializeNativeCodePage(); - ret1 = NativeCodePage + native.Return1; - ret0 = NativeCodePage + native.Return0; - } + IntPtr ret1 = NativeCodePage + native.Return1; + IntPtr ret0 = NativeCodePage + native.Return0; - bool canOverride = !PythonEngine.SoftShutdown; - InitializeSlot(type, ret0, "tp_traverse", canOverride); - InitializeSlot(type, ret0, "tp_clear", canOverride); - // InitializeSlot(type, ret1, "tp_is_gc"); + InitializeSlot(type, ret0, "tp_traverse", canOverride); + InitializeSlot(type, ret0, "tp_clear", canOverride); + } + else + { + if (!IsSlotSet(type, "tp_traverse")) + { + var thunkRet0 = Interop.GetThunk(((Func)Return0).Method); + var offset = GetSlotOffset("tp_traverse"); + Marshal.WriteIntPtr(type, offset, thunkRet0.Address); + if (slotsHolder != null) + { + slotsHolder.Add(type + offset, thunkRet0.Target); + } + } + if (!IsSlotSet(type, "tp_clear")) + { + var thunkRet0 = Interop.GetThunk(((Func)Return0).Method); + var offset = GetSlotOffset("tp_clear"); + Marshal.WriteIntPtr(type, offset, thunkRet0.Address); + if (slotsHolder != null) + { + slotsHolder.Add(type + offset, thunkRet0.Target); + } + } + } } static int Return1(IntPtr _) => 1; @@ -788,6 +827,16 @@ internal static void InitializeSlots(IntPtr type, Type impl) /// Name of the method. /// Can override the slot when it existed static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) + { + var offset = GetSlotOffset(name); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, slot); + } + + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { Type typeOffset = typeof(TypeOffset); FieldInfo fi = typeOffset.GetField(name); @@ -797,7 +846,22 @@ static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverri { return; } - Marshal.WriteIntPtr(type, offset, slot); + Marshal.WriteIntPtr(type, offset, thunk.Address); + slotsHolder.Add(type + offset, thunk.Target); + } + + static int GetSlotOffset(string name) + { + Type typeOffset = typeof(TypeOffset); + FieldInfo fi = typeOffset.GetField(name); + var offset = (int)fi.GetValue(typeOffset); + return offset; + } + + static bool IsSlotSet(IntPtr type, string name) + { + int offset = GetSlotOffset(name); + return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } /// @@ -846,4 +910,45 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) Marshal.WriteIntPtr(to, offset, fp); } } + + + class SlotsHolder + { + private List> _slots = new List>(); + private GCHandle _handle; + private Interop.DestructorFunc _destructor; + private IntPtr _capsule; + + public void Add(IntPtr slotPtr, Delegate d) + { + _slots.Add(new KeyValuePair(slotPtr, d)); + } + + public IntPtr ToCapsule() + { + if (_capsule != IntPtr.Zero) + { + Runtime.XIncref(_capsule); + return _capsule; + } + _handle = GCHandle.Alloc(this); + _destructor = OnDestruct; + var fp = Marshal.GetFunctionPointerForDelegate(_destructor); + _capsule = Runtime.PyCapsule_New((IntPtr)_handle, null, fp); + return _capsule; + } + + private static void OnDestruct(IntPtr ob) + { + var ptr = Runtime.PyCapsule_GetPointer(ob, null); + PythonException.ThrowIfIsNull(ptr); + var handle = GCHandle.FromIntPtr(ptr); + var self = (SlotsHolder)handle.Target; + handle.Free(); + foreach (var item in self._slots) + { + Marshal.WriteIntPtr(item.Key, IntPtr.Zero); + } + } + } } From 91f64b94a027a13d7b97113cd648123018516a30 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 20 Sep 2019 15:56:43 +0800 Subject: [PATCH 006/134] Fixed leaking of tp_name --- src/runtime/runtime.cs | 4 ++++ src/runtime/typemanager.cs | 8 ++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2b8460e14..25c520e38 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1408,6 +1408,10 @@ internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); + #elif PYTHON2 internal static IntPtr PyString_FromStringAndSize(string value, long size) { diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 97bfc89ad..f50aed37f 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -464,12 +464,8 @@ internal static IntPtr AllocateTypeObject(string name) // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. #if PYTHON3 - // For python3 we leak two objects. One for the ASCII representation - // required for tp_name, and another for the Unicode representation - // for ht_name. - IntPtr temp = Runtime.PyBytes_FromString(name); - IntPtr raw = Runtime.PyBytes_AS_STRING(temp); - temp = Runtime.PyUnicode_FromString(name); + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); #elif PYTHON2 IntPtr temp = Runtime.PyString_FromString(name); IntPtr raw = Runtime.PyString_AsString(temp); From b07b844908c56da0bf9152f2e39e4b2b52dc82a8 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 20 Sep 2019 23:20:27 +0800 Subject: [PATCH 007/134] * Reset type slots * Fix ref error at PythoneEngine.With --- src/runtime/classbase.cs | 17 +--- src/runtime/classmanager.cs | 2 +- src/runtime/extensiontype.cs | 16 --- src/runtime/finalizer.cs | 2 +- src/runtime/interop.cs | 25 ++++- src/runtime/metatype.cs | 9 ++ src/runtime/methodbinding.cs | 10 +- src/runtime/methodobject.cs | 4 +- src/runtime/moduleobject.cs | 12 +-- src/runtime/pythonengine.cs | 11 ++- src/runtime/runtime.cs | 18 ++-- src/runtime/typemanager.cs | 187 +++++++++++++++++++++++++++++++---- 12 files changed, 226 insertions(+), 87 deletions(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index af00c58fe..10882009d 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -29,13 +29,6 @@ internal virtual bool CanSubclass() return !type.IsEnum; } - /// - /// Implements __init__ for reflected classes and value types. - /// - public static int tp_init(IntPtr ob, IntPtr args, IntPtr kw) - { - return 0; - } /// /// Default implementation of [] semantics for reflected types. @@ -254,13 +247,9 @@ public static IntPtr tp_str(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - if (self.pyHandle != self.tpHandle) - { - ClearObjectDict(ob); - } + tp_clear(ob); Runtime.PyObject_GC_UnTrack(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); self.FreeGCHandle(); } @@ -271,9 +260,7 @@ public static int tp_clear(IntPtr ob) { ClearObjectDict(ob); } - Runtime.XDecref(self.tpHandle); - self.tpHandle = IntPtr.Zero; - self.FreeGCHandle(); + Runtime.Py_CLEAR(ref self.tpHandle); return 0; } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 3343371f1..43892aabc 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -54,7 +54,7 @@ internal static void RemoveClasses() foreach (var cls in cache.Values) { cls.TypeTraverse(OnVisit, visitedPtr); - // XXX: Force release some resouces. + // XXX: Force release instance resources but not dealloc itself. cls.TypeClear(); } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 634b1d1a5..8d30da16e 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -92,21 +92,5 @@ public static void tp_dealloc(IntPtr ob) ManagedType self = GetManagedObject(ob); FinalizeObject(self); } - - public static int tp_clear(IntPtr ob) - { - ManagedType self = GetManagedObject(ob); - Runtime.XDecref(self.tpHandle); - self.FreeGCHandle(); - return 0; - } - - //public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) - //{ - // ManagedType self = GetManagedObject(ob); - // int res = PyVisit(self.tpHandle, visit, arg); - // if (res != 0) return res; - // return 0; - //} } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index dd5c0b4dd..7630b843f 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -59,7 +59,7 @@ public IncorrectRefCountException(IntPtr ptr) IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); string name = Runtime.GetManagedString(pyname); Runtime.XDecref(pyname); - _message = $"{name} may has a incorrect ref count"; + _message = $"<{name}> may has a incorrect ref count"; } } diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 7b03c58a2..fd811d1b1 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Text; using System.Collections.Generic; +using System.Linq; namespace Python.Runtime { @@ -262,6 +263,14 @@ public static void FreeModuleDef(IntPtr ptr) } #endif // PYTHON3 + static class TypeOffsetHelper + { + public static string GetSlotNameByOffset(int offset) + { + return typeof(TypeOffset).GetFields().First(fi => (int)fi.GetValue(null) == offset).Name; + } + } + /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. @@ -448,11 +457,6 @@ internal static Type GetPrototype(string name) } internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) - { - return GetThunkImpl(method, funcType); - } - - private static ThunkInfo GetThunkImpl(MethodInfo method, string funcType) { Type dt; if (funcType != null) @@ -473,6 +477,7 @@ private static ThunkInfo GetThunkImpl(MethodInfo method, string funcType) return new ThunkInfo(d, fp); } + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate IntPtr UnaryFunc(IntPtr ob); @@ -552,4 +557,14 @@ struct PyGC_Head { public PyGC_Node gc; } + + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + struct PyMethodDef + { + public IntPtr ml_name; + public IntPtr ml_meth; + public int ml_flags; + public IntPtr ml_doc; + } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..112566d7f 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -22,6 +22,15 @@ public static IntPtr Initialize() return PyCLRMetaType; } + public static void Release() + { + if (Runtime.Refcount(PyCLRMetaType) > 1) + { + SlotsHolder.ReleaseTypeSlots(PyCLRMetaType); + } + Runtime.XDecref(PyCLRMetaType); + PyCLRMetaType = IntPtr.Zero; + } /// /// Metatype __new__ implementation. This is called to create a new diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index b917d9bb9..5c4164067 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -38,10 +38,8 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer private void ClearMembers() { - Runtime.XDecref(target); - target = IntPtr.Zero; - Runtime.XDecref(targetType); - targetType = IntPtr.Zero; + Runtime.Py_CLEAR(ref target); + Runtime.Py_CLEAR(ref targetType); } /// @@ -249,11 +247,11 @@ public static IntPtr tp_repr(IntPtr ob) FinalizeObject(self); } - public new static int tp_clear(IntPtr ob) + public static int tp_clear(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); self.ClearMembers(); - return ExtensionType.tp_clear(ob); + return 0; } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 2531bc285..847183c73 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -211,11 +211,11 @@ public static IntPtr tp_repr(IntPtr ob) ExtensionType.FinalizeObject(self); } - public new static int tp_clear(IntPtr ob) + public static int tp_clear(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); self.ClearMembers(); - return ExtensionType.tp_clear(ob); + return 0; } } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 23488f676..717cc5028 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -304,13 +304,7 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (ModuleObject)GetManagedObject(ob); - Runtime.XDecref(self.dict); - self.dict = IntPtr.Zero; - foreach (var attr in self.cache.Values) - { - Runtime.XDecref(attr.pyHandle); - } - self.cache.Clear(); + tp_clear(ob); ExtensionType.tp_dealloc(ob); } @@ -327,7 +321,7 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) return 0; } - public new static int tp_clear(IntPtr ob) + public static int tp_clear(IntPtr ob) { var self = (ModuleObject)GetManagedObject(ob); Runtime.XDecref(self.dict); @@ -337,7 +331,7 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) Runtime.XDecref(attr.pyHandle); } self.cache.Clear(); - return ExtensionType.tp_clear(ob); + return 0; } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index f427d4a44..0ad928caa 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -194,10 +194,10 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // register the atexit callback (this doesn't use Py_AtExit as the C atexit // callbacks are called after python is fully finalized but the python ones // are called while the python engine is still running). - string code = - "import atexit, clr\n" + - "atexit.register(clr._AtExit)\n"; - PythonEngine.Exec(code); + //string code = + // "import atexit, clr\n" + + // "atexit.register(clr._AtExit)\n"; + //PythonEngine.Exec(code); } // Load the clr.py resource into the clr module @@ -756,6 +756,9 @@ public static void With(PyObject obj, Action Body) traceBack = ex.PyTB; } + Runtime.XIncref(type); + Runtime.XIncref(val); + Runtime.XIncref(traceBack); var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack)); if (ex != null && !exitResult.IsTrue()) throw ex; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 25c520e38..484649694 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -311,9 +311,7 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); - var metaType = MetaType.Initialize(); - XIncref(metaType); - SetPyMember(ref PyCLRMetaType, metaType); + PyCLRMetaType = MetaType.Initialize(); // Steal a reference Exceptions.Initialize(); ImportHook.Initialize(); @@ -388,11 +386,14 @@ internal static void Shutdown(bool soft) ClearClrModules(); ClassManager.RemoveClasses(); TypeManager.RemoveTypes(); + + MetaType.Release(); + PyCLRMetaType = IntPtr.Zero; + RemoveClrRootModule(); if (soft) { PyGC_Collect(); - RemoveClrObjects(); RuntimeState.Restore(); ResetPyMembers(); return; @@ -471,9 +472,6 @@ private static void PyDictTryDelItem(IntPtr dict, string key) PyErr_Clear(); } - private static void RemoveClrObjects() - { - } internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; @@ -2036,6 +2034,12 @@ internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; } + internal static void Py_CLEAR(ref IntPtr ob) + { + XDecref(ob); + ob = IntPtr.Zero; + } + //==================================================================== // Python Capsules API //==================================================================== diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index f50aed37f..0663ad081 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using Python.Runtime.Platform; using System.Linq; +using System.Diagnostics; namespace Python.Runtime { @@ -37,6 +38,12 @@ internal static void RemoveTypes() { foreach (var tpHandle in cache.Values) { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + SlotsHolder.ReleaseTypeSlots(tpHandle); + } Runtime.XDecref(tpHandle); } cache.Clear(); @@ -103,7 +110,7 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - SlotsHolder slotsHolder = new SlotsHolder(); + SlotsHolder slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | @@ -121,7 +128,7 @@ internal static IntPtr CreateType(Type impl) Runtime.XDecref(mod); IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(dict, "_slotsHodler", capsule); + Runtime.PyDict_SetItemString(dict, SlotsHolder.HolderKeyName, capsule); Runtime.XDecref(capsule); InitMethods(type, impl); @@ -176,7 +183,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); - SlotsHolder slotsHolder = new SlotsHolder(); + SlotsHolder slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl.GetType(), slotsHolder); if (base_ != IntPtr.Zero) @@ -208,7 +215,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XDecref(mod); IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(dict, "_slotsHodler", capsule); + Runtime.PyDict_SetItemString(dict, SlotsHolder.HolderKeyName, capsule); Runtime.XDecref(capsule); // Hide the gchandle of the implementation in a magic type slot. @@ -327,6 +334,24 @@ internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); } + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + internal static IntPtr CreateMetaType(Type impl) { // The managed metatype is functionally little different than the @@ -352,7 +377,7 @@ internal static IntPtr CreateMetaType(Type impl) //CopySlot(py_type, type, TypeOffset.tp_clear); //CopySlot(py_type, type, TypeOffset.tp_is_gc); - SlotsHolder slotsHolder = new SlotsHolder(); + SlotsHolder slotsHolder = new SlotsHolder(type); // Override type slots with those of the managed implementation. InitializeSlots(type, impl, slotsHolder); @@ -365,9 +390,18 @@ internal static IntPtr CreateMetaType(Type impl) // We need space for 3 PyMethodDef structs, each of them // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); + Debug.Assert(4 * IntPtr.Size == Marshal.SizeOf(typeof(PyMethodDef))); IntPtr mdefStart = mdef; ThunkInfo thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); - slotsHolder.Add(mdef, thunk.Target); + slotsHolder.KeeapAlive(thunk.Target); + + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + FreeMethodDef(mdefAddr); + }); + } mdef = WriteMethodDef( mdef, "__instancecheck__", @@ -375,7 +409,14 @@ internal static IntPtr CreateMetaType(Type impl) ); thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); - slotsHolder.Add(mdef, thunk.Target); + slotsHolder.KeeapAlive(thunk.Target); + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + FreeMethodDef(mdefAddr); + }); + } mdef = WriteMethodDef( mdef, "__subclasscheck__", @@ -386,6 +427,12 @@ internal static IntPtr CreateMetaType(Type impl) mdef = WriteMethodDefSentinel(mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + slotsHolder.Add(TypeOffset.tp_methods, (t, offset) => + { + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); + }); if (Runtime.PyType_Ready(type) != 0) { @@ -397,7 +444,7 @@ internal static IntPtr CreateMetaType(Type impl) Runtime.PyDict_SetItemString(dict, "__module__", mod); IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(dict, "_slotsHodler", capsule); + Runtime.PyDict_SetItemString(dict, SlotsHolder.HolderKeyName, capsule); Runtime.XDecref(capsule); //DebugUtil.DumpType(type); @@ -434,7 +481,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - SlotsHolder slotsHolder = new SlotsHolder(); + SlotsHolder slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl, slotsHolder); if (Runtime.PyType_Ready(type) != 0) @@ -447,7 +494,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) Runtime.PyDict_SetItemString(tp_dict, "__module__", mod); IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(tp_dict, "_slotsHodler", capsule); + Runtime.PyDict_SetItemString(tp_dict, SlotsHolder.HolderKeyName, capsule); Runtime.XDecref(capsule); return type; } @@ -791,7 +838,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo Marshal.WriteIntPtr(type, offset, thunkRet0.Address); if (slotsHolder != null) { - slotsHolder.Add(type + offset, thunkRet0.Target); + slotsHolder.Add(offset, thunkRet0); } } if (!IsSlotSet(type, "tp_clear")) @@ -801,7 +848,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo Marshal.WriteIntPtr(type, offset, thunkRet0.Address); if (slotsHolder != null) { - slotsHolder.Add(type + offset, thunkRet0.Target); + slotsHolder.Add(offset, thunkRet0); } } } @@ -843,7 +890,7 @@ static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolde return; } Marshal.WriteIntPtr(type, offset, thunk.Address); - slotsHolder.Add(type + offset, thunk.Target); + slotsHolder.Add(offset, thunk); } static int GetSlotOffset(string name) @@ -910,14 +957,45 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) class SlotsHolder { - private List> _slots = new List>(); + public const string HolderKeyName = "_slots_holder"; + public delegate void Resetor(IntPtr type, int offset); + private GCHandle _handle; private Interop.DestructorFunc _destructor; private IntPtr _capsule; + private IntPtr _type; + private Dictionary _slots = new Dictionary(); + private List _keepalive = new List(); + private Dictionary _customRestors = new Dictionary(); + private List _deallocators = new List(); + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(IntPtr type) + { + _type = type; + } + + public void Add(int offset, ThunkInfo thunk) + { + _slots.Add(offset, thunk); + } + + public void Add(int offset, Resetor resetor) + { + _customRestors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + _deallocators.Add(deallocate); + } - public void Add(IntPtr slotPtr, Delegate d) + public void KeeapAlive(Delegate d) { - _slots.Add(new KeyValuePair(slotPtr, d)); + _keepalive.Add(d); } public IntPtr ToCapsule() @@ -934,17 +1012,84 @@ public IntPtr ToCapsule() return _capsule; } + public static void ReleaseTypeSlots(IntPtr type) + { + IntPtr capsule = Runtime.PyObject_GetAttrString(type, HolderKeyName); + if (capsule == IntPtr.Zero) + { + return; + } + var self = RecoverFromCapsule(capsule); + self.ResetSlots(); + } + + private void ResetSlots() + { + IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); + foreach (var offset in _slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); + Marshal.WriteIntPtr(_type, offset, ptr); + } + + foreach (var action in _deallocators) + { + action(); + } + + foreach (var pair in _customRestors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(_type, offset); + } + + _customRestors.Clear(); + _slots.Clear(); + _keepalive.Clear(); + _deallocators.Clear(); + + // Custom reset + IntPtr tp_base = Marshal.ReadIntPtr(_type, TypeOffset.tp_base); + Runtime.XDecref(tp_base); + Marshal.WriteIntPtr(_type, TypeOffset.tp_base, IntPtr.Zero); + } + private static void OnDestruct(IntPtr ob) + { + var self = RecoverFromCapsule(ob); + self._handle.Free(); + self.ResetSlots(); + } + + private static SlotsHolder RecoverFromCapsule(IntPtr ob) { var ptr = Runtime.PyCapsule_GetPointer(ob, null); PythonException.ThrowIfIsNull(ptr); - var handle = GCHandle.FromIntPtr(ptr); - var self = (SlotsHolder)handle.Target; - handle.Free(); - foreach (var item in self._slots) + GCHandle handle = GCHandle.FromIntPtr(ptr); + return (SlotsHolder)handle.Target; + } + + private static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear + || offset == TypeOffset.tp_traverse) + { + return TypeManager.NativeCodePage + TypeManager.NativeCode.Active.Return0; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObejct_GC_Del. + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) { - Marshal.WriteIntPtr(item.Key, IntPtr.Zero); + return IntPtr.Zero; } + + return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); } } } From 7db724ed9c93ecf67b5f9fc3ee94c217597f0f57 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 21 Sep 2019 01:59:13 +0800 Subject: [PATCH 008/134] Clear ExtensionType --- src/runtime/extensiontype.cs | 2 +- src/runtime/managedtype.cs | 31 +++++++++++++++---------------- src/runtime/runtime.cs | 23 ++++++++++++++++++++++- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 8d30da16e..02789e359 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -28,7 +28,7 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - GCHandle gc = AllocGCHandle(); + GCHandle gc = AllocGCHandle(true); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); // We have to support gc because the type machinery makes it very diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 6f0e8b88b..04e59a115 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Linq; namespace Python.Runtime { @@ -16,9 +17,7 @@ internal abstract class ManagedType internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * -#if NPY_TRACK_OBJECT private static readonly HashSet _managedObjs = new HashSet(); -#endif internal void DecrRefCount() { @@ -41,22 +40,24 @@ internal long RefCount } } - internal GCHandle AllocGCHandle() + internal GCHandle AllocGCHandle(bool track = false) { gcHandle = GCHandle.Alloc(this); -#if NPY_TRACK_OBJECT - _managedObjs.Add(this); -#endif + if (track) + { + _managedObjs.Add(this); + } return gcHandle; } internal void FreeGCHandle() { - gcHandle.Free(); -#if NPY_TRACK_OBJECT _managedObjs.Remove(this); -#endif - gcHandle = new GCHandle(); + if (gcHandle.IsAllocated) + { + gcHandle.Free(); + gcHandle = new GCHandle(); + } } /// @@ -120,17 +121,15 @@ internal static bool IsManagedType(IntPtr ob) return false; } -#if NPY_TRACK_OBJECT - internal static void Reset() + internal static ICollection GetManagedObjects() { - _managedObjs.Clear(); + return _managedObjs; } - internal static ICollection GetManagedObjects() + internal static void ClearTrackedObjects() { - return _managedObjs; + _managedObjs.Clear(); } -#endif internal static int PyVisit(IntPtr ob, IntPtr visit, IntPtr arg) { diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 484649694..1f9731855 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Collections.Generic; using Python.Runtime.Platform; +using System.Linq; namespace Python.Runtime { @@ -384,13 +385,15 @@ internal static void Shutdown(bool soft) Finalizer.Shutdown(); ClearClrModules(); + RemoveClrRootModule(); + + MoveClrInstancesOnwershipToPython(); ClassManager.RemoveClasses(); TypeManager.RemoveTypes(); MetaType.Release(); PyCLRMetaType = IntPtr.Zero; - RemoveClrRootModule(); if (soft) { PyGC_Collect(); @@ -472,6 +475,24 @@ private static void PyDictTryDelItem(IntPtr dict, string key) PyErr_Clear(); } + private static void MoveClrInstancesOnwershipToPython() + { + var copyObjs = ManagedType.GetManagedObjects().ToArray(); + var objs = ManagedType.GetManagedObjects(); + foreach (var obj in copyObjs) + { + if (objs.Contains(obj)) + { + continue; + } + obj.TypeClear(); + PyObject_GC_Track(obj.pyHandle); + obj.gcHandle.Free(); + obj.gcHandle = new GCHandle(); + } + ManagedType.ClearTrackedObjects(); + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; From 2a88be47a0fded6b1cb7e1aa71925d2d312a31fa Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 21 Sep 2019 04:00:01 +0800 Subject: [PATCH 009/134] Del SlotsHolder from tp_dict when shutting down --- src/runtime/metatype.cs | 4 +++- src/runtime/runtime.cs | 8 +++++++- src/runtime/typemanager.cs | 26 ++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 112566d7f..7cbab76c2 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -28,7 +28,8 @@ public static void Release() { SlotsHolder.ReleaseTypeSlots(PyCLRMetaType); } - Runtime.XDecref(PyCLRMetaType); + // FIXME: Crash on .netcore if decref PyCLRMetaType + // Runtime.XDecref(PyCLRMetaType); PyCLRMetaType = IntPtr.Zero; } @@ -275,6 +276,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) return Runtime.PyFalse; } + Runtime.XIncref(args); using (var argsObj = new PyList(args)) { if (argsObj.Length() != 1) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1f9731855..5e3c6e549 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -225,6 +225,8 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); XDecref(op); + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super")); + op = PyObject_GetAttrString(builtins, "KeyError"); SetPyMember(ref PyExc_KeyError, op); @@ -481,11 +483,13 @@ private static void MoveClrInstancesOnwershipToPython() var objs = ManagedType.GetManagedObjects(); foreach (var obj in copyObjs) { - if (objs.Contains(obj)) + if (!objs.Contains(obj)) { continue; } obj.TypeClear(); + // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), + // thus just be safe to give it back to GC chain. PyObject_GC_Track(obj.pyHandle); obj.gcHandle.Free(); obj.gcHandle = new GCHandle(); @@ -502,6 +506,7 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -895,6 +900,7 @@ public static extern int Py_Main( //==================================================================== /// + /// Return value: Borrowed reference. /// A macro-like method to get the type of a Python object. This is /// designed to be lean and mean in IL & avoid managed <-> unmanaged /// transitions. Note that this does not incref the type object. diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 0663ad081..401099ec0 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -826,8 +826,8 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo IntPtr ret1 = NativeCodePage + native.Return1; IntPtr ret0 = NativeCodePage + native.Return0; - InitializeSlot(type, ret0, "tp_traverse", canOverride); - InitializeSlot(type, ret0, "tp_clear", canOverride); + InitializeSlot(type, ret0, "tp_traverse", false); + InitializeSlot(type, ret0, "tp_clear", false); } else { @@ -968,6 +968,7 @@ class SlotsHolder private List _keepalive = new List(); private Dictionary _customRestors = new Dictionary(); private List _deallocators = new List(); + private bool _alredyReset = false; /// /// Create slots holder for holding the delegate of slots and be able to reset them. @@ -1021,10 +1022,22 @@ public static void ReleaseTypeSlots(IntPtr type) } var self = RecoverFromCapsule(capsule); self.ResetSlots(); + Runtime.XDecref(capsule); + + IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, HolderKeyName) != 0) + { + throw new PythonException(); + } } private void ResetSlots() { + if (_alredyReset) + { + return; + } + _alredyReset = true; IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); string typeName = Marshal.PtrToStringAnsi(tp_name); foreach (var offset in _slots.Keys) @@ -1055,6 +1068,10 @@ private void ResetSlots() IntPtr tp_base = Marshal.ReadIntPtr(_type, TypeOffset.tp_base); Runtime.XDecref(tp_base); Marshal.WriteIntPtr(_type, TypeOffset.tp_base, IntPtr.Zero); + + IntPtr tp_bases = Marshal.ReadIntPtr(_type, TypeOffset.tp_bases); + Runtime.XDecref(tp_bases); + Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, IntPtr.Zero); } private static void OnDestruct(IntPtr ob) @@ -1088,6 +1105,11 @@ private static IntPtr GetDefaultSlot(int offset) { return IntPtr.Zero; } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); } From 294097347dcf96924f64c246230402bbcdbbff03 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 21 Sep 2019 14:30:29 +0800 Subject: [PATCH 010/134] Fix refcnt error of qualname --- src/runtime/metatype.cs | 3 +-- src/runtime/typemanager.cs | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 7cbab76c2..51f3eddd7 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -28,8 +28,7 @@ public static void Release() { SlotsHolder.ReleaseTypeSlots(PyCLRMetaType); } - // FIXME: Crash on .netcore if decref PyCLRMetaType - // Runtime.XDecref(PyCLRMetaType); + Runtime.XDecref(PyCLRMetaType); PyCLRMetaType = IntPtr.Zero; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 401099ec0..0660258c5 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -399,6 +399,12 @@ internal static IntPtr CreateMetaType(Type impl) IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => { + IntPtr t = type; + IntPtr tp_dict = Marshal.ReadIntPtr(t, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, "__instancecheck__") != 0) + { + Runtime.PyErr_Print(); + } FreeMethodDef(mdefAddr); }); } @@ -414,6 +420,12 @@ internal static IntPtr CreateMetaType(Type impl) IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => { + IntPtr t = type; + IntPtr tp_dict = Marshal.ReadIntPtr(t, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, "__subclasscheck__") != 0) + { + Runtime.PyErr_Print(); + } FreeMethodDef(mdefAddr); }); } @@ -521,6 +533,7 @@ internal static IntPtr AllocateTypeObject(string name) Marshal.WriteIntPtr(type, TypeOffset.name, temp); #if PYTHON3 + Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); #endif @@ -1101,6 +1114,11 @@ private static IntPtr GetDefaultSlot(int offset) // tp_free of PyTypeType is point to PyObejct_GC_Del. return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } else if (offset == TypeOffset.tp_call) { return IntPtr.Zero; @@ -1110,6 +1128,16 @@ private static IntPtr GetDefaultSlot(int offset) // PyType_GenericNew return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); } From d1089138d409f66181e7f50fcc9b8e8b52369349 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 23 Sep 2019 00:02:30 +0800 Subject: [PATCH 011/134] Since ClassBase not override tp_init for BaseException, repr(ExceptionObject) will not care 'u' on Python2. --- src/runtime/runtime.cs | 8 ++------ src/tests/test_exceptions.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5e3c6e549..21ca50bc2 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -227,9 +227,6 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super")); - op = PyObject_GetAttrString(builtins, "KeyError"); - SetPyMember(ref PyExc_KeyError, op); - XDecref(builtins); } @@ -345,6 +342,7 @@ private static void InitializePlatformData() fn = PyObject_GetAttrString(platformModule, "system"); op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); + PythonException.ThrowIfIsNull(op); OperatingSystemName = GetManagedString(op); XDecref(op); XDecref(fn); @@ -470,7 +468,7 @@ private static void PyDictTryDelItem(IntPtr dict, string key) { return; } - if (!PythonException.Matches(PyExc_KeyError)) + if (!PythonException.Matches(Exceptions.KeyError)) { throw new PythonException(); } @@ -541,8 +539,6 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyNone; internal static IntPtr Error; - internal static IntPtr PyExc_KeyError; - /// /// Check if any Python Exceptions occurred. /// If any exist throw new PythonException. diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index a10d9a183..b03c974d1 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -286,7 +286,7 @@ def test_python_compat_of_managed_exceptions(): strexp = "OverflowException('Simple message" assert repr(e)[:len(strexp)] == strexp elif PY2: - assert repr(e) == "OverflowException(u'Simple message',)" + assert repr(e) == "OverflowException('Simple message',)" def test_exception_is_instance_of_system_object(): From cc9e7e5cd813755d0bce0938b438417cc0aa2c31 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 23 Sep 2019 00:32:14 +0800 Subject: [PATCH 012/134] * Use environment variable `PYTHONNET_SOFT_SHUTDOWN` to enable soft shutdown forcedly * Disable restore objects on default --- .travis.yml | 2 ++ appveyor.yml | 7 +++-- src/runtime/pythonengine.cs | 3 +- src/runtime/runtime.cs | 7 +++++ src/runtime/runtime_state.cs | 56 ++++++++++++++++++++++++------------ src/runtime/typemanager.cs | 1 - 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9689c0422..047a2810c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ env: matrix: - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" + - PYTHONNET_SOFT_SHUTDOWN="1" BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ + - PYTHONNET_SOFT_SHUTDOWN="1" BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so diff --git a/appveyor.yml b/appveyor.yml index 445f9bb5a..6d4d5eacb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ build: off image: - Visual Studio 2017 - + platform: - x86 - x64 @@ -15,7 +15,7 @@ environment: CODECOV_ENV: PYTHON_VERSION, PLATFORM matrix: - - PYTHON_VERSION: 2.7 + - PYTHON_VERSION: 2.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.5 BUILD_OPTS: --xplat @@ -28,6 +28,9 @@ environment: - PYTHON_VERSION: 3.6 - PYTHON_VERSION: 3.7 + - PYTHON_VERSION: 3.7 + PYTHONNET_SOFT_SHUTDOWN: 1 + matrix: allow_failures: - PYTHON_VERSION: 3.4 diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 0ad928caa..3c6475610 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,7 +12,7 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { - public static bool SoftShutdown { get; private set; } + public static bool SoftShutdown => Runtime.SoftShutdown; private static DelegateManager delegateManager; private static bool initialized; @@ -171,7 +171,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, delegateManager = new DelegateManager(); Runtime.Initialize(initSigs, softShutdown); initialized = true; - SoftShutdown = softShutdown; Exceptions.Clear(); // Make sure we clean up properly on app domain unload. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 21ca50bc2..e125629a1 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -166,6 +166,7 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + public static bool SoftShutdown { get; private set; } private static PyReferenceCollection _typeRefs; /// @@ -173,6 +174,12 @@ public class Runtime /// internal static void Initialize(bool initSigs = false, bool softShutdown = false) { + if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") + { + softShutdown = true; + } + + SoftShutdown = softShutdown; if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 44729bd62..372a6d25b 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -9,6 +9,9 @@ namespace Python.Runtime { class RuntimeState { + public static bool ShouldRestoreObjects { get; set; } = false; + public static bool UseDummyGC { get; set; } = false; + public static void Save() { if (PySys_GetObject("dummy_gc") != IntPtr.Zero) @@ -16,6 +19,16 @@ public static void Save() throw new Exception("Runtime State set already"); } + IntPtr objs = IntPtr.Zero; + if (ShouldRestoreObjects) + { + objs = PySet_New(IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + AddObjPtrToSet(objs, obj); + } + } + var modules = PySet_New(IntPtr.Zero); foreach (var name in GetModuleNames()) { @@ -23,11 +36,6 @@ public static void Save() PythonException.ThrowIfIsNotZero(res); } - var objs = PySet_New(IntPtr.Zero); - foreach (var obj in PyGCGetObjects()) - { - AddObjPtrToSet(objs, obj); - } var dummyGCHead = PyMem_Malloc(Marshal.SizeOf(typeof(PyGC_Head))); unsafe @@ -43,7 +51,6 @@ public static void Save() PythonException.ThrowIfIsNotZero(res); XDecref(pyDummyGC); - AddObjPtrToSet(objs, modules); try { res = PySys_SetObject("initial_modules", modules); @@ -54,17 +61,19 @@ public static void Save() XDecref(modules); } - AddObjPtrToSet(objs, objs); - try - { - res = PySys_SetObject("initial_objs", objs); - PythonException.ThrowIfIsNotZero(res); - } - finally + if (ShouldRestoreObjects) { - XDecref(objs); + AddObjPtrToSet(objs, modules); + try + { + res = PySys_SetObject("initial_objs", objs); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(objs); + } } - } } @@ -77,7 +86,10 @@ public static void Restore() } var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); ResotreModules(dummyGC); - RestoreObjects(dummyGC); + if (ShouldRestoreObjects) + { + RestoreObjects(dummyGC); + } } private static void ResotreModules(IntPtr dummyGC) @@ -92,16 +104,24 @@ private static void ResotreModules(IntPtr dummyGC) continue; } var module = PyDict_GetItem(modules, name); - if (_PyObject_GC_IS_TRACKED(module)) + + if (UseDummyGC && _PyObject_GC_IS_TRACKED(module)) { ExchangeGCChain(module, dummyGC); } - PyDict_DelItem(modules, name); + if (PyDict_DelItem(modules, name) != 0) + { + PyErr_Print(); + } } } private static void RestoreObjects(IntPtr dummyGC) { + if (!UseDummyGC) + { + throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); + } var intialObjs = PySys_GetObject("initial_objs"); Debug.Assert(intialObjs != null); foreach (var obj in PyGCGetObjects()) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 0660258c5..1fbefc973 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -818,7 +818,6 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo impl = impl.BaseType; } - bool canOverride = !PythonEngine.SoftShutdown; var native = NativeCode.Active; // The garbage collection related slots always have to return 1 or 0 From 1cb8e8cfcc1c168c11fe429747dd7af3e06aec8b Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 13 Oct 2019 17:37:43 +0800 Subject: [PATCH 013/134] Fixed deadlock on finalizer after shutdown --- src/runtime/runtime.cs | 15 +++++++++++++++ src/runtime/typemanager.cs | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e125629a1..fc4b5484d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -193,6 +193,11 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } + else + { + PyGILState_Ensure(); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; + } IsFinalizing = false; @@ -406,6 +411,16 @@ internal static void Shutdown(bool soft) PyGC_Collect(); RuntimeState.Restore(); ResetPyMembers(); + GC.Collect(); + try + { + GC.WaitForFullGCComplete(); + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + } + PyEval_SaveThread(); return; } ResetPyMembers(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 1fbefc973..72fe9cd67 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1050,8 +1050,10 @@ private void ResetSlots() return; } _alredyReset = true; +#if DEBUG IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif foreach (var offset in _slots.Keys) { IntPtr ptr = GetDefaultSlot(offset); From b3e889b9f6f320f7a40f67dfe6cbe315e03215bb Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 1 Dec 2019 21:35:05 +0800 Subject: [PATCH 014/134] * Load PyModuleType without LibraryLoader * Drop C module dependency when getting _PyObject_NextNotImplemented * Exception details for SetNoSiteFlag --- src/runtime/runtime.cs | 62 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 945c8f809..8b4ef6cbe 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -292,26 +292,18 @@ internal static void Initialize(bool initSigs = false) Error = new IntPtr(-1); + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + IntPtr sys = PyImport_ImportModule("sys"); + PyModuleType = PyObject_Type(sys); + XDecref(sys); + } + // Initialize data about the platform we're running on. We need // this for the type manager and potentially other details. Must // happen after caching the python types, above. InitializePlatformData(); - IntPtr dllLocal = IntPtr.Zero; - var loader = LibraryLoader.Get(OperatingSystem); - - // Since `_PyObject_NextNotImplemented` would set to a heap class - // for tp_iternext which doesn't implement __next__. - // Thus we need a heap class to get it, the ZipImportError is a - // heap class and it's in builtins, so we can use it as a trick. - var zipimport = PyImport_ImportModule("zipimport"); - var ZipImportError = PyObject_GetAttrString(zipimport, "ZipImportError"); - _PyObject_NextNotImplemented = Marshal.ReadIntPtr(ZipImportError, TypeOffset.tp_iternext); - XDecref(ZipImportError); - XDecref(zipimport); - PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); - - // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); PyCLRMetaType = MetaType.Initialize(); @@ -328,6 +320,29 @@ internal static void Initialize(bool initSigs = false) AssemblyManager.UpdatePath(); } + private static IntPtr Get_PyObject_NextNotImplemented() + { + IntPtr globals = PyDict_New(); + if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0) + { + XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + XDecref(globals); + throw new PythonException(); + } + XDecref(res); + IntPtr A = PyDict_GetItemString(globals, "A"); + IntPtr iternext = Marshal.ReadIntPtr(A, TypeOffset.tp_iternext); + XDecref(globals); + XDecref(A); + return iternext; + } + /// /// Initializes the data about platforms. /// @@ -1893,14 +1908,16 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static void SetNoSiteFlag() { + if (_PythonDll == "__Internal") + { + throw new NotSupportedException("SetNoSiteFlag didn't support on static compile"); + } var loader = LibraryLoader.Get(OperatingSystem); - - IntPtr dllLocal; - if (_PythonDll != "__Internal") + IntPtr dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) { - dllLocal = loader.Load(_PythonDll); + throw new Exception($"Cannot load {_PythonDll}"); } - try { Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); @@ -1908,10 +1925,7 @@ internal static void SetNoSiteFlag() } finally { - if (dllLocal != IntPtr.Zero) - { - loader.Free(dllLocal); - } + loader.Free(dllLocal); } } } From 5150e618612a299af3a5c970f5ec299b3ca974f0 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 15 Dec 2019 02:34:28 +0800 Subject: [PATCH 015/134] Prevent exception override --- src/runtime/runtime.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 153738930..358a3946b 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -332,8 +332,14 @@ private static IntPtr Get_PyObject_NextNotImplemented() IntPtr res = PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); if (res == IntPtr.Zero) { - XDecref(globals); - throw new PythonException(); + try + { + throw new PythonException(); + } + finally + { + XDecref(globals); + } } XDecref(res); IntPtr A = PyDict_GetItemString(globals, "A"); From 65e209e00d76a39007b86b4a3c825ab1e38a1d2c Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 16 Dec 2019 01:33:42 +0800 Subject: [PATCH 016/134] Rollback symbol loading for __Internal --- src/runtime/runtime.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 358a3946b..495dd7bb7 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1946,15 +1946,15 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static void SetNoSiteFlag() { - if (_PythonDll == "__Internal") - { - throw new NotSupportedException("SetNoSiteFlag didn't support on static compile"); - } var loader = LibraryLoader.Get(OperatingSystem); - IntPtr dllLocal = loader.Load(_PythonDll); - if (dllLocal == IntPtr.Zero) + IntPtr dllLocal; + if (_PythonDll != "__Internal") { - throw new Exception($"Cannot load {_PythonDll}"); + dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) + { + throw new Exception($"Cannot load {_PythonDll}"); + } } try { @@ -1963,7 +1963,10 @@ internal static void SetNoSiteFlag() } finally { - loader.Free(dllLocal); + if (dllLocal != IntPtr.Zero) + { + loader.Free(dllLocal); + } } } } From 0dee5dad765e6c2482c59d02b3111ca25b4f7723 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 20 Dec 2019 17:51:45 +0800 Subject: [PATCH 017/134] Free GC handle for all subclass of ExtensionType --- src/runtime/constructorbinding.cs | 4 ++-- src/runtime/eventbinding.cs | 2 +- src/runtime/eventobject.cs | 2 +- src/runtime/extensiontype.cs | 9 +++++++-- src/runtime/methodbinding.cs | 2 +- src/runtime/methodobject.cs | 2 +- src/runtime/moduleobject.cs | 2 +- src/runtime/overload.cs | 2 +- 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index a9f6c961b..2ec698f90 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -145,7 +145,7 @@ public static IntPtr tp_repr(IntPtr ob) var self = (ConstructorBinding)GetManagedObject(ob); Runtime.XDecref(self.repr); Runtime.XDecref(self.pyTypeHndl); - ExtensionType.FinalizeObject(self); + self.Dealloc(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) @@ -242,7 +242,7 @@ public static IntPtr tp_repr(IntPtr ob) var self = (BoundContructor)GetManagedObject(ob); Runtime.XDecref(self.repr); Runtime.XDecref(self.pyTypeHndl); - FinalizeObject(self); + self.Dealloc(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index b8b4c82ad..c844f5fd4 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -118,7 +118,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (EventBinding)GetManagedObject(ob); Runtime.XDecref(self.target); - ExtensionType.FinalizeObject(self); + self.Dealloc(); } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 5f18c4609..2a98896fe 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -202,7 +202,7 @@ public static IntPtr tp_repr(IntPtr ob) { Runtime.XDecref(self.unbound.pyHandle); } - FinalizeObject(self); + self.Dealloc(); } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index bc6872412..f7380b22c 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -54,6 +54,11 @@ public static void FinalizeObject(ManagedType self) self.gcHandle.Free(); } + protected void Dealloc() + { + FinalizeObject(this); + FreeGCHandle(); + } /// /// Type __setattr__ implementation. @@ -88,8 +93,8 @@ public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. - ManagedType self = GetManagedObject(ob); - FinalizeObject(self); + var self = (ExtensionType)GetManagedObject(ob); + self.Dealloc(); } } } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 5c4164067..1caec322c 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -244,7 +244,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); self.ClearMembers(); - FinalizeObject(self); + self.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 847183c73..4b0987c13 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -208,7 +208,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); self.ClearMembers(); - ExtensionType.FinalizeObject(self); + self.Dealloc(); } public static int tp_clear(IntPtr ob) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 4b4f09a41..420bce85a 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -305,7 +305,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (ModuleObject)GetManagedObject(ob); tp_clear(ob); - ExtensionType.tp_dealloc(ob); + self.Dealloc(); } public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index 67868a1b1..e9fa91d3b 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -65,7 +65,7 @@ public static IntPtr tp_repr(IntPtr op) { var self = (OverloadMapper)GetManagedObject(ob); Runtime.XDecref(self.target); - FinalizeObject(self); + self.Dealloc(); } } } From 00a0b320af3e30d75f001f873d0f5471d690e991 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 20 Dec 2019 21:20:34 +0800 Subject: [PATCH 018/134] Specific exception types --- src/embed_tests/TestDomainReload.cs | 2 +- src/runtime/runtime_state.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 6084c4062..2953a60e1 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -212,7 +212,7 @@ static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) { Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); } - catch (Exception) + catch (AppDomainUnloadedException) { Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); } diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 372a6d25b..bb91b125c 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -82,7 +82,7 @@ public static void Restore() var dummyGCAddr = PySys_GetObject("dummy_gc"); if (dummyGCAddr == IntPtr.Zero) { - throw new Exception("Runtime state have not set"); + throw new InvalidOperationException("Runtime state have not set"); } var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); ResotreModules(dummyGC); @@ -122,8 +122,8 @@ private static void RestoreObjects(IntPtr dummyGC) { throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); } - var intialObjs = PySys_GetObject("initial_objs"); - Debug.Assert(intialObjs != null); + IntPtr intialObjs = PySys_GetObject("initial_objs"); + Debug.Assert(intialObjs != IntPtr.Zero); foreach (var obj in PyGCGetObjects()) { var p = PyLong_FromVoidPtr(obj); @@ -192,7 +192,7 @@ private static void ExchangeGCChain(IntPtr obj, IntPtr gc) var head = _Py_AS_GC(obj); if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) { - throw new Exception("GC object untracked"); + throw new ArgumentException("GC object untracked"); } unsafe { From 77da6dff83b6315cf328c164367571c2e13acc61 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 23 Dec 2019 15:59:59 +0800 Subject: [PATCH 019/134] Record mp_length slot --- src/runtime/typemanager.cs | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 771aa05ee..e7eb8e0f1 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -184,16 +184,17 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + // add a __len__ slot for inheritors of ICollection and ICollection<> if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) { - InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + var method = typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length)); + InitializeSlot(type, TypeOffset.mp_length, method, slotsHolder); } - // we want to do this after the slot stuff above in case the class itself implements a slot method - SlotsHolder slotsHolder = new SlotsHolder(type); - InitializeSlots(type, impl.GetType(), slotsHolder); - if (base_ != IntPtr.Zero) { Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); @@ -239,12 +240,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) - { - var thunk = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, slotOffset, thunk.Address); - } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -912,7 +907,20 @@ static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolde return; } Marshal.WriteIntPtr(type, offset, thunk.Address); - slotsHolder.Add(offset, thunk); + if (slotsHolder != null) + { + slotsHolder.Add(offset, thunk); + } + } + + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder = null) + { + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Add(slotOffset, thunk); + } } static int GetSlotOffset(string name) From 2039e69dbd97c528ce3628303b23f3689806b571 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 23 Dec 2019 16:04:39 +0800 Subject: [PATCH 020/134] * Fix deadlock on domain unload * Reset Runtime.Exceptions --- src/embed_tests/TestDomainReload.cs | 7 ++++--- src/runtime/exceptions.cs | 22 +++++++++++----------- src/runtime/pythonengine.cs | 19 ++++++++++--------- src/runtime/runtime.cs | 2 ++ 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 2953a60e1..8fddf6b93 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -56,8 +56,8 @@ public static void DomainReloadAndGC() Assembly pythonRunner1 = BuildAssembly("test1"); RunAssemblyAndUnload(pythonRunner1, "test1"); - // Verify that python is not initialized even though we ran it. - //Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); + Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); // This caused a crash because objects allocated in pythonRunner1 // still existed in memory, but the code to do python GC on those @@ -83,7 +83,7 @@ public static void RunPython() { AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - //PythonEngine.Initialize(softShutdown: true); + PythonEngine.Initialize(softShutdown: true); using (Py.GIL()) { try { var pyScript = string.Format(""import clr\n"" @@ -99,6 +99,7 @@ public static void RunPython() { Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); } } + PythonEngine.BeginAllowThreads(); } static void OnDomainUnload(object sender, EventArgs e) { System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 31c367eb2..c1edaced2 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -132,21 +132,21 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) { - Type type = typeof(Exceptions); - foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) + return; + } + Type type = typeof(Exceptions); + foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) + { + var op = (IntPtr)fi.GetValue(type); + if (op != IntPtr.Zero) { - var op = (IntPtr)fi.GetValue(type); - if (op != IntPtr.Zero) - { - Runtime.XDecref(op); - } + Runtime.XDecref(op); } - Runtime.XDecref(exceptions_module); - Runtime.PyObject_HasAttrString(warnings_module, "xx"); - Runtime.XDecref(warnings_module); } + Runtime.Py_CLEAR(ref exceptions_module); + Runtime.Py_CLEAR(ref warnings_module); } /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index cbf829bdb..d8e733e4d 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -324,18 +324,19 @@ public static void InitExt() /// public static void Shutdown() { - if (initialized) + if (!initialized) { - PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + return; + } + // If the shutdown handlers trigger a domain unload, + // don't call shutdown again. + AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - ExecuteShutdownHandlers(); + PyScopeManager.Global.Clear(); + ExecuteShutdownHandlers(); - initialized = false; - } + initialized = false; + } /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a94aa00b9..5d5c4a59a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -394,6 +394,8 @@ internal static void Shutdown(bool soft) { return; } + PyGILState_Ensure(); + AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); From fe5050dfccb466d9824a63ef0e54123a5bf41d1e Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 26 Dec 2019 22:26:05 +0800 Subject: [PATCH 021/134] Fix refcnt error --- src/runtime/classbase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 73d461fa1..de02e5237 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -260,6 +260,7 @@ public static IntPtr tp_repr(IntPtr ob) //otherwise use the standard object.__repr__(inst) IntPtr args = Runtime.PyTuple_New(1); + Runtime.XIncref(ob); Runtime.PyTuple_SetItem(args, 0, ob); IntPtr reprFunc = Runtime.PyObject_GetAttrString(Runtime.PyBaseObjectType, "__repr__"); var output = Runtime.PyObject_Call(reprFunc, args, IntPtr.Zero); From 593fb00fc31066f1c10e104d5ecf5a25b648344a Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 26 Dec 2019 22:30:15 +0800 Subject: [PATCH 022/134] Reset a tuple for tp_bases --- src/runtime/typemanager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e7eb8e0f1..1290903d3 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1103,7 +1103,8 @@ private void ResetSlots() IntPtr tp_bases = Marshal.ReadIntPtr(_type, TypeOffset.tp_bases); Runtime.XDecref(tp_bases); - Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, IntPtr.Zero); + tp_bases = Runtime.PyTuple_New(0); + Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); } private static void OnDestruct(IntPtr ob) From fba616a84ac9e105e34a6938031da481f185619b Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 17 Jan 2020 22:12:45 +0800 Subject: [PATCH 023/134] Fix refcnt error --- src/runtime/extensiontype.cs | 3 +-- src/runtime/managedtype.cs | 5 +++++ src/runtime/moduleobject.cs | 6 +++++- src/runtime/typemanager.cs | 12 +++++------- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index f7380b22c..04302743d 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -51,13 +51,12 @@ public static void FinalizeObject(ManagedType self) { Runtime.PyObject_GC_Del(self.pyHandle); // Not necessary for decref of `tpHandle`. - self.gcHandle.Free(); + self.FreeGCHandle(); } protected void Dealloc() { FinalizeObject(this); - FreeGCHandle(); } /// diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 04e59a115..4b1296d31 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -19,6 +19,11 @@ internal abstract class ManagedType private static readonly HashSet _managedObjs = new HashSet(); + internal void IncrRefCount() + { + Runtime.XIncref(pyHandle); + } + internal void DecrRefCount() { Runtime.XDecref(pyHandle); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 420bce85a..bdd1551a6 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -174,7 +174,11 @@ public ManagedType GetAttribute(string name, bool guess) /// private void StoreAttribute(string name, ManagedType ob) { - Runtime.PyDict_SetItemString(dict, name, ob.pyHandle); + if (Runtime.PyDict_SetItemString(dict, name, ob.pyHandle) != 0) + { + throw new PythonException(); + } + ob.IncrRefCount(); cache[name] = ob; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 1290903d3..5c9004622 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -17,14 +17,9 @@ namespace Python.Runtime /// internal class TypeManager { - private static BindingFlags tbFlags; + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; private static Dictionary cache; - static TypeManager() - { - tbFlags = BindingFlags.Public | BindingFlags.Static; - } - public static void Reset() { cache = new Dictionary(128); @@ -74,6 +69,7 @@ internal static IntPtr GetTypeHandle(Type type) /// + /// Return value: Borrowed reference. /// Get the handle of a Python type that reflects the given CLR type. /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. @@ -311,7 +307,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr // derived class we want the python overrides in there instead if they exist. IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); Runtime.PyDict_Update(cls_dict, py_dict); - + Runtime.XIncref(py_type); return py_type; } catch (Exception e) @@ -1075,7 +1071,9 @@ private void ResetSlots() foreach (var offset in _slots.Keys) { IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif Marshal.WriteIntPtr(_type, offset, ptr); } From bdc0f72daf16bf96e5074bc9951faeee2df42fb6 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 18 Jan 2020 00:05:44 +0800 Subject: [PATCH 024/134] * Use subtype slots instead JIT code * Correct signature of PyRun_String --- src/runtime/classbase.cs | 22 +-------- src/runtime/classmanager.cs | 8 ++-- src/runtime/extensiontype.cs | 1 + src/runtime/managedtype.cs | 30 +++++++++++- src/runtime/methodobject.cs | 2 + src/runtime/pyscope.cs | 6 +-- src/runtime/pythonengine.cs | 2 +- src/runtime/runtime.cs | 13 ++---- src/runtime/typemanager.cs | 89 ++++++++++++++++++++++++++++-------- 9 files changed, 112 insertions(+), 61 deletions(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index de02e5237..10a451fe3 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -299,29 +299,9 @@ public static int tp_clear(IntPtr ob) { ClearObjectDict(ob); } + self.tpHandle = IntPtr.Zero; Runtime.Py_CLEAR(ref self.tpHandle); return 0; } - - private static IntPtr GetObjectDict(IntPtr ob) - { - return Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); - } - - private static void SetObjectDict(IntPtr ob, IntPtr value) - { - Marshal.WriteIntPtr(ob, ObjectOffset.DictOffset(ob), value); - } - - private static void ClearObjectDict(IntPtr ob) - { - IntPtr dict = GetObjectDict(ob); - if (dict == IntPtr.Zero) - { - return; - } - SetObjectDict(ob, IntPtr.Zero); - Runtime.XDecref(dict); - } } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 43892aabc..aa6fd6e51 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -53,9 +53,9 @@ internal static void RemoveClasses() { foreach (var cls in cache.Values) { - cls.TypeTraverse(OnVisit, visitedPtr); + cls.CallTypeTraverse(OnVisit, visitedPtr); // XXX: Force release instance resources but not dealloc itself. - cls.TypeClear(); + cls.CallTypeClear(); } } finally @@ -75,8 +75,8 @@ private static int OnVisit(IntPtr ob, IntPtr arg) var clrObj = ManagedType.GetManagedObject(ob); if (clrObj != null) { - clrObj.TypeTraverse(OnVisit, arg); - clrObj.TypeClear(); + clrObj.CallTypeTraverse(OnVisit, arg); + clrObj.CallTypeClear(); } return 0; } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 04302743d..c1aff3ca0 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -49,6 +49,7 @@ public ExtensionType() /// public static void FinalizeObject(ManagedType self) { + ClearObjectDict(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); // Not necessary for decref of `tpHandle`. self.FreeGCHandle(); diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 4b1296d31..a93628524 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -149,7 +149,7 @@ internal static int PyVisit(IntPtr ob, IntPtr visit, IntPtr arg) /// /// Wrapper for calling tp_clear /// - internal void TypeClear() + internal void CallTypeClear() { if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) { @@ -168,7 +168,7 @@ internal void TypeClear() /// /// Wrapper for calling tp_traverse /// - internal void TypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) + internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) { if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) { @@ -184,5 +184,31 @@ internal void TypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) // TODO: Handle errors base on return value traverseFunc(pyHandle, visiPtr, arg); } + + protected void TypeClear() + { + ClearObjectDict(pyHandle); + } + + protected static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + SetObjectDict(ob, IntPtr.Zero); + Runtime.XDecref(dict); + } + + private static IntPtr GetObjectDict(IntPtr ob) + { + return Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + } + + private static void SetObjectDict(IntPtr ob, IntPtr value) + { + Marshal.WriteIntPtr(ob, ObjectOffset.DictOffset(ob), value); + } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 4b0987c13..3942aae52 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -208,6 +208,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); self.ClearMembers(); + ClearObjectDict(ob); self.Dealloc(); } @@ -215,6 +216,7 @@ public static int tp_clear(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); self.ClearMembers(); + ClearObjectDict(ob); return 0; } } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 4008ce29a..2a7652e3c 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -277,9 +277,8 @@ public PyObject Eval(string code, PyDict locals = null) { Check(); IntPtr _locals = locals == null ? variables : locals.obj; - var flag = (IntPtr)Runtime.Py_eval_input; IntPtr ptr = Runtime.PyRun_String( - code, flag, variables, _locals + code, RunFlagType.Eval, variables, _locals ); Runtime.CheckExceptionOccurred(); return new PyObject(ptr); @@ -315,9 +314,8 @@ public void Exec(string code, PyDict locals = null) private void Exec(string code, IntPtr _globals, IntPtr _locals) { - var flag = (IntPtr)Runtime.Py_file_input; IntPtr ptr = Runtime.PyRun_String( - code, flag, _globals, _locals + code, RunFlagType.File, _globals, _locals ); Runtime.CheckExceptionOccurred(); if (ptr != Runtime.PyNone) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index d8e733e4d..e036bab33 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -600,7 +600,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { IntPtr result = Runtime.PyRun_String( - code, (IntPtr)flag, globals.Value, locals.Value + code, flag, globals.Value, locals.Value ); Runtime.CheckExceptionOccurred(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 0b89838d8..18dc0c113 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -209,7 +209,7 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false PyScopeManager.Reset(); ClassManager.Reset(); ClassDerivedObject.Reset(); - TypeManager.Reset(); + TypeManager.Initialize(); IntPtr op; { @@ -527,7 +527,7 @@ private static void MoveClrInstancesOnwershipToPython() { continue; } - obj.TypeClear(); + obj.CallTypeClear(); // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), // thus just be safe to give it back to GC chain. PyObject_GC_Track(obj.pyHandle); @@ -537,11 +537,6 @@ private static void MoveClrInstancesOnwershipToPython() ManagedType.ClearTrackedObjects(); } - - internal static IntPtr Py_single_input = (IntPtr)256; - internal static IntPtr Py_file_input = (IntPtr)257; - internal static IntPtr Py_eval_input = (IntPtr)258; - internal static IntPtr PyBaseObjectType; internal static IntPtr PyModuleType; internal static IntPtr PyClassType; @@ -903,7 +898,7 @@ public static extern int Py_Main( internal static extern int PyRun_SimpleString(string code); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyRun_String(string code, IntPtr st, IntPtr globals, IntPtr locals); + internal static extern IntPtr PyRun_String(string code, RunFlagType st, IntPtr globals, IntPtr locals); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); @@ -2013,7 +2008,7 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type¡¯s base class. Return 0 on success, or return -1 and sets an exception on error. + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 5c9004622..1e54b3345 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -17,12 +17,20 @@ namespace Python.Runtime /// internal class TypeManager { + internal static IntPtr subtype_traverse; + internal static IntPtr subtype_clear; + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; private static Dictionary cache; - public static void Reset() + public static void Initialize() { cache = new Dictionary(128); + + IntPtr type = SlotHelper.CreateObjectType(); + subtype_traverse = Marshal.ReadIntPtr(type, TypeOffset.tp_traverse); + subtype_clear = Marshal.ReadIntPtr(type, TypeOffset.tp_clear); + Runtime.XDecref(type); } public static IList GetManagedTypes() @@ -440,7 +448,7 @@ internal static IntPtr CreateMetaType(Type impl) mdef = WriteMethodDefSentinel(mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - slotsHolder.Add(TypeOffset.tp_methods, (t, offset) => + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => { var p = Marshal.ReadIntPtr(t, offset); Runtime.PyMem_Free(p); @@ -851,7 +859,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo Marshal.WriteIntPtr(type, offset, thunkRet0.Address); if (slotsHolder != null) { - slotsHolder.Add(offset, thunkRet0); + slotsHolder.Set(offset, thunkRet0); } } if (!IsSlotSet(type, "tp_clear")) @@ -861,7 +869,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo Marshal.WriteIntPtr(type, offset, thunkRet0.Address); if (slotsHolder != null) { - slotsHolder.Add(offset, thunkRet0); + slotsHolder.Set(offset, thunkRet0); } } } @@ -905,7 +913,7 @@ static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolde Marshal.WriteIntPtr(type, offset, thunk.Address); if (slotsHolder != null) { - slotsHolder.Add(offset, thunk); + slotsHolder.Set(offset, thunk); } } @@ -915,7 +923,7 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots Marshal.WriteIntPtr(type, slotOffset, thunk.Address); if (slotsHolder != null) { - slotsHolder.Add(slotOffset, thunk); + slotsHolder.Set(slotOffset, thunk); } } @@ -992,9 +1000,9 @@ class SlotsHolder private IntPtr _type; private Dictionary _slots = new Dictionary(); private List _keepalive = new List(); - private Dictionary _customRestors = new Dictionary(); + private Dictionary _customResetors = new Dictionary(); private List _deallocators = new List(); - private bool _alredyReset = false; + private bool _alreadyReset = false; /// /// Create slots holder for holding the delegate of slots and be able to reset them. @@ -1005,14 +1013,14 @@ public SlotsHolder(IntPtr type) _type = type; } - public void Add(int offset, ThunkInfo thunk) + public void Set(int offset, ThunkInfo thunk) { - _slots.Add(offset, thunk); + _slots[offset] = thunk; } - public void Add(int offset, Resetor resetor) + public void Set(int offset, Resetor resetor) { - _customRestors[offset] = resetor; + _customResetors[offset] = resetor; } public void AddDealloctor(Action deallocate) @@ -1059,11 +1067,11 @@ public static void ReleaseTypeSlots(IntPtr type) private void ResetSlots() { - if (_alredyReset) + if (_alreadyReset) { return; } - _alredyReset = true; + _alreadyReset = true; #if DEBUG IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); string typeName = Marshal.PtrToStringAnsi(tp_name); @@ -1082,14 +1090,14 @@ private void ResetSlots() action(); } - foreach (var pair in _customRestors) + foreach (var pair in _customResetors) { int offset = pair.Key; var resetor = pair.Value; resetor?.Invoke(_type, offset); } - _customRestors.Clear(); + _customResetors.Clear(); _slots.Clear(); _keepalive.Clear(); _deallocators.Clear(); @@ -1097,12 +1105,16 @@ private void ResetSlots() // Custom reset IntPtr tp_base = Marshal.ReadIntPtr(_type, TypeOffset.tp_base); Runtime.XDecref(tp_base); - Marshal.WriteIntPtr(_type, TypeOffset.tp_base, IntPtr.Zero); + Runtime.XIncref(Runtime.PyBaseObjectType); + Marshal.WriteIntPtr(_type, TypeOffset.tp_base, Runtime.PyBaseObjectType); IntPtr tp_bases = Marshal.ReadIntPtr(_type, TypeOffset.tp_bases); Runtime.XDecref(tp_bases); tp_bases = Runtime.PyTuple_New(0); Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); + + // FIXME: release dict; + Marshal.WriteIntPtr(_type, TypeOffset.tp_dictoffset, IntPtr.Zero); } private static void OnDestruct(IntPtr ob) @@ -1122,10 +1134,13 @@ private static SlotsHolder RecoverFromCapsule(IntPtr ob) private static IntPtr GetDefaultSlot(int offset) { - if (offset == TypeOffset.tp_clear - || offset == TypeOffset.tp_traverse) + if (offset == TypeOffset.tp_clear) + { + return TypeManager.subtype_clear; + } + else if (offset == TypeOffset.tp_traverse) { - return TypeManager.NativeCodePage + TypeManager.NativeCode.Active.Return0; + return TypeManager.subtype_traverse; } else if (offset == TypeOffset.tp_dealloc) { @@ -1160,4 +1175,38 @@ private static IntPtr GetDefaultSlot(int offset) return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); } } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + // TODO: extract the prepare actions as a common method + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + Runtime.XDecref(res); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } + } } From 49d98e85f4ae249ba6813e580631cbade3a6b4c6 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 18 Jan 2020 13:35:39 +0800 Subject: [PATCH 025/134] Clear tp_dict of ModuleObject --- src/runtime/classbase.cs | 1 - src/runtime/constructorbinding.cs | 8 ++------ src/runtime/interop.cs | 11 ----------- src/runtime/moduleobject.cs | 5 +++-- src/runtime/typemanager.cs | 3 --- 5 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 10a451fe3..e98bed638 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -300,7 +300,6 @@ public static int tp_clear(IntPtr ob) ClearObjectDict(ob); } self.tpHandle = IntPtr.Zero; - Runtime.Py_CLEAR(ref self.tpHandle); return 0; } } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 2ec698f90..82bcb49f1 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -29,8 +29,7 @@ internal class ConstructorBinding : ExtensionType public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; repr = IntPtr.Zero; } @@ -144,7 +143,6 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (ConstructorBinding)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); self.Dealloc(); } @@ -179,8 +177,7 @@ internal class BoundContructor : ExtensionType public BoundContructor(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; ctorInfo = ci; repr = IntPtr.Zero; @@ -241,7 +238,6 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (BoundContructor)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); self.Dealloc(); } diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 9aacbb07e..25b3882da 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -515,17 +515,6 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal struct Thunk - { - public Delegate fn; - - public Thunk(Delegate d) - { - fn = d; - } - } - internal class ThunkInfo { public readonly Delegate Target; diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index bdd1551a6..06f1c0581 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -53,6 +53,7 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); + Runtime.XIncref(dict); Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); InitializeModuleMembers(); @@ -328,8 +329,8 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) public static int tp_clear(IntPtr ob) { var self = (ModuleObject)GetManagedObject(ob); - Runtime.XDecref(self.dict); - self.dict = IntPtr.Zero; + Runtime.Py_CLEAR(ref self.dict); + ClearObjectDict(ob); foreach (var attr in self.cache.Values) { Runtime.XDecref(attr.pyHandle); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 1e54b3345..e5c01ba3f 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1112,9 +1112,6 @@ private void ResetSlots() Runtime.XDecref(tp_bases); tp_bases = Runtime.PyTuple_New(0); Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); - - // FIXME: release dict; - Marshal.WriteIntPtr(_type, TypeOffset.tp_dictoffset, IntPtr.Zero); } private static void OnDestruct(IntPtr ob) From 433d0f65872492b8ff71c0b69a84f5a95e1c5c43 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 18 Jan 2020 14:11:02 +0800 Subject: [PATCH 026/134] Manipulate SlotsHolder manually instead of Capsule mechanism --- src/runtime/metatype.cs | 10 ++-- src/runtime/typemanager.cs | 108 +++++++++---------------------------- 2 files changed, 30 insertions(+), 88 deletions(-) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 51f3eddd7..68954ef0a 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -11,14 +11,14 @@ namespace Python.Runtime internal class MetaType : ManagedType { private static IntPtr PyCLRMetaType; - + private static SlotsHolder _metaSlotsHodler; /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// public static IntPtr Initialize() { - PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType)); + PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); return PyCLRMetaType; } @@ -26,10 +26,10 @@ public static void Release() { if (Runtime.Refcount(PyCLRMetaType) > 1) { - SlotsHolder.ReleaseTypeSlots(PyCLRMetaType); + _metaSlotsHodler.ResetSlots(); } - Runtime.XDecref(PyCLRMetaType); - PyCLRMetaType = IntPtr.Zero; + Runtime.Py_CLEAR(ref PyCLRMetaType); + _metaSlotsHodler = null; } /// diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e5c01ba3f..bd549550b 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -21,12 +21,11 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static Dictionary cache; + private static readonly Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); public static void Initialize() { - cache = new Dictionary(128); - IntPtr type = SlotHelper.CreateObjectType(); subtype_traverse = Marshal.ReadIntPtr(type, TypeOffset.tp_traverse); subtype_clear = Marshal.ReadIntPtr(type, TypeOffset.tp_clear); @@ -42,15 +41,20 @@ internal static void RemoveTypes() { foreach (var tpHandle in cache.Values) { - // If refcount > 1, it needs to reset the managed slot, - // otherwise it can dealloc without any trick. - if (Runtime.Refcount(tpHandle) > 1) + SlotsHolder holder; + if (_slotsHolders.TryGetValue(tpHandle, out holder)) { - SlotsHolder.ReleaseTypeSlots(tpHandle); + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + holder.ResetSlots(); + } } Runtime.XDecref(tpHandle); } cache.Clear(); + _slotsHolders.Clear(); } /// @@ -115,7 +119,7 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - SlotsHolder slotsHolder = new SlotsHolder(type); + SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | @@ -132,10 +136,6 @@ internal static IntPtr CreateType(Type impl) Runtime.PyDict_SetItemString(dict, "__module__", mod); Runtime.XDecref(mod); - IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(dict, SlotsHolder.HolderKeyName, capsule); - Runtime.XDecref(capsule); - InitMethods(type, impl); return type; } @@ -189,7 +189,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); // we want to do this after the slot stuff above in case the class itself implements a slot method - SlotsHolder slotsHolder = new SlotsHolder(type); + SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl.GetType(), slotsHolder); // add a __len__ slot for inheritors of ICollection and ICollection<> @@ -227,10 +227,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.PyDict_SetItemString(dict, "__module__", mod); Runtime.XDecref(mod); - IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(dict, SlotsHolder.HolderKeyName, capsule); - Runtime.XDecref(capsule); - // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); @@ -365,7 +361,7 @@ internal static void FreeMethodDef(IntPtr mdef) } } - internal static IntPtr CreateMetaType(Type impl) + internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the // standard Python metatype (PyType_Type). It overrides certain of @@ -383,11 +379,9 @@ internal static IntPtr CreateMetaType(Type impl) // tp_basicsize, tp_itemsize, // tp_dictoffset, tp_weaklistoffset, // tp_traverse, tp_clear, tp_is_gc, etc. - //CopySlot(py_type, type, TypeOffset.tp_basicsize); - //CopySlot(py_type, type, TypeOffset.tp_itemsize); - SlotsHolder slotsHolder = new SlotsHolder(type); // Override type slots with those of the managed implementation. + slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default; @@ -464,10 +458,6 @@ internal static IntPtr CreateMetaType(Type impl) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); - IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(dict, SlotsHolder.HolderKeyName, capsule); - Runtime.XDecref(capsule); - //DebugUtil.DumpType(type); return type; @@ -502,7 +492,7 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - SlotsHolder slotsHolder = new SlotsHolder(type); + SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); if (Runtime.PyType_Ready(type) != 0) @@ -514,9 +504,6 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(tp_dict, "__module__", mod); - IntPtr capsule = slotsHolder.ToCapsule(); - Runtime.PyDict_SetItemString(tp_dict, SlotsHolder.HolderKeyName, capsule); - Runtime.XDecref(capsule); return type; } @@ -986,18 +973,21 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) IntPtr fp = Marshal.ReadIntPtr(from, offset); Marshal.WriteIntPtr(to, offset, fp); } + + private static SlotsHolder CreateSolotsHolder(IntPtr type) + { + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } } class SlotsHolder { - public const string HolderKeyName = "_slots_holder"; public delegate void Resetor(IntPtr type, int offset); - private GCHandle _handle; - private Interop.DestructorFunc _destructor; - private IntPtr _capsule; - private IntPtr _type; + private readonly IntPtr _type; private Dictionary _slots = new Dictionary(); private List _keepalive = new List(); private Dictionary _customResetors = new Dictionary(); @@ -1033,39 +1023,7 @@ public void KeeapAlive(Delegate d) _keepalive.Add(d); } - public IntPtr ToCapsule() - { - if (_capsule != IntPtr.Zero) - { - Runtime.XIncref(_capsule); - return _capsule; - } - _handle = GCHandle.Alloc(this); - _destructor = OnDestruct; - var fp = Marshal.GetFunctionPointerForDelegate(_destructor); - _capsule = Runtime.PyCapsule_New((IntPtr)_handle, null, fp); - return _capsule; - } - - public static void ReleaseTypeSlots(IntPtr type) - { - IntPtr capsule = Runtime.PyObject_GetAttrString(type, HolderKeyName); - if (capsule == IntPtr.Zero) - { - return; - } - var self = RecoverFromCapsule(capsule); - self.ResetSlots(); - Runtime.XDecref(capsule); - - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - if (Runtime.PyDict_DelItemString(tp_dict, HolderKeyName) != 0) - { - throw new PythonException(); - } - } - - private void ResetSlots() + public void ResetSlots() { if (_alreadyReset) { @@ -1114,21 +1072,6 @@ private void ResetSlots() Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); } - private static void OnDestruct(IntPtr ob) - { - var self = RecoverFromCapsule(ob); - self._handle.Free(); - self.ResetSlots(); - } - - private static SlotsHolder RecoverFromCapsule(IntPtr ob) - { - var ptr = Runtime.PyCapsule_GetPointer(ob, null); - PythonException.ThrowIfIsNull(ptr); - GCHandle handle = GCHandle.FromIntPtr(ptr); - return (SlotsHolder)handle.Target; - } - private static IntPtr GetDefaultSlot(int offset) { if (offset == TypeOffset.tp_clear) @@ -1178,7 +1121,6 @@ static class SlotHelper { public static IntPtr CreateObjectType() { - // TODO: extract the prepare actions as a common method IntPtr globals = Runtime.PyDict_New(); if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) { From 1b466df0ad10985c562e6f29a81094a863918f6d Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 23 Jan 2020 15:54:20 +0800 Subject: [PATCH 027/134] * Drop NativeCodePage dependency * Use TypeOffset.members(memory of the first PythonMemberDef) as magic slot * Improve accuracy of slot implementation detection * Fix refcnt errors --- src/embed_tests/TestDomainReload.cs | 2 + src/embed_tests/TestRuntime.cs | 13 +- src/embed_tests/TestTypeManager.cs | 12 +- src/embed_tests/pyimport.cs | 1 + src/embed_tests/pyinitialize.cs | 2 + src/runtime/Python.Runtime.csproj | 1 + src/runtime/classderived.cs | 4 + src/runtime/interop.cs | 32 +++ src/runtime/interop27.cs | 114 ++++++-- src/runtime/interop34.cs | 105 +++++-- src/runtime/interop35.cs | 116 ++++++-- src/runtime/interop36.cs | 116 ++++++-- src/runtime/interop37.cs | 116 ++++++-- src/runtime/interop38.cs | 113 ++++++-- src/runtime/platform/NativeCodePage.cs | 322 +++++++++++++++++++++ src/runtime/runtime.cs | 132 +++------ src/runtime/slots/mp_length.cs | 33 ++- src/runtime/typemanager.cs | 370 +++++-------------------- tools/geninterop/geninterop.py | 133 ++++++--- 19 files changed, 1177 insertions(+), 560 deletions(-) create mode 100644 src/runtime/platform/NativeCodePage.cs diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 8fddf6b93..5bfde11c1 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -64,6 +64,8 @@ public static void DomainReloadAndGC() // objects is gone. Assembly pythonRunner2 = BuildAssembly("test2"); RunAssemblyAndUnload(pythonRunner2, "test2"); + + PythonEngine.Shutdown(); } // diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 25b70fac5..fd02b4a82 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -27,15 +27,14 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); + Assert.That(NativeCodePageHelper.Machine, Is.Not.EqualTo(MachineType.Other)); + Assert.That(!string.IsNullOrEmpty(NativeCodePageHelper.MachineName)); - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); + Assert.That(NativeCodePageHelper.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); + Assert.That(!string.IsNullOrEmpty(NativeCodePageHelper.OperatingSystemName)); - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } + Runtime.Runtime.Shutdown(); + } [Test] public static void Py_IsInitializedValue() diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs index 931c44236..43155e1bf 100644 --- a/src/embed_tests/TestTypeManager.cs +++ b/src/embed_tests/TestTypeManager.cs @@ -1,5 +1,6 @@ using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; using System.Runtime.InteropServices; namespace Python.EmbeddingTest @@ -15,22 +16,21 @@ public static void Init() [TearDown] public static void Fini() { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. + Runtime.Runtime.Shutdown(); } [Test] public static void TestNativeCode() { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); + Assert.That(() => { var _ = NativeCodePageHelper.NativeCode.Active; }, Throws.Nothing); + Assert.That(NativeCodePageHelper.NativeCode.Active.Code.Length, Is.GreaterThan(0)); } [Test] public static void TestMemoryMapping() { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); + Assert.That(() => { var _ = NativeCodePageHelper.CreateMemoryMapper(); }, Throws.Nothing); + var mapper = NativeCodePageHelper.CreateMemoryMapper(); // Allocate a read-write page. int len = 12; diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index acb3433de..d76c95797 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -40,6 +40,7 @@ public void SetUp() IntPtr str = Runtime.Runtime.PyString_FromString(testPath); IntPtr path = Runtime.Runtime.PySys_GetObject("path"); Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.XDecref(str); } [TearDown] diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..69ed127bd 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -26,6 +26,7 @@ public static void LoadDefaultArgs() using (new PythonEngine()) using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { + Runtime.Runtime.XIncref(argv.Handle); Assert.AreNotEqual(0, argv.Length()); } } @@ -37,6 +38,7 @@ public static void LoadSpecificArgs() using (new PythonEngine(args)) using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { + Runtime.Runtime.XIncref(argv.Handle); Assert.AreEqual(args[0], argv[0].ToString()); Assert.AreEqual(args[1], argv[1].ToString()); } diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 508e8a668..a00f37440 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -143,6 +143,7 @@ + diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..f9c019cfe 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -99,6 +99,10 @@ internal static IntPtr ToPython(IPythonDerivedType obj) // collected while Python still has a reference to it. if (Runtime.Refcount(self.pyHandle) == 1) { + +#if PYTHON_WITH_PYDEBUG + Runtime._Py_NewReference(self.pyHandle); +#endif GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); self.gcHandle.Free(); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 25b3882da..d4b4b5119 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -69,6 +69,37 @@ public ModulePropertyAttribute() } } + internal static partial class TypeOffset + { + static class Helper + { + public static int magic; + public static readonly Dictionary NameMapping = new Dictionary(); + } + + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fields = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fields.Length; i++) + { + int offset = i * size; + FieldInfo fi = fields[i]; + fi.SetValue(null, offset); + Helper.NameMapping[fi.Name] = offset; + } + // XXX: Use the members after PyHeapTypeObject as magic slot + Helper.magic = members; + } + + public static int magic() => Helper.magic; + + public static int GetSlotOffset(string name) + { + return Helper.NameMapping[name]; + } + } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] internal class ObjectOffset @@ -556,4 +587,5 @@ struct PyMethodDef public int ml_flags; public IntPtr ml_doc; } + } diff --git a/src/runtime/interop27.cs b/src/runtime/interop27.cs index 4782e9d3b..7186502fd 100644 --- a/src/runtime/interop27.cs +++ b/src/runtime/interop27.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON27 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -145,6 +131,94 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_divide; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_nonzero; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_coerce; + public IntPtr nb_int; + public IntPtr nb_long; + public IntPtr nb_float; + public IntPtr nb_oct; + public IntPtr nb_hex; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_divide; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr sq_slice; + public IntPtr sq_ass_item; + public IntPtr sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getreadbuffer; + public IntPtr bf_getwritebuffer; + public IntPtr bf_getsegcount; + public IntPtr bf_getcharbuffer; + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop34.cs b/src/runtime/interop34.cs index 6857ff2d0..ae5f55135 100644 --- a/src/runtime/interop34.cs +++ b/src/runtime/interop34.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON34 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -139,6 +125,85 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop35.cs b/src/runtime/interop35.cs index a30bfa4fd..d13da73d4 100644 --- a/src/runtime/interop35.cs +++ b/src/runtime/interop35.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON35 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop36.cs b/src/runtime/interop36.cs index c46bcc2f5..d68539d56 100644 --- a/src/runtime/interop36.cs +++ b/src/runtime/interop36.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON36 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs index d5fc76ad3..c85d06525 100644 --- a/src/runtime/interop37.cs +++ b/src/runtime/interop37.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON37 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs index 9126bca6a..a87573e90 100644 --- a/src/runtime/interop38.cs +++ b/src/runtime/interop38.cs @@ -13,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -147,6 +132,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs new file mode 100644 index 000000000..3f89e68ab --- /dev/null +++ b/src/runtime/platform/NativeCodePage.cs @@ -0,0 +1,322 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + class NativeCodePageHelper + { + /// + /// Gets the operating system as reported by python's platform.system(). + /// + public static OperatingSystemType OperatingSystem { get; private set; } + + /// + /// Gets the operating system as reported by python's platform.system(). + /// + public static string OperatingSystemName { get; private set; } + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + public static string MachineName { get; private set; } + + /// + /// Initialized by InitializeNativeCodePage. + /// + /// This points to a page of memory allocated using mmap or VirtualAlloc + /// (depending on the system), and marked read and execute (not write). + /// Very much on purpose, the page is *not* released on a shutdown and + /// is instead leaked. See the TestDomainReload test case. + /// + /// The contents of the page are two native functions: one that returns 0, + /// one that returns 1. + /// + /// If python didn't keep its gc list through a Py_Finalize we could remove + /// this entire section. + /// + internal static IntPtr NativeCodePage = IntPtr.Zero; + + + static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() + { + { "Windows", OperatingSystemType.Windows }, + { "Darwin", OperatingSystemType.Darwin }, + { "Linux", OperatingSystemType.Linux }, + }; + + /// + /// Map lower-case version of the python machine name to the processor + /// type. There are aliases, e.g. x86_64 and amd64 are two names for + /// the same thing. Make sure to lower-case the search string, because + /// capitalization can differ. + /// + static readonly Dictionary MachineTypeMapping = new Dictionary() + { + ["i386"] = MachineType.i386, + ["i686"] = MachineType.i386, + ["x86"] = MachineType.i386, + ["x86_64"] = MachineType.x86_64, + ["amd64"] = MachineType.x86_64, + ["x64"] = MachineType.x86_64, + ["em64t"] = MachineType.x86_64, + ["armv7l"] = MachineType.armv7l, + ["armv8"] = MachineType.armv8, + ["aarch64"] = MachineType.aarch64, + }; + + /// + /// Structure to describe native code. + /// + /// Use NativeCode.Active to get the native code for the current platform. + /// + /// Generate the code by creating the following C code: + /// + /// int Return0() { return 0; } + /// int Return1() { return 1; } + /// + /// Then compiling on the target platform, e.g. with gcc or clang: + /// cc -c -fomit-frame-pointer -O2 foo.c + /// And then analyzing the resulting functions with a hex editor, e.g.: + /// objdump -disassemble foo.o + /// + internal class NativeCode + { + /// + /// The code, as a string of bytes. + /// + public byte[] Code { get; private set; } + + /// + /// Where does the "return 0" function start? + /// + public int Return0 { get; private set; } + + /// + /// Where does the "return 1" function start? + /// + public int Return1 { get; private set; } + + public static NativeCode Active + { + get + { + switch (Machine) + { + case MachineType.i386: + return I386; + case MachineType.x86_64: + return X86_64; + default: + return null; + } + } + } + + /// + /// Code for x86_64. See the class comment for how it was generated. + /// + public static readonly NativeCode X86_64 = new NativeCode() + { + Return0 = 0x10, + Return1 = 0, + Code = new byte[] + { + // First Return1: + 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax + 0xc3, // ret + + // Now some padding so that Return0 can be 16-byte-aligned. + // I put Return1 first so there's not as much padding to type in. + 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop + + // Now Return0. + 0x31, 0xc0, // xorl %eax, %eax + 0xc3, // ret + } + }; + + /// + /// Code for X86. + /// + /// It's bitwise identical to X86_64, so we just point to it. + /// + /// + public static readonly NativeCode I386 = X86_64; + } + + /// + /// Platform-dependent mmap and mprotect. + /// + internal interface IMemoryMapper + { + /// + /// Map at least numBytes of memory. Mark the page read-write (but not exec). + /// + IntPtr MapWriteable(int numBytes); + + /// + /// Sets the mapped memory to be read-exec (but not write). + /// + void SetReadExec(IntPtr mappedMemory, int numBytes); + } + + class WindowsMemoryMapper : IMemoryMapper + { + const UInt32 MEM_COMMIT = 0x1000; + const UInt32 MEM_RESERVE = 0x2000; + const UInt32 PAGE_READWRITE = 0x04; + const UInt32 PAGE_EXECUTE_READ = 0x20; + + [DllImport("kernel32.dll")] + static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); + + [DllImport("kernel32.dll")] + static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); + + public IntPtr MapWriteable(int numBytes) + { + return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + UInt32 _; + VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); + } + } + + class UnixMemoryMapper : IMemoryMapper + { + const int PROT_READ = 0x1; + const int PROT_WRITE = 0x2; + const int PROT_EXEC = 0x4; + + const int MAP_PRIVATE = 0x2; + int MAP_ANONYMOUS + { + get + { + switch (OperatingSystem) + { + case OperatingSystemType.Darwin: + return 0x1000; + case OperatingSystemType.Linux: + return 0x20; + default: + throw new NotImplementedException($"mmap is not supported on {OperatingSystemName}"); + } + } + } + + [DllImport("libc")] + static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); + + [DllImport("libc")] + static extern int mprotect(IntPtr addr, IntPtr len, int prot); + + public IntPtr MapWriteable(int numBytes) + { + // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. + // It doesn't hurt on darwin, so just do it. + return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); + } + } + + /// + /// Initializes the data about platforms. + /// + /// This must be the last step when initializing the runtime: + /// GetManagedString needs to have the cached values for types. + /// But it must run before initializing anything outside the runtime + /// because those rely on the platform data. + /// + public static void InitializePlatformData() + { + IntPtr op; + IntPtr fn; + IntPtr platformModule = Runtime.PyImport_ImportModule("platform"); + IntPtr emptyTuple = Runtime.PyTuple_New(0); + + fn = Runtime.PyObject_GetAttrString(platformModule, "system"); + op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); + PythonException.ThrowIfIsNull(op); + OperatingSystemName = Runtime.GetManagedString(op); + Runtime.XDecref(op); + Runtime.XDecref(fn); + + fn = Runtime.PyObject_GetAttrString(platformModule, "machine"); + op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); + MachineName = Runtime.GetManagedString(op); + Runtime.XDecref(op); + Runtime.XDecref(fn); + + Runtime.XDecref(emptyTuple); + Runtime.XDecref(platformModule); + + // Now convert the strings into enum values so we can do switch + // statements rather than constant parsing. + OperatingSystemType OSType; + if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) + { + OSType = OperatingSystemType.Other; + } + OperatingSystem = OSType; + + MachineType MType; + if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) + { + MType = MachineType.Other; + } + Machine = MType; + } + + internal static IMemoryMapper CreateMemoryMapper() + { + switch (OperatingSystem) + { + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: + return new UnixMemoryMapper(); + case OperatingSystemType.Windows: + return new WindowsMemoryMapper(); + default: + throw new NotImplementedException($"No support for {OperatingSystemName}"); + } + } + + /// + /// Initializes the native code page. + /// + /// Safe to call if we already initialized (this function is idempotent). + /// + /// + internal static void InitializeNativeCodePage() + { + // Do nothing if we already initialized. + if (NativeCodePage != IntPtr.Zero) + { + return; + } + + // Allocate the page, write the native code into it, then set it + // to be executable. + IMemoryMapper mapper = CreateMemoryMapper(); + int codeLength = NativeCode.Active.Code.Length; + NativeCodePage = mapper.MapWriteable(codeLength); + Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); + mapper.SetReadExec(NativeCodePage, codeLength); + } + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 18dc0c113..0123bc499 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -107,59 +107,13 @@ public class Runtime internal static object IsFinalizingLock = new object(); internal static bool IsFinalizing; + private static bool _isInitialized = false; + internal static bool Is32Bit = IntPtr.Size == 4; // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static string OperatingSystemName { get; private set; } - - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - ["armv7l"] = MachineType.armv7l, - ["armv8"] = MachineType.armv8, - ["aarch64"] = MachineType.aarch64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; @@ -178,6 +132,12 @@ public class Runtime /// internal static void Initialize(bool initSigs = false, bool softShutdown = false) { + if (_isInitialized) + { + return; + } + _isInitialized = true; + if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") { softShutdown = true; @@ -329,10 +289,10 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false // Initialize data about the platform we're running on. We need // this for the type manager and potentially other details. Must // happen after caching the python types, above. - InitializePlatformData(); + NativeCodePageHelper.InitializePlatformData(); IntPtr dllLocal = IntPtr.Zero; - var loader = LibraryLoader.Get(OperatingSystem); + var loader = LibraryLoader.Get(NativeCodePageHelper.OperatingSystem); if (_PythonDll != "__Internal") { @@ -362,60 +322,15 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() - { - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - PythonException.ThrowIfIsNull(op); - OperatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); - XDecref(op); - XDecref(fn); - XDecref(emptyTuple); - XDecref(platformModule); - - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) - { - OSType = OperatingSystemType.Other; - } - OperatingSystem = OSType; - - MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) - { - MType = MachineType.Other; - } - Machine = MType; - } - - internal static void Shutdown(bool soft) + internal static void Shutdown(bool soft = false) { - if (Py_IsInitialized() == 0) + if (Py_IsInitialized() == 0 || !_isInitialized) { return; } + _isInitialized = false; + PyGILState_Ensure(); AssemblyManager.Shutdown(); @@ -531,7 +446,10 @@ private static void MoveClrInstancesOnwershipToPython() // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), // thus just be safe to give it back to GC chain. PyObject_GC_Track(obj.pyHandle); - obj.gcHandle.Free(); + if (obj.gcHandle.IsAllocated) + { + obj.gcHandle.Free(); + } obj.gcHandle = new GCHandle(); } ManagedType.ClearTrackedObjects(); @@ -747,7 +665,11 @@ internal static unsafe void XDecref(IntPtr op) internal static unsafe long Refcount(IntPtr op) { +#if PYTHON_WITH_PYDEBUG + var p = (void*)(op + TypeOffset.ob_refcnt); +#else var p = (void*)op; +#endif if ((void*)0 == p) { return 0; @@ -1132,6 +1054,10 @@ internal static long PyObject_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Dir(IntPtr pointer); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _Py_NewReference(IntPtr ob); +#endif //==================================================================== // Python number API @@ -1926,7 +1852,11 @@ internal static bool PyIter_Check(IntPtr pointer) internal static extern string PyModule_GetFilename(IntPtr module); #if PYTHON3 +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] +#else [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] +#endif internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); #endif @@ -2188,7 +2118,7 @@ internal static void Py_CLEAR(ref IntPtr ob) internal static void SetNoSiteFlag() { - var loader = LibraryLoader.Get(OperatingSystem); + var loader = LibraryLoader.Get(NativeCodePageHelper.OperatingSystem); IntPtr dllLocal; if (_PythonDll != "__Internal") diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs index b0a2e8d79..42448a2e9 100644 --- a/src/runtime/slots/mp_length.cs +++ b/src/runtime/slots/mp_length.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; @@ -8,11 +9,41 @@ namespace Python.Runtime.Slots { internal static class mp_length_slot { + private static MethodInfo _lengthMethod; + public static MethodInfo Method + { + get + { + if (_lengthMethod != null) + { + return _lengthMethod; + } + _lengthMethod = typeof(mp_length_slot).GetMethod( + nameof(mp_length_slot.mp_length), + BindingFlags.Static | BindingFlags.NonPublic); + Debug.Assert(_lengthMethod != null); + return _lengthMethod; + } + } + + public static bool CanAssgin(Type clrType) + { + if (typeof(ICollection).IsAssignableFrom(clrType)) + { + return true; + } + if (clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return true; + } + return false; + } + /// /// Implements __len__ for classes that implement ICollection /// (this includes any IList implementer or Array subclass) /// - public static int mp_length(IntPtr ob) + private static int mp_length(IntPtr ob) { var co = ManagedType.GetManagedObject(ob) as CLRObject; if (co == null) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index bd549550b..888131c44 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; -using Python.Runtime.Platform; using System.Diagnostics; using Python.Runtime.Slots; @@ -24,8 +23,17 @@ internal class TypeManager private static readonly Dictionary cache = new Dictionary(); private static readonly Dictionary _slotsHolders = new Dictionary(); + // Slots which must be set + private static readonly string[] _requiredSlots = new string[] + { + "tp_traverse", + "tp_clear", + }; + public static void Initialize() { + Debug.Assert(cache.Count == 0, "Cache should be empty", + "Some errors may occurred on last shutdown"); IntPtr type = SlotHelper.CreateObjectType(); subtype_traverse = Marshal.ReadIntPtr(type, TypeOffset.tp_traverse); subtype_clear = Marshal.ReadIntPtr(type, TypeOffset.tp_clear); @@ -192,11 +200,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl.GetType(), slotsHolder); - // add a __len__ slot for inheritors of ICollection and ICollection<> - if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + if (Marshal.ReadIntPtr(type, TypeOffset.mp_length) == IntPtr.Zero + && mp_length_slot.CanAssgin(clrType)) { - var method = typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length)); - InitializeSlot(type, TypeOffset.mp_length, method, slotsHolder); + InitializeSlot(type, TypeOffset.mp_length, mp_length_slot.Method, slotsHolder); } if (base_ != IntPtr.Zero) @@ -384,6 +391,9 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl, slotsHolder); + Marshal.WriteIntPtr(type, TypeOffset.tp_traverse, Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_traverse)); + Marshal.WriteIntPtr(type, TypeOffset.tp_clear, Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_clear)); + int flags = TypeFlags.Default; flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; @@ -396,7 +406,7 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) Debug.Assert(4 * IntPtr.Size == Marshal.SizeOf(typeof(PyMethodDef))); IntPtr mdefStart = mdef; ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); - slotsHolder.KeeapAlive(thunkInfo.Target); + slotsHolder.KeeapAlive(thunkInfo); { IntPtr mdefAddr = mdef; @@ -418,7 +428,7 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) ); thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); - slotsHolder.KeeapAlive(thunkInfo.Target); + slotsHolder.KeeapAlive(thunkInfo); { IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => @@ -514,6 +524,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) internal static IntPtr AllocateTypeObject(string name) { IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyTypeType, 0); + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to @@ -553,226 +567,6 @@ internal static IntPtr AllocateTypeObject(string name) return type; } - - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch (Runtime.Machine) - { - case MachineType.i386: - return I386; - case MachineType.x86_64: - return X86_64; - default: - return null; - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case OperatingSystemType.Darwin: - return 0x1000; - case OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case OperatingSystemType.Darwin: - case OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } - #endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of @@ -791,12 +585,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo foreach (MethodInfo method in methods) { string name = method.Name; - if (!(name.StartsWith("tp_") || - name.StartsWith("nb_") || - name.StartsWith("sq_") || - name.StartsWith("mp_") || - name.StartsWith("bf_") - )) + if (!name.StartsWith("tp_") && !SlotTypes.IsSlotName(name)) { continue; } @@ -814,58 +603,17 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo impl = impl.BaseType; } - var native = NativeCode.Active; - - // The garbage collection related slots always have to return 1 or 0 - // since .NET objects don't take part in Python's gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // These have to be defined, though, so by default we fill these with - // static C# functions from this class. - if (native != null) - { - // If we want to support domain reload, the C# implementation - // cannot be used as the assembly may get released before - // CPython calls these functions. Instead, for amd64 and x86 we - // load them into a separate code page that is leaked - // intentionally. - InitializeNativeCodePage(); - IntPtr ret1 = NativeCodePage + native.Return1; - IntPtr ret0 = NativeCodePage + native.Return0; - - InitializeSlot(type, ret0, "tp_traverse", false); - InitializeSlot(type, ret0, "tp_clear", false); - } - else + foreach (string slot in _requiredSlots) { - if (!IsSlotSet(type, "tp_traverse")) + if (IsSlotSet(type, slot)) { - var thunkRet0 = Interop.GetThunk(((Func)Return0).Method); - var offset = GetSlotOffset("tp_traverse"); - Marshal.WriteIntPtr(type, offset, thunkRet0.Address); - if (slotsHolder != null) - { - slotsHolder.Set(offset, thunkRet0); - } - } - if (!IsSlotSet(type, "tp_clear")) - { - var thunkRet0 = Interop.GetThunk(((Func)Return0).Method); - var offset = GetSlotOffset("tp_clear"); - Marshal.WriteIntPtr(type, offset, thunkRet0.Address); - if (slotsHolder != null) - { - slotsHolder.Set(offset, thunkRet0); - } + continue; } + var offset = TypeOffset.GetSlotOffset(slot); + Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } } - static int Return1(IntPtr _) => 1; - - static int Return0(IntPtr _) => 0; - /// /// Helper for InitializeSlots. /// @@ -879,7 +627,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo /// Can override the slot when it existed static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) { - var offset = GetSlotOffset(name); + var offset = TypeOffset.GetSlotOffset(name); if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) { return; @@ -914,17 +662,9 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots } } - static int GetSlotOffset(string name) - { - Type typeOffset = typeof(TypeOffset); - FieldInfo fi = typeOffset.GetField(name); - var offset = (int)fi.GetValue(typeOffset); - return offset; - } - static bool IsSlotSet(IntPtr type, string name) { - int offset = GetSlotOffset(name); + int offset = TypeOffset.GetSlotOffset(name); return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } @@ -989,7 +729,7 @@ class SlotsHolder private readonly IntPtr _type; private Dictionary _slots = new Dictionary(); - private List _keepalive = new List(); + private List _keepalive = new List(); private Dictionary _customResetors = new Dictionary(); private List _deallocators = new List(); private bool _alreadyReset = false; @@ -1018,9 +758,9 @@ public void AddDealloctor(Action deallocate) _deallocators.Add(deallocate); } - public void KeeapAlive(Delegate d) + public void KeeapAlive(ThunkInfo thunk) { - _keepalive.Add(d); + _keepalive.Add(thunk); } public void ResetSlots() @@ -1070,9 +810,29 @@ public void ResetSlots() Runtime.XDecref(tp_bases); tp_bases = Runtime.PyTuple_New(0); Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); + try + { + IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); + if (handlePtr != IntPtr.Zero) + { + GCHandle handle = GCHandle.FromIntPtr(handlePtr); + if (handle.IsAllocated) + { + handle.Free(); + } + Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); + } + + } + catch (Exception) + { + + throw; + } + } - private static IntPtr GetDefaultSlot(int offset) + public static IntPtr GetDefaultSlot(int offset) { if (offset == TypeOffset.tp_clear) { @@ -1148,4 +908,28 @@ public static IntPtr CreateObjectType() return A; } } + + + static partial class SlotTypes + { + private static Dictionary _typeMap = new Dictionary(); + private static Dictionary _nameMap = new Dictionary(); + + static SlotTypes() + { + foreach (var type in Types) + { + FieldInfo[] fields = type.GetFields(); + foreach (var fi in fields) + { + _nameMap[fi.Name] = type; + } + } + } + + public static bool IsSlotName(string name) + { + return _nameMap.ContainsKey(name); + } + } } diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index 1f4751939..322d43ad9 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -21,6 +21,11 @@ import sysconfig import subprocess +if sys.version_info.major > 2: + from io import StringIO +else: + from StringIO import StringIO + from pycparser import c_ast, c_parser _log = logging.getLogger() @@ -55,15 +60,18 @@ def __init__(self): self.__struct_members_stack = [] self.__ptr_decl_depth = 0 self.__struct_members = {} + self.__decl_names = {} def get_struct_members(self, name): """return a list of (name, type) of struct members""" - if name in self.__typedefs: - node = self.__get_leaf_node(self.__typedefs[name]) - name = node.name - if name not in self.__struct_members: - raise Exception("Unknown struct '%s'" % name) - return self.__struct_members[name] + defs = self.__typedefs.get(name) + if defs is None: + return None + node = self.__get_leaf_node(defs) + name = node.name + if name is None: + name = defs.declname + return self.__struct_members.get(name) def visit(self, node): if isinstance(node, c_ast.FileAST): @@ -92,6 +100,7 @@ def visit_typedef(self, typedef): self.visit(typedef.type) def visit_typedecl(self, typedecl): + self.__decl_names[typedecl.type] = typedecl.declname self.visit(typedecl.type) def visit_struct(self, struct): @@ -160,7 +169,22 @@ def __get_leaf_node(self, node): return node def __get_struct_name(self, node): - return node.name or "_struct_%d" % id(node) + return node.name or self.__decl_names.get(node) or "_struct_%d" % id(node) + + +class Writer(object): + + def __init__(self): + self._stream = StringIO() + + def append(self, indent=0, code=""): + self._stream.write("%s%s\n" % (indent * " ", code)) + + def extend(self, s): + self._stream.write(s) + + def to_string(self): + return self._stream.getvalue() def preprocess_python_headers(): @@ -186,6 +210,7 @@ def preprocess_python_headers(): defines.extend([ "-D", "__inline=inline", "-D", "__ptr32=", + "-D", "__ptr64=", "-D", "__declspec(x)=", ]) @@ -209,9 +234,8 @@ def preprocess_python_headers(): return "\n".join(lines) -def gen_interop_code(members): - """Generate the TypeOffset C# class""" +def gen_interop_head(writer): defines = [ "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR) ] @@ -241,27 +265,26 @@ def gen_interop_code(members): namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } +""" % (filename, defines_str) + writer.extend(class_definition) + + +def gen_interop_tail(writer): + tail = """} +#endif +""" + writer.extend(tail) + +def gen_heap_type_members(parser, writer): + """Generate the TypeOffset C# class""" + members = parser.get_struct_members("PyHeapTypeObject") + class_definition = """ + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h -""" % (filename, defines_str) +""" # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. @@ -273,11 +296,36 @@ def gen_interop_code(members): /* here are optional user slots, followed by the members. */ public static int members = 0; } -} -#endif """ - return class_definition + writer.extend(class_definition) + + +def gen_structure_code(parser, writer, type_name, indent): + members = parser.get_struct_members(type_name) + if members is None: + return False + out = writer.append + out(indent, "[StructLayout(LayoutKind.Sequential)]") + out(indent, "internal struct %s" % type_name) + out(indent, "{") + for name, tpy in members: + out(indent + 1, "public IntPtr %s;" % name) + out(indent, "}") + out() + return True + + +def gen_supported_slot_record(writer, types, indent): + out = writer.append + out(indent, "internal static partial class SlotTypes") + out(indent, "{") + out(indent + 1, "public static readonly Type[] Types = {") + for name in types: + out(indent + 2, "typeof(%s)," % name) + out(indent + 1, "};") + out(indent, "}") + out() def main(): @@ -290,10 +338,29 @@ def main(): ast_parser = AstParser() ast_parser.visit(ast) + writer = Writer() # generate the C# code - members = ast_parser.get_struct_members("PyHeapTypeObject") - interop_cs = gen_interop_code(members) + gen_interop_head(writer) + + gen_heap_type_members(ast_parser, writer) + slots_types = [ + "PyNumberMethods", + "PySequenceMethods", + "PyMappingMethods", + "PyAsyncMethods", + "PyBufferProcs", + ] + supported_types = [] + indent = 1 + for type_name in slots_types: + if not gen_structure_code(ast_parser, writer, type_name, indent): + continue + supported_types.append(type_name) + gen_supported_slot_record(writer, supported_types, indent) + + gen_interop_tail(writer) + interop_cs = writer.to_string() if len(sys.argv) > 1: with open(sys.argv[1], "w") as fh: fh.write(interop_cs) From 49130c490cf092137f5a9e67b0025e789fed290f Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 23 Jan 2020 16:27:10 +0800 Subject: [PATCH 028/134] Remove unused code --- src/runtime/typemanager.cs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 888131c44..8a493ae77 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -810,8 +810,7 @@ public void ResetSlots() Runtime.XDecref(tp_bases); tp_bases = Runtime.PyTuple_New(0); Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); - try - { + IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); if (handlePtr != IntPtr.Zero) { @@ -822,14 +821,6 @@ public void ResetSlots() } Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); } - - } - catch (Exception) - { - - throw; - } - } public static IntPtr GetDefaultSlot(int offset) From 76ba510e7fa8f6419e5de08107755ef1e951ee2e Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 1 Feb 2020 18:33:15 +0800 Subject: [PATCH 029/134] Add tp_clear for constructorbinding --- src/runtime/constructorbinding.cs | 14 ++++++++++++++ src/runtime/typemanager.cs | 1 - 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 82bcb49f1..bd414229d 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -146,6 +146,13 @@ public static IntPtr tp_repr(IntPtr ob) self.Dealloc(); } + public static int tp_clear(IntPtr ob) + { + var self = (ConstructorBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (ConstructorBinding)GetManagedObject(ob); @@ -241,6 +248,13 @@ public static IntPtr tp_repr(IntPtr ob) self.Dealloc(); } + public static int tp_clear(IntPtr ob) + { + var self = (BoundContructor)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) { var self = (BoundContructor)GetManagedObject(ob); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 8a493ae77..bd3109dc6 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -903,7 +903,6 @@ public static IntPtr CreateObjectType() static partial class SlotTypes { - private static Dictionary _typeMap = new Dictionary(); private static Dictionary _nameMap = new Dictionary(); static SlotTypes() From 1ff21ac0c62cd17c219c8e1f8e9ab156963c0719 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 3 Feb 2020 11:22:25 +0800 Subject: [PATCH 030/134] tp_clear for EventBinding --- src/runtime/eventbinding.cs | 7 +++++++ src/runtime/typemanager.cs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index c844f5fd4..dc3a6dfae 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -120,5 +120,12 @@ public static IntPtr tp_repr(IntPtr ob) Runtime.XDecref(self.target); self.Dealloc(); } + + public static int tp_clear(IntPtr ob) + { + var self = (EventBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.target); + return 0; + } } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index bd3109dc6..0910c5128 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -30,7 +30,7 @@ internal class TypeManager "tp_clear", }; - public static void Initialize() + internal static void Initialize() { Debug.Assert(cache.Count == 0, "Cache should be empty", "Some errors may occurred on last shutdown"); From 0b013787a3f3ba6c15ea9cbd63c8f4bea97a88b3 Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 4 Feb 2020 11:14:34 +0800 Subject: [PATCH 031/134] All base type corrected, remove unnecessary slot set --- src/runtime/typemanager.cs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 0910c5128..3d6e66a03 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -391,9 +391,6 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl, slotsHolder); - Marshal.WriteIntPtr(type, TypeOffset.tp_traverse, Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_traverse)); - Marshal.WriteIntPtr(type, TypeOffset.tp_clear, Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_clear)); - int flags = TypeFlags.Default; flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; @@ -801,16 +798,6 @@ public void ResetSlots() _deallocators.Clear(); // Custom reset - IntPtr tp_base = Marshal.ReadIntPtr(_type, TypeOffset.tp_base); - Runtime.XDecref(tp_base); - Runtime.XIncref(Runtime.PyBaseObjectType); - Marshal.WriteIntPtr(_type, TypeOffset.tp_base, Runtime.PyBaseObjectType); - - IntPtr tp_bases = Marshal.ReadIntPtr(_type, TypeOffset.tp_bases); - Runtime.XDecref(tp_bases); - tp_bases = Runtime.PyTuple_New(0); - Marshal.WriteIntPtr(_type, TypeOffset.tp_bases, tp_bases); - IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); if (handlePtr != IntPtr.Zero) { From bf3d9f847594e2a5558e83e954614e9a9c0f5982 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 5 Feb 2020 18:54:20 +0800 Subject: [PATCH 032/134] Add basic `reload` shutdown mode --- src/embed_tests/TestDomainReload.cs | 2 +- src/runtime/classbase.cs | 1 + src/runtime/classmanager.cs | 17 ++- src/runtime/classobject.cs | 1 + src/runtime/clrobject.cs | 1 + src/runtime/constructorbinder.cs | 1 + src/runtime/constructorbinding.cs | 4 + src/runtime/delegateobject.cs | 1 + src/runtime/eventbinding.cs | 7 ++ src/runtime/eventobject.cs | 1 + src/runtime/extensiontype.cs | 15 +++ src/runtime/indexer.cs | 1 + src/runtime/interop.cs | 81 ++++++++------- src/runtime/managedtype.cs | 20 +++- src/runtime/metatype.cs | 31 ++++++ src/runtime/methodbinder.cs | 1 + src/runtime/methodbinding.cs | 1 + src/runtime/methodobject.cs | 2 + src/runtime/modulefunctionobject.cs | 1 + src/runtime/moduleobject.cs | 19 ++++ src/runtime/nativecall.cs | 134 ++---------------------- src/runtime/propertyobject.cs | 1 + src/runtime/pythonengine.cs | 15 +-- src/runtime/runtime.cs | 55 ++++++++-- src/runtime/runtime_state.cs | 79 +++++++++++++- src/runtime/typemanager.cs | 156 +++++++++++++++------------- 26 files changed, 387 insertions(+), 261 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 5bfde11c1..d987ad30b 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -85,7 +85,7 @@ public static void RunPython() { AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - PythonEngine.Initialize(softShutdown: true); + PythonEngine.Initialize(mode: ShutdownMode.Reload); using (Py.GIL()) { try { var pyScript = string.Format(""import clr\n"" diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index e98bed638..3f10849fd 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -13,6 +13,7 @@ namespace Python.Runtime /// concrete subclasses provide slot implementations appropriate for /// each variety of reflected type. /// + [Serializable] internal class ClassBase : ManagedType { internal Indexer indexer; diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index aa6fd6e51..3b9922a3d 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -19,7 +19,7 @@ namespace Python.Runtime internal class ClassManager { private static Dictionary cache; - private static Type dtype; + private static readonly Type dtype; private ClassManager() { @@ -81,6 +81,17 @@ private static int OnVisit(IntPtr ob, IntPtr arg) return 0; } + + internal static void StashPush(Stack stack) + { + stack.Push(cache); + } + + internal static void StashPop(Stack stack) + { + cache = (Dictionary)stack.Pop(); + } + /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. @@ -239,7 +250,7 @@ private static void InitClassBase(Type type, ClassBase impl) private static ClassInfo GetClassInfo(Type type) { - var ci = new ClassInfo(type); + var ci = new ClassInfo(); var methods = new Hashtable(); ArrayList list; MethodInfo meth; @@ -443,7 +454,7 @@ internal class ClassInfo public Indexer indexer; public Hashtable members; - internal ClassInfo(Type t) + internal ClassInfo() { members = new Hashtable(); indexer = null; diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 83d761fd0..9b63f76c7 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// Python type objects. Each of those type objects is associated with /// an instance of ClassObject, which provides its implementation. /// + [Serializable] internal class ClassObject : ClassBase { internal ConstructorBinder binder; diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 29c57b28f..7c4330219 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -9,6 +9,7 @@ internal class CLRObject : ManagedType internal CLRObject(object ob, IntPtr tp) { + System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 1fc541920..c2d30217f 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// standard MethodBinder because of a difference in invoking constructors /// using reflection (which is seems to be a CLR bug). /// + [Serializable] internal class ConstructorBinder : MethodBinder { private Type _containingType; diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index bd414229d..0c81c0a93 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -19,11 +19,14 @@ namespace Python.Runtime /// and creating the BoundContructor object which contains ContructorInfo object. /// 3) In tp_call, if ctorInfo is not null, ctorBinder.InvokeRaw() is called. /// + [Serializable] internal class ConstructorBinding : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; + + [NonSerialized] private IntPtr repr; public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) @@ -173,6 +176,7 @@ public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) /// An earlier implementation hung the __call__ on the ContructorBinding class and /// returned an Incref()ed self.pyHandle from the __get__ function. /// + [Serializable] internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index e1103cbc7..24060b8bb 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// Each of those type objects is associated an instance of this class, /// which provides its implementation. /// + [Serializable] internal class DelegateObject : ClassBase { private MethodBinder binder; diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index dc3a6dfae..5dbace9d9 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -5,6 +5,7 @@ namespace Python.Runtime /// /// Implements a Python event binding type, similar to a method binding. /// + [Serializable] internal class EventBinding : ExtensionType { private EventObject e; @@ -127,5 +128,11 @@ public static int tp_clear(IntPtr ob) Runtime.Py_CLEAR(ref self.target); return 0; } + + protected override void OnSave() + { + base.OnSave(); + Runtime.XIncref(target); + } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 2a98896fe..0f2796a14 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR events. /// + [Serializable] internal class EventObject : ExtensionType { internal string name; diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index c1aff3ca0..006d616b2 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// type object, such as the types that represent CLR methods, fields, /// etc. Instances implemented by this class do not support sub-typing. /// + [Serializable] internal abstract class ExtensionType : ManagedType { public ExtensionType() @@ -96,5 +97,19 @@ public static void tp_dealloc(IntPtr ob) var self = (ExtensionType)GetManagedObject(ob); self.Dealloc(); } + + protected override void OnSave() + { + base.OnSave(); + Runtime.XIncref(pyHandle); + } + + protected override void OnLoad() + { + base.OnLoad(); + GCHandle gc = AllocGCHandle(true); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + Runtime.PyObject_GC_UnTrack(pyHandle); + } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..0772b57c6 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Bundles the information required to support an indexer property. /// + [Serializable] internal class Indexer { public MethodBinder GetterBinder; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index d4b4b5119..58464ec7f 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -311,45 +311,45 @@ public static string GetSlotNameByOffset(int offset) internal class TypeFlags { #if PYTHON2 // these flags were removed in Python 3 - public static int HaveGetCharBuffer = (1 << 0); - public static int HaveSequenceIn = (1 << 1); - public static int GC = 0; - public static int HaveInPlaceOps = (1 << 3); - public static int CheckTypes = (1 << 4); - public static int HaveRichCompare = (1 << 5); - public static int HaveWeakRefs = (1 << 6); - public static int HaveIter = (1 << 7); - public static int HaveClass = (1 << 8); + public const int HaveGetCharBuffer = (1 << 0); + public const int HaveSequenceIn = (1 << 1); + public const int GC = 0; + public const int HaveInPlaceOps = (1 << 3); + public const int CheckTypes = (1 << 4); + public const int HaveRichCompare = (1 << 5); + public const int HaveWeakRefs = (1 << 6); + public const int HaveIter = (1 << 7); + public const int HaveClass = (1 << 8); #endif - public static int HeapType = (1 << 9); - public static int BaseType = (1 << 10); - public static int Ready = (1 << 12); - public static int Readying = (1 << 13); - public static int HaveGC = (1 << 14); + public const int HeapType = (1 << 9); + public const int BaseType = (1 << 10); + public const int Ready = (1 << 12); + public const int Readying = (1 << 13); + public const int HaveGC = (1 << 14); // 15 and 16 are reserved for stackless - public static int HaveStacklessExtension = 0; + public const int HaveStacklessExtension = 0; /* XXX Reusing reserved constants */ - public static int Managed = (1 << 15); // PythonNet specific - public static int Subclass = (1 << 16); // PythonNet specific - public static int HaveIndex = (1 << 17); + public const int Managed = (1 << 15); // PythonNet specific + public const int Subclass = (1 << 16); // PythonNet specific + public const int HaveIndex = (1 << 17); /* Objects support nb_index in PyNumberMethods */ - public static int HaveVersionTag = (1 << 18); - public static int ValidVersionTag = (1 << 19); - public static int IsAbstract = (1 << 20); - public static int HaveNewBuffer = (1 << 21); + public const int HaveVersionTag = (1 << 18); + public const int ValidVersionTag = (1 << 19); + public const int IsAbstract = (1 << 20); + public const int HaveNewBuffer = (1 << 21); // TODO: Implement FastSubclass functions - public static int IntSubclass = (1 << 23); - public static int LongSubclass = (1 << 24); - public static int ListSubclass = (1 << 25); - public static int TupleSubclass = (1 << 26); - public static int StringSubclass = (1 << 27); - public static int UnicodeSubclass = (1 << 28); - public static int DictSubclass = (1 << 29); - public static int BaseExceptionSubclass = (1 << 30); - public static int TypeSubclass = (1 << 31); + public const int IntSubclass = (1 << 23); + public const int LongSubclass = (1 << 24); + public const int ListSubclass = (1 << 25); + public const int TupleSubclass = (1 << 26); + public const int StringSubclass = (1 << 27); + public const int UnicodeSubclass = (1 << 28); + public const int DictSubclass = (1 << 29); + public const int BaseExceptionSubclass = (1 << 30); + public const int TypeSubclass = (1 << 31); #if PYTHON2 // Default flags for Python 2 - public static int Default = ( + public const int Default = ( HaveGetCharBuffer | HaveSequenceIn | HaveInPlaceOps | @@ -361,7 +361,7 @@ internal class TypeFlags HaveIndex | 0); #elif PYTHON3 // Default flags for Python 3 - public static int Default = ( + public const int Default = ( HaveStacklessExtension | HaveVersionTag); #endif @@ -499,9 +499,16 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { return ThunkInfo.Empty; } - Delegate d = Delegate.CreateDelegate(dt, method); - var info = new ThunkInfo(d); - return info; + try + { + Delegate d = Delegate.CreateDelegate(dt, method); + var info = new ThunkInfo(d); + return info; + } + catch (Exception) + { + throw; + } } @@ -579,7 +586,7 @@ struct PyGC_Head } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + [StructLayout(LayoutKind.Sequential)] struct PyMethodDef { public IntPtr ml_name; diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index a93628524..cd735898e 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -11,9 +11,12 @@ namespace Python.Runtime /// code. It defines the common fields that associate CLR and Python /// objects and common utilities to convert between those identities. /// + [Serializable] internal abstract class ManagedType { + [NonSerialized] internal GCHandle gcHandle; // Native handle + internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * @@ -190,6 +193,19 @@ protected void TypeClear() ClearObjectDict(pyHandle); } + internal void Save() + { + OnSave(); + } + + internal void Load() + { + OnLoad(); + } + + protected virtual void OnSave() { } + protected virtual void OnLoad() { } + protected static void ClearObjectDict(IntPtr ob) { IntPtr dict = GetObjectDict(ob); @@ -201,12 +217,12 @@ protected static void ClearObjectDict(IntPtr ob) Runtime.XDecref(dict); } - private static IntPtr GetObjectDict(IntPtr ob) + protected static IntPtr GetObjectDict(IntPtr ob) { return Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); } - private static void SetObjectDict(IntPtr ob, IntPtr value) + protected static void SetObjectDict(IntPtr ob, IntPtr value) { Marshal.WriteIntPtr(ob, ObjectOffset.DictOffset(ob), value); } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 68954ef0a..b54c2eb90 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,4 +1,6 @@ using System; +using System.Collections; +using System.IO; using System.Runtime.InteropServices; namespace Python.Runtime @@ -13,6 +15,12 @@ internal class MetaType : ManagedType private static IntPtr PyCLRMetaType; private static SlotsHolder _metaSlotsHodler; + internal static readonly string[] CustomMethods = new string[] + { + "__instancecheck__", + "__subclasscheck__", + }; + /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// @@ -32,6 +40,29 @@ public static void Release() _metaSlotsHodler = null; } + internal static void StashPush(Stack stack) + { + Runtime.XIncref(PyCLRMetaType); + stack.Push(PyCLRMetaType); + } + + internal static IntPtr StashPop(Stack stack) + { + PyCLRMetaType = (IntPtr)stack.Pop(); + _metaSlotsHodler = new SlotsHolder(PyCLRMetaType); + TypeManager.InitializeSlots(PyCLRMetaType, typeof(MetaType), _metaSlotsHodler); + + IntPtr mdef = Marshal.ReadIntPtr(PyCLRMetaType, TypeOffset.tp_methods); + foreach (var methodName in CustomMethods) + { + var mi = typeof(MetaType).GetMethod(methodName); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + _metaSlotsHodler.KeeapAlive(thunkInfo); + mdef = TypeManager.WriteMethodDef(mdef, methodName, thunkInfo.Address); + } + return PyCLRMetaType; + } + /// /// Metatype __new__ implementation. This is called to create a new /// class / type when a reflected class is subclassed. diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 8a7fc1930..0773263ef 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -13,6 +13,7 @@ namespace Python.Runtime /// a set of Python arguments. This is also used as a base class for the /// ConstructorBinder, a minor variation used to invoke constructors. /// + [Serializable] internal class MethodBinder { public ArrayList list; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 1caec322c..c14e592d5 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// standard Python method bindings, but the same type is used to bind /// both static and instance methods. /// + [Serializable] internal class MethodBinding : ExtensionType { internal MethodInfo info; diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 3942aae52..14fb0cd19 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// /// TODO: ForbidPythonThreadsAttribute per method info /// + [Serializable] internal class MethodObject : ExtensionType { internal MethodInfo[] info; @@ -17,6 +18,7 @@ internal class MethodObject : ExtensionType internal MethodBinding unbound; internal MethodBinder binder; internal bool is_static = false; + [NonSerialized] internal IntPtr doc; internal Type type; diff --git a/src/runtime/modulefunctionobject.cs b/src/runtime/modulefunctionobject.cs index 8f8692af9..e7a2c515a 100644 --- a/src/runtime/modulefunctionobject.cs +++ b/src/runtime/modulefunctionobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Module level functions /// + [Serializable] internal class ModuleFunctionObject : MethodObject { public ModuleFunctionObject(Type type, string name, MethodInfo[] info, bool allow_threads) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 06f1c0581..0d42914e5 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -10,9 +10,12 @@ namespace Python.Runtime /// Implements a Python type that provides access to CLR namespaces. The /// type behaves like a Python module, and can contain other sub-modules. /// + [Serializable] internal class ModuleObject : ExtensionType { + [NonSerialized] private Dictionary cache; + internal string moduleName; internal IntPtr dict; protected string _namespace; @@ -338,6 +341,21 @@ public static int tp_clear(IntPtr ob) self.cache.Clear(); return 0; } + + protected override void OnSave() + { + base.OnSave(); + Runtime.XIncref(dict); + Runtime.XIncref(GetObjectDict(pyHandle)); + } + + protected override void OnLoad() + { + base.OnLoad(); + cache = new Dictionary(); + Runtime.XIncref(dict); + SetObjectDict(pyHandle, dict); + } } /// @@ -345,6 +363,7 @@ public static int tp_clear(IntPtr ob) /// to import assemblies. It has a fixed module name "clr" and doesn't /// provide a namespace. /// + [Serializable] internal class CLRModule : ModuleObject { protected static bool hacked = false; diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index 4a7bf05c8..ec0bf338c 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -23,7 +23,6 @@ namespace Python.Runtime /// internal class NativeCall { -#if NETSTANDARD [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate void Void_1_Delegate(IntPtr a1); @@ -32,148 +31,27 @@ internal class NativeCall public static void Void_Call_1(IntPtr fp, IntPtr a1) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); return d(a1, a2, a3); } public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); return d(a1, a2, a3); } -#else - private static AssemblyBuilder aBuilder; - private static ModuleBuilder mBuilder; - public static INativeCall Impl; - - static NativeCall() - { - // The static constructor is responsible for generating the - // assembly and the methods that implement the IJW thunks. - // - // To do this, we actually use reflection on the INativeCall - // interface (defined below) and generate the required thunk - // code based on the method signatures. - - var aname = new AssemblyName { Name = "e__NativeCall_Assembly" }; - var aa = AssemblyBuilderAccess.Run; - - aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("e__NativeCall_Module"); - - var ta = TypeAttributes.Public; - TypeBuilder tBuilder = mBuilder.DefineType("e__NativeCall", ta); - - Type iType = typeof(INativeCall); - tBuilder.AddInterfaceImplementation(iType); - - // Use reflection to loop over the INativeCall interface methods, - // calling GenerateThunk to create a managed thunk for each one. - - foreach (MethodInfo method in iType.GetMethods()) - { - GenerateThunk(tBuilder, method); - } - - Type theType = tBuilder.CreateType(); - - Impl = (INativeCall)Activator.CreateInstance(theType); - } - - private static void GenerateThunk(TypeBuilder tb, MethodInfo method) - { - ParameterInfo[] pi = method.GetParameters(); - int count = pi.Length; - int argc = count - 1; - - var args = new Type[count]; - for (var i = 0; i < count; i++) - { - args[i] = pi[i].ParameterType; - } - - MethodBuilder mb = tb.DefineMethod( - method.Name, - MethodAttributes.Public | - MethodAttributes.Virtual, - method.ReturnType, - args - ); - - // Build the method signature for the actual native function. - // This is essentially the signature of the wrapper method - // minus the first argument (the passed in function pointer). - - var nargs = new Type[argc]; - for (var i = 1; i < count; i++) - { - nargs[i - 1] = args[i]; - } - - // IL generation: the (implicit) first argument of the method - // is the 'this' pointer and the second is the function pointer. - // This code pushes the real args onto the stack, followed by - // the function pointer, then the calli opcode to make the call. - - ILGenerator il = mb.GetILGenerator(); - - for (var i = 0; i < argc; i++) - { - il.Emit(OpCodes.Ldarg_S, i + 2); - } - - il.Emit(OpCodes.Ldarg_1); - - il.EmitCalli(OpCodes.Calli, - CallingConvention.Cdecl, - method.ReturnType, - nargs - ); - - il.Emit(OpCodes.Ret); - - tb.DefineMethodOverride(mb, method); - } - - - public static void Void_Call_1(IntPtr fp, IntPtr a1) - { - Impl.Void_Call_1(fp, a1); - } - - public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) - { - return Impl.Call_3(fp, a1, a2, a3); - } - - public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) + private static T GetDelegate(IntPtr fp) where T: Delegate { - return Impl.Int_Call_3(fp, a1, a2, a3); + // Use Marshal.GetDelegateForFunctionPointer<> directly after upgrade the framework + return (T)Marshal.GetDelegateForFunctionPointer(fp, typeof(T)); } -#endif - } - -#if !NETSTANDARD - /// - /// Defines native call signatures to be generated by NativeCall. - /// - public interface INativeCall - { - void Void_Call_0(IntPtr funcPtr); - - void Void_Call_1(IntPtr funcPtr, IntPtr arg1); - - int Int_Call_3(IntPtr funcPtr, IntPtr t, IntPtr n, IntPtr v); - - IntPtr Call_3(IntPtr funcPtr, IntPtr a1, IntPtr a2, IntPtr a3); } -#endif } diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..ac1d077f9 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that manages CLR properties. /// + [Serializable] internal class PropertyObject : ExtensionType { private PropertyInfo info; diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 8688fe17f..301eea44e 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,7 +12,7 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { - public static bool SoftShutdown => Runtime.SoftShutdown; + public static ShutdownMode ShutdownMode => Runtime.ShutdownMode; private static DelegateManager delegateManager; private static bool initialized; @@ -152,9 +152,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false, bool softShutdown = false) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Normal) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, softShutdown); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); } /// @@ -167,7 +167,7 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false, boo /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, bool softShutdown = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Normal) { if (initialized) { @@ -179,7 +179,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs, softShutdown); + Runtime.Initialize(initSigs, mode); initialized = true; Exceptions.Clear(); @@ -187,7 +187,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; // Remember to shut down the runtime. - AddShutdownHandler(() => Runtime.Shutdown(softShutdown)); + AddShutdownHandler(() => Runtime.Shutdown(mode)); // The global scope gets used implicitly quite early on, remember // to clear it out when we shut down. @@ -198,8 +198,9 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, Py.SetArgv(args); } - if (!softShutdown) + if (mode == ShutdownMode.Normal) { + // TOOD: Check if this can be remove completely or not. // register the atexit callback (this doesn't use Py_AtExit as the C atexit // callbacks are called after python is fully finalized but the python ones // are called while the python engine is still running). diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 0123bc499..71a58dfef 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using Python.Runtime.Platform; using System.Linq; +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; namespace Python.Runtime { @@ -124,13 +126,13 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - public static bool SoftShutdown { get; private set; } + public static ShutdownMode ShutdownMode { get; private set; } private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false, bool softShutdown = false) + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Normal) { if (_isInitialized) { @@ -140,10 +142,14 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") { - softShutdown = true; + mode = ShutdownMode.Soft; + } + else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") + { + mode = ShutdownMode.Reload; } - SoftShutdown = softShutdown; + ShutdownMode = mode; if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); @@ -151,7 +157,7 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false { PyEval_InitThreads(); } - if (softShutdown) + if (mode == ShutdownMode.Soft) { RuntimeState.Save(); } @@ -308,7 +314,16 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); - PyCLRMetaType = MetaType.Initialize(); // Steal a reference + + if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) + { + RuntimeData.StashPop(); + } + else + { + PyCLRMetaType = MetaType.Initialize(); // Steal a reference + } + Exceptions.Initialize(); ImportHook.Initialize(); @@ -323,7 +338,7 @@ internal static void Initialize(bool initSigs = false, bool softShutdown = false } - internal static void Shutdown(bool soft = false) + internal static void Shutdown(ShutdownMode mode = ShutdownMode.Normal) { if (Py_IsInitialized() == 0 || !_isInitialized) { @@ -333,12 +348,17 @@ internal static void Shutdown(bool soft = false) PyGILState_Ensure(); + RuntimeData.Stash(); + AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); - ClearClrModules(); + if (mode != ShutdownMode.Reload) + { + ClearClrModules(); + } RemoveClrRootModule(); MoveClrInstancesOnwershipToPython(); @@ -348,10 +368,13 @@ internal static void Shutdown(bool soft = false) MetaType.Release(); PyCLRMetaType = IntPtr.Zero; - if (soft) + if (mode != ShutdownMode.Normal) { PyGC_Collect(); - RuntimeState.Restore(); + if (mode == ShutdownMode.Soft) + { + RuntimeState.Restore(); + } ResetPyMembers(); GC.Collect(); try @@ -657,7 +680,7 @@ internal static unsafe void XDecref(IntPtr op) { return; } - NativeCall.Impl.Void_Call_1(new IntPtr(f), op); + NativeCall.Void_Call_1(new IntPtr(f), op); } } #endif @@ -2099,6 +2122,8 @@ internal static void Py_CLEAR(ref IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyCapsule_GetPointer(IntPtr capsule, string name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCapsule_SetPointer(IntPtr capsule, IntPtr pointer); //==================================================================== // Miscellaneous @@ -2151,6 +2176,14 @@ internal static IntPtr GetBuiltins() } + public enum ShutdownMode + { + Normal, + Soft, + Reload, + } + + class PyReferenceCollection { private List> _actions = new List>(); diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index bb91b125c..d7f6f5558 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -1,8 +1,11 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; -using System.Linq; +using System.Runtime.Serialization.Formatters.Binary; + using static Python.Runtime.Runtime; namespace Python.Runtime @@ -233,4 +236,78 @@ private static unsafe IntPtr GetNextGCNode(IntPtr node) return ((PyGC_Head*)node)->gc.gc_next; } } + + + class RuntimeData + { + internal static void Stash() + { + var formatter = new BinaryFormatter(); + var ms = new MemoryStream(); + var stack = new Stack(); + MetaType.StashPush(stack); + TypeManager.StashPush(stack); + ClassManager.StashPush(stack); + var objs = ManagedType.GetManagedObjects(); + foreach (var obj in objs) + { + obj.Save(); + } + stack.Push(objs); + formatter.Serialize(ms, stack); + + byte[] data = ms.GetBuffer(); + // TODO: use buffer api instead + System.Diagnostics.Debug.Assert(data.Length <= int.MaxValue); + IntPtr mem = PyMem_Malloc(data.LongLength + IntPtr.Size); + Marshal.WriteIntPtr(mem, (IntPtr)data.LongLength); + Marshal.Copy(data, 0, mem + IntPtr.Size, data.Length); + + IntPtr capsule = PySys_GetObject("clr_data"); + if (capsule != IntPtr.Zero) + { + IntPtr oldData = PyCapsule_GetPointer(capsule, null); + PyMem_Free(oldData); + PyCapsule_SetPointer(capsule, IntPtr.Zero); + } + capsule = PyCapsule_New(mem, null, IntPtr.Zero); + PySys_SetObject("clr_data", capsule); + XDecref(capsule); + } + + internal static void StashPop() + { + IntPtr capsule = PySys_GetObject("clr_data"); + if (capsule == IntPtr.Zero) + { + return; + } + IntPtr mem = PyCapsule_GetPointer(capsule, null); + int length = (int)Marshal.ReadIntPtr(mem); + byte[] data = new byte[length]; + Marshal.Copy(mem + IntPtr.Size, data, 0, length); + var ms = new MemoryStream(data); + var formatter = new BinaryFormatter(); + + var stack = (Stack)formatter.Deserialize(ms); + + var loadObjs = (ICollection)stack.Pop(); + var objs = ManagedType.GetManagedObjects(); + foreach (var obj in loadObjs) + { + obj.Load(); + objs.Add(obj); + } + + ClassManager.StashPop(stack); + TypeManager.StashPop(stack); + PyCLRMetaType = MetaType.StashPop(stack); + } + + public static bool HasStashData() + { + return PySys_GetObject("clr_data") != IntPtr.Zero; + } + + } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3d6e66a03..d824c1fb4 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -20,7 +20,7 @@ internal class TypeManager internal static IntPtr subtype_clear; private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static readonly Dictionary cache = new Dictionary(); + private static Dictionary cache = new Dictionary(); private static readonly Dictionary _slotsHolders = new Dictionary(); // Slots which must be set @@ -65,6 +65,30 @@ internal static void RemoveTypes() _slotsHolders.Clear(); } + internal static void StashPush(Stack stack) + { + foreach (var tpHandle in cache.Values) + { + Runtime.XIncref(tpHandle); + } + //formatter.Serialize(stream, cache); + stack.Push(cache); + } + + internal static void StashPop(Stack stack) + { + Debug.Assert(cache == null || cache.Count == 0); + cache = (Dictionary)stack.Pop(); + foreach (var entry in cache) + { + Type type = entry.Key; + IntPtr handle = entry.Value; + SlotsHolder holder = CreateSolotsHolder(handle); + InitializeSlots(handle, type, holder); + // FIXME: mp_length_slot.CanAssgin(clrType) + } + } + /// /// Return value: Borrowed reference. /// Given a managed Type derived from ExtensionType, get the handle to @@ -381,96 +405,89 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.HaveGC; + Util.WriteCLong(type, TypeOffset.tp_flags, flags); + // Slots will inherit from TypeType, it's not neccesary for setting them. // Inheried slots: // tp_basicsize, tp_itemsize, // tp_dictoffset, tp_weaklistoffset, // tp_traverse, tp_clear, tp_is_gc, etc. + slotsHolder = SetupMetaSlots(impl, type); + + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } + IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + IntPtr mod = Runtime.PyString_FromString("CLR"); + Runtime.PyDict_SetItemString(dict, "__module__", mod); + + //DebugUtil.DumpType(type); + + return type; + } + + internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) + { // Override type slots with those of the managed implementation. - slotsHolder = new SlotsHolder(type); + SlotsHolder slotsHolder = new SlotsHolder(type); InitializeSlots(type, impl, slotsHolder); - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); - - // We need space for 3 PyMethodDef structs, each of them - // 4 int-ptrs in size. - IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); - Debug.Assert(4 * IntPtr.Size == Marshal.SizeOf(typeof(PyMethodDef))); + // We need space for 3 PyMethodDef structs. + int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); + IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); IntPtr mdefStart = mdef; - ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); - slotsHolder.KeeapAlive(thunkInfo); + foreach (var methodName in MetaType.CustomMethods) + { + mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); + } + mdef = WriteMethodDefSentinel(mdef); + Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); + Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) { - IntPtr mdefAddr = mdef; - slotsHolder.AddDealloctor(() => + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => { - IntPtr t = type; - IntPtr tp_dict = Marshal.ReadIntPtr(t, TypeOffset.tp_dict); - if (Runtime.PyDict_DelItemString(tp_dict, "__instancecheck__") != 0) - { - Runtime.PyErr_Print(); - } - FreeMethodDef(mdefAddr); + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); }); } - mdef = WriteMethodDef( - mdef, - "__instancecheck__", - thunkInfo.Address - ); + return slotsHolder; + } - thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); + private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, SlotsHolder slotsHolder) + { + MethodInfo mi = typeof(MetaType).GetMethod(name); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); slotsHolder.KeeapAlive(thunkInfo); + + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) { IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => { - IntPtr t = type; - IntPtr tp_dict = Marshal.ReadIntPtr(t, TypeOffset.tp_dict); - if (Runtime.PyDict_DelItemString(tp_dict, "__subclasscheck__") != 0) + IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) { Runtime.PyErr_Print(); + Debug.Fail($"Cannot remove {name} from metatype"); } FreeMethodDef(mdefAddr); }); } - mdef = WriteMethodDef( - mdef, - "__subclasscheck__", - thunkInfo.Address - ); - - // FIXME: mdef is not used - mdef = WriteMethodDefSentinel(mdef); - - Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => - { - var p = Marshal.ReadIntPtr(t, offset); - Runtime.PyMem_Free(p); - Marshal.WriteIntPtr(t, offset, IntPtr.Zero); - }); - - if (Runtime.PyType_Ready(type) != 0) - { - throw new PythonException(); - } - - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(dict, "__module__", mod); - - //DebugUtil.DumpType(type); - - return type; + mdef = WriteMethodDef(mdef, name, thunkInfo.Address); + return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) { // Utility to create a subtype of a std Python type, but with @@ -543,22 +560,19 @@ internal static IntPtr AllocateTypeObject(string name) Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); #endif - - long ptr = type.ToInt64(); // 64-bit safe - - temp = new IntPtr(ptr + TypeOffset.nb_add); + temp = type + TypeOffset.nb_add; Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); - temp = new IntPtr(ptr + TypeOffset.sq_length); + temp = type + TypeOffset.sq_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); - temp = new IntPtr(ptr + TypeOffset.mp_length); + temp = type + TypeOffset.mp_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); #if PYTHON3 - temp = new IntPtr(ptr + TypeOffset.bf_getbuffer); + temp = type + TypeOffset.bf_getbuffer; #elif PYTHON2 - temp = new IntPtr(ptr + TypeOffset.bf_getreadbuffer); + temp = type + TypeOffset.bf_getreadbuffer; #endif Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); return type; @@ -602,7 +616,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo foreach (string slot in _requiredSlots) { - if (IsSlotSet(type, slot)) + if (seen.Contains(slot)) { continue; } From da97502006791bb0597446766ad00a6f9d291895 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 5 Feb 2020 21:25:22 +0800 Subject: [PATCH 033/134] Data synchronization for PyScopeTest.TestThread --- src/embed_tests/TestPyScope.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 21c0d2b3f..1f03eae54 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -337,10 +337,13 @@ public void TestThread() //add function to the scope //can be call many times, more efficient than ast ps.Exec( + "import threading\n"+ + "lock = threading.Lock()\n"+ "def update():\n" + - " global res, th_cnt\n" + - " res += bb + 1\n" + - " th_cnt += 1\n" + " global res, th_cnt\n" + + " with lock:\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" ); } int th_cnt = 3; From e8b31605691462d5bd143c30d57269902a144d05 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 6 Feb 2020 13:33:35 +0800 Subject: [PATCH 034/134] Disable `ShutdownMode.Reload` on `NETSTANDARD` --- src/runtime/runtime.cs | 22 ++++++++++++++++++---- src/runtime/typemanager.cs | 6 +++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 71a58dfef..063ed6d99 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -144,10 +144,13 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd { mode = ShutdownMode.Soft; } +#if !NETSTANDARD else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") { mode = ShutdownMode.Reload; } +#endif + //mode = ShutdownMode.Reload; ShutdownMode = mode; if (Py_IsInitialized() == 0) @@ -314,16 +317,16 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); - +#if !NETSTANDARD if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) { RuntimeData.StashPop(); } else +#endif { PyCLRMetaType = MetaType.Initialize(); // Steal a reference } - Exceptions.Initialize(); ImportHook.Initialize(); @@ -348,17 +351,23 @@ internal static void Shutdown(ShutdownMode mode = ShutdownMode.Normal) PyGILState_Ensure(); - RuntimeData.Stash(); - +#if !NETSTANDARD + if (mode == ShutdownMode.Reload) + { + RuntimeData.Stash(); + } +#endif AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); +#if !NETSTANDARD if (mode != ShutdownMode.Reload) { ClearClrModules(); } +#endif RemoveClrRootModule(); MoveClrInstancesOnwershipToPython(); @@ -970,6 +979,9 @@ internal static bool PyObject_IsIterable(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, string name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, IntPtr name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value); @@ -2180,7 +2192,9 @@ public enum ShutdownMode { Normal, Soft, +#if !NETSTANDARD Reload, +#endif } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d824c1fb4..6475fa919 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -71,7 +71,6 @@ internal static void StashPush(Stack stack) { Runtime.XIncref(tpHandle); } - //formatter.Serialize(stream, cache); stack.Push(cache); } @@ -450,6 +449,8 @@ internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + +#if !NETSTANDARD // XXX: Hard code with mode check. if (Runtime.ShutdownMode != ShutdownMode.Reload) { @@ -460,6 +461,7 @@ internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) Marshal.WriteIntPtr(t, offset, IntPtr.Zero); }); } +#endif return slotsHolder; } @@ -469,8 +471,10 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); slotsHolder.KeeapAlive(thunkInfo); +#if !NETSTANDARD // XXX: Hard code with mode check. if (Runtime.ShutdownMode != ShutdownMode.Reload) +#endif { IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => From 80d4fa0bcb342ff8a41e2b7c27c349e32fda381b Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 9 Feb 2020 20:54:24 +0800 Subject: [PATCH 035/134] * Serialize CLRObject * Domain reload test --- src/embed_tests/DomainCode.cs | 151 ++++++++++++++++++ .../Python.EmbeddingTest.15.csproj | 4 + src/embed_tests/TestDomainReload.cs | 135 +++++++++------- src/runtime/clrobject.cs | 17 +- src/runtime/extensiontype.cs | 4 +- src/runtime/managedtype.cs | 19 ++- src/runtime/moduleobject.cs | 5 +- src/runtime/pyobject.cs | 4 + src/runtime/pythonengine.cs | 9 +- src/runtime/runtime.cs | 22 ++- src/runtime/runtime_state.cs | 15 +- 11 files changed, 298 insertions(+), 87 deletions(-) create mode 100644 src/embed_tests/DomainCode.cs diff --git a/src/embed_tests/DomainCode.cs b/src/embed_tests/DomainCode.cs new file mode 100644 index 000000000..0c694ce5f --- /dev/null +++ b/src/embed_tests/DomainCode.cs @@ -0,0 +1,151 @@ +using Python.Runtime; +using System; +using System.Diagnostics; +using System.Reflection; + +// +// The code we'll test. All that really matters is +// using GIL { Python.Exec(pyScript); } +// but the rest is useful for debugging. +// +// What matters in the python code is gc.collect and clr.AddReference. +// +// Note that the language version is 2.0, so no $"foo{bar}" syntax. +// +static class PythonRunner +{ + static readonly Action XIncref; + static readonly Action XDecref; + + static PythonRunner() + { + const BindingFlags flags = BindingFlags.Static | BindingFlags.NonPublic; + MethodInfo incMethod = typeof(Runtime).GetMethod("XIncref", flags); + MethodInfo decMethod = typeof(Runtime).GetMethod("XDecref", flags); + + XIncref = (Action)Delegate.CreateDelegate(typeof(Action), incMethod); + XDecref = (Action)Delegate.CreateDelegate(typeof(Action), decMethod); + } + + public static void RunPython() + { + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + string name = AppDomain.CurrentDomain.FriendlyName; + Console.WriteLine(string.Format("[{0} in .NET] In PythonRunner.RunPython", name)); + PythonEngine.Initialize(mode: ShutdownMode.Reload); + using (Py.GIL()) + { + try + { + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + } + } + PythonEngine.BeginAllowThreads(); + } + + + private static IntPtr _state; + + public static void InitPython(ShutdownMode mode) + { + PythonEngine.Initialize(mode: mode); + _state = PythonEngine.BeginAllowThreads(); + } + + public static void ShutdownPython() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.Shutdown(); + } + + public static void ShutdownPythonCompletely() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.ShutdownMode = ShutdownMode.Normal; + PythonEngine.Shutdown(); + } + + public static IntPtr GetTestObject() + { + try + { + Type type = typeof(Python.EmbeddingTest.Domain.MyClass); + string code = string.Format(@" +import clr +clr.AddReference('{0}') + +from Python.EmbeddingTest.Domain import MyClass +obj = MyClass() +obj.Method() +obj.StaticMethod() +", Assembly.GetExecutingAssembly().FullName); + + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(code); + using (PyObject obj = scope.Get("obj")) + { + Debug.Assert(obj.AsManagedObject(type).GetType() == type); + // We only needs its Python handle + XIncref(obj.Handle); + return obj.Handle; + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + } + + public static void RunTestObject(IntPtr handle) + { + using (Py.GIL()) + { + + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + } + } + } + + public static void ReleaseTestObject(IntPtr handle) + { + using (Py.GIL()) + { + XDecref(handle); + } + } + + static void OnDomainUnload(object sender, EventArgs e) + { + Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); + } +} + + +namespace Python.EmbeddingTest.Domain +{ + [Serializable] + public class MyClass + { + public void Method() { } + public static void StaticMethod() { } + } +} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index 4f6b2de46..a55f32e53 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -65,9 +65,13 @@ $(DefineConstants) + + + + diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index d987ad30b..c1b7a8d91 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -1,5 +1,9 @@ using System; using System.CodeDom.Compiler; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; using System.Reflection; using NUnit.Framework; using Python.Runtime; @@ -68,46 +72,40 @@ public static void DomainReloadAndGC() PythonEngine.Shutdown(); } - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - PythonEngine.Initialize(mode: ShutdownMode.Reload); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); - } - } - PythonEngine.BeginAllowThreads(); - } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); - } - }"; + [Test] + public static void CrossDomainObject() + { + IntPtr handle; + Type type = typeof(Proxy); + Assembly assembly = BuildAssembly("test_domain_reload"); + { + AppDomain domain = CreateDomain("test_domain_reload"); + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + theProxy.InitAssembly(assembly.Location); + theProxy.Call("InitPython", ShutdownMode.Reload); + handle = (IntPtr)theProxy.Call("GetTestObject"); + theProxy.Call("ShutdownPython"); + AppDomain.Unload(domain); + } + { + AppDomain domain = CreateDomain("test_domain_reload"); + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + theProxy.InitAssembly(assembly.Location); + theProxy.Call("InitPython", ShutdownMode.Reload); + + // handle refering a clr object created in previous domain, + // it should had been deserialized and became callable agian. + theProxy.Call("RunTestObject", handle); + theProxy.Call("ShutdownPythonCompletely"); + AppDomain.Unload(domain); + } + Assert.IsTrue(Runtime.Runtime.Py_IsInitialized() == 0); + } /// /// Build an assembly out of the source code above. @@ -119,15 +117,19 @@ static void OnDomainUnload(object sender, EventArgs e) { static Assembly BuildAssembly(string assemblyName) { var provider = CodeDomProvider.CreateProvider("CSharp"); - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); + var assemblies = from assembly in AppDomain.CurrentDomain.GetAssemblies() + where !assembly.IsDynamic + select assembly.Location; + compilerparams.ReferencedAssemblies.AddRange(assemblies.ToArray()); + compilerparams.GenerateExecutable = false; compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; + compilerparams.IncludeDebugInformation = true; compilerparams.OutputAssembly = assemblyName; - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); + var dir = Path.GetDirectoryName(new StackTrace(true).GetFrame(0).GetFileName()); + var results = provider.CompileAssemblyFromFile(compilerparams, Path.Combine(dir, "DomainCode.cs")); if (results.Errors.HasErrors) { var errors = new System.Text.StringBuilder("Compiler Errors:\n"); @@ -168,6 +170,13 @@ public void RunPython() Console.WriteLine("[Proxy] Leaving RunPython"); } + + public object Call(string methodName, params object[] args) + { + var pythonrunner = theAssembly.GetType("PythonRunner"); + var method = pythonrunner.GetMethod(methodName); + return method.Invoke(null, args); + } } /// @@ -178,26 +187,11 @@ static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) { Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); - + AppDomain domain = CreateDomain(assemblyName); // Create a Proxy object in the new domain, where we want the // assembly (and Python .NET) to reside - Type type = typeof(Proxy); System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); + Type type = typeof(Proxy); var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( type.Assembly.FullName, type.FullName); @@ -214,6 +208,7 @@ static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) try { Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); + Assert.Fail($"{theProxy} should be invlaid now"); } catch (AppDomainUnloadedException) { @@ -221,6 +216,26 @@ static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) } } + private static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = currentDomain.SetupInformation.ApplicationBase, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + return domain; + } + /// /// Resolves the assembly. Why doesn't this just work normally? /// diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 7c4330219..b004eb23b 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -3,6 +3,7 @@ namespace Python.Runtime { + [Serializable] internal class CLRObject : ManagedType { internal object inst; @@ -23,7 +24,7 @@ internal CLRObject(object ob, IntPtr tp) } } - GCHandle gc = AllocGCHandle(); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; @@ -68,5 +69,19 @@ internal static IntPtr GetInstHandle(object ob) CLRObject co = GetInstance(ob); return co.pyHandle; } + + protected override void OnSave() + { + base.OnSave(); + Runtime.XIncref(pyHandle); + } + + protected override void OnLoad() + { + base.OnLoad(); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + Runtime.XDecref(pyHandle); + } } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 006d616b2..fc839644e 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -29,7 +29,7 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - GCHandle gc = AllocGCHandle(true); + GCHandle gc = AllocGCHandle(TrackTypes.Extension); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); // We have to support gc because the type machinery makes it very @@ -107,7 +107,7 @@ protected override void OnSave() protected override void OnLoad() { base.OnLoad(); - GCHandle gc = AllocGCHandle(true); + GCHandle gc = AllocGCHandle(TrackTypes.Extension); Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); Runtime.PyObject_GC_UnTrack(pyHandle); } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index cd735898e..99e897570 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -14,13 +14,20 @@ namespace Python.Runtime [Serializable] internal abstract class ManagedType { + internal enum TrackTypes + { + Untrack, + Extension, + Wrapper, + } + [NonSerialized] internal GCHandle gcHandle; // Native handle internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * - private static readonly HashSet _managedObjs = new HashSet(); + private static readonly Dictionary _managedObjs = new Dictionary(); internal void IncrRefCount() { @@ -48,12 +55,12 @@ internal long RefCount } } - internal GCHandle AllocGCHandle(bool track = false) + internal GCHandle AllocGCHandle(TrackTypes track = TrackTypes.Untrack) { gcHandle = GCHandle.Alloc(this); - if (track) + if (track != TrackTypes.Untrack) { - _managedObjs.Add(this); + _managedObjs.Add(this, track); } return gcHandle; } @@ -64,7 +71,7 @@ internal void FreeGCHandle() if (gcHandle.IsAllocated) { gcHandle.Free(); - gcHandle = new GCHandle(); + gcHandle = default; } } @@ -129,7 +136,7 @@ internal static bool IsManagedType(IntPtr ob) return false; } - internal static ICollection GetManagedObjects() + internal static IDictionary GetManagedObjects() { return _managedObjs; } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 0d42914e5..8661d74ae 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -345,15 +345,16 @@ public static int tp_clear(IntPtr ob) protected override void OnSave() { base.OnSave(); + System.Diagnostics.Debug.Assert(dict == GetObjectDict(pyHandle)); + // Decref twice in tp_clear, equilibrate them. + Runtime.XIncref(dict); Runtime.XIncref(dict); - Runtime.XIncref(GetObjectDict(pyHandle)); } protected override void OnLoad() { base.OnLoad(); cache = new Dictionary(); - Runtime.XIncref(dict); SetObjectDict(pyHandle, dict); } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 8ae99ecd0..58359e5c5 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -20,6 +20,7 @@ public interface IPyDisposable : IDisposable /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// + //[Serializable] public class PyObject : DynamicObject, IEnumerable, IPyDisposable { #if TRACE_ALLOC @@ -30,7 +31,10 @@ public class PyObject : DynamicObject, IEnumerable, IPyDisposable #endif protected internal IntPtr obj = IntPtr.Zero; + + [NonSerialized] private bool disposed = false; + [NonSerialized] private bool _finalized = false; /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 301eea44e..044377a6b 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,7 +12,12 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { - public static ShutdownMode ShutdownMode => Runtime.ShutdownMode; + public static ShutdownMode ShutdownMode + { + get => Runtime.ShutdownMode; + set => Runtime.ShutdownMode = value; + } + private static DelegateManager delegateManager; private static bool initialized; @@ -187,7 +192,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; // Remember to shut down the runtime. - AddShutdownHandler(() => Runtime.Shutdown(mode)); + AddShutdownHandler(Runtime.Shutdown); // The global scope gets used implicitly quite early on, remember // to clear it out when we shut down. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 063ed6d99..9e6eef50d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -126,7 +126,7 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - public static ShutdownMode ShutdownMode { get; private set; } + public static ShutdownMode ShutdownMode { get; internal set; } private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); /// @@ -321,6 +321,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) { RuntimeData.StashPop(); + RuntimeData.ClearStash(); } else #endif @@ -341,7 +342,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } - internal static void Shutdown(ShutdownMode mode = ShutdownMode.Normal) + internal static void Shutdown() { if (Py_IsInitialized() == 0 || !_isInitialized) { @@ -351,6 +352,7 @@ internal static void Shutdown(ShutdownMode mode = ShutdownMode.Normal) PyGILState_Ensure(); + var mode = ShutdownMode; #if !NETSTANDARD if (mode == ShutdownMode.Reload) { @@ -468,16 +470,20 @@ private static void MoveClrInstancesOnwershipToPython() { var copyObjs = ManagedType.GetManagedObjects().ToArray(); var objs = ManagedType.GetManagedObjects(); - foreach (var obj in copyObjs) + foreach (var entry in copyObjs) { - if (!objs.Contains(obj)) + ManagedType obj = entry.Key; + if (!objs.ContainsKey(obj)) { continue; } - obj.CallTypeClear(); - // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), - // thus just be safe to give it back to GC chain. - PyObject_GC_Track(obj.pyHandle); + if (entry.Value == ManagedType.TrackTypes.Extension) + { + obj.CallTypeClear(); + // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), + // thus just be safe to give it back to GC chain. + PyObject_GC_Track(obj.pyHandle); + } if (obj.gcHandle.IsAllocated) { obj.gcHandle.Free(); diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index d7f6f5558..ae01169ab 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -249,7 +249,7 @@ internal static void Stash() TypeManager.StashPush(stack); ClassManager.StashPush(stack); var objs = ManagedType.GetManagedObjects(); - foreach (var obj in objs) + foreach (var obj in objs.Keys) { obj.Save(); } @@ -291,14 +291,13 @@ internal static void StashPop() var stack = (Stack)formatter.Deserialize(ms); - var loadObjs = (ICollection)stack.Pop(); - var objs = ManagedType.GetManagedObjects(); - foreach (var obj in loadObjs) + var loadObjs = (IDictionary)stack.Pop(); + foreach (var entry in loadObjs) { + ManagedType obj = entry.Key; obj.Load(); - objs.Add(obj); } - + Debug.Assert(ManagedType.GetManagedObjects().Count == loadObjs.Count); ClassManager.StashPop(stack); TypeManager.StashPop(stack); PyCLRMetaType = MetaType.StashPop(stack); @@ -309,5 +308,9 @@ public static bool HasStashData() return PySys_GetObject("clr_data") != IntPtr.Zero; } + public static void ClearStash() + { + PySys_SetObject("clr_data", IntPtr.Zero); + } } } From 9874cd123393900364bf0f0f1c1a20af5dbf7763 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 10 Feb 2020 03:07:45 +0800 Subject: [PATCH 036/134] Add ShutdownMode.Default refer to normal mode if no EnvironmentVariable control exists --- src/embed_tests/DomainCode.cs | 18 +++++++++++++----- src/embed_tests/TestRuntime.cs | 4 ++++ src/runtime/pythonengine.cs | 2 +- src/runtime/runtime.cs | 27 +++++++++++++++++---------- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/embed_tests/DomainCode.cs b/src/embed_tests/DomainCode.cs index 0c694ce5f..48f4011d4 100644 --- a/src/embed_tests/DomainCode.cs +++ b/src/embed_tests/DomainCode.cs @@ -114,15 +114,23 @@ from Python.EmbeddingTest.Domain import MyClass public static void RunTestObject(IntPtr handle) { - using (Py.GIL()) + try { - - using (PyObject obj = new PyObject(handle)) + using (Py.GIL()) { - obj.InvokeMethod("Method"); - obj.InvokeMethod("StaticMethod"); + + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + } } } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } } public static void ReleaseTestObject(IntPtr handle) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index fd02b4a82..d4f4b1c98 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -39,6 +39,10 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { + if (Runtime.Runtime.Py_IsInitialized() == 1) + { + Runtime.Runtime.PyGILState_Ensure(); + } Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 044377a6b..05a31ff68 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -172,7 +172,7 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false, Shu /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Normal) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { if (initialized) { diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9e6eef50d..922e13b46 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -132,7 +132,7 @@ public class Runtime /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Normal) + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { if (_isInitialized) { @@ -140,17 +140,23 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } _isInitialized = true; - if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") + if (mode == ShutdownMode.Default) { - mode = ShutdownMode.Soft; - } -#if !NETSTANDARD - else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") - { - mode = ShutdownMode.Reload; - } + if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") + { + mode = ShutdownMode.Soft; + } + #if !NETSTANDARD + else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") + { + mode = ShutdownMode.Reload; + } #endif - //mode = ShutdownMode.Reload; + else + { + mode = ShutdownMode.Normal; + } + } ShutdownMode = mode; if (Py_IsInitialized() == 0) @@ -2196,6 +2202,7 @@ internal static IntPtr GetBuiltins() public enum ShutdownMode { + Default, Normal, Soft, #if !NETSTANDARD From 8da561b232456f9567ea7ab13ea89bab3f4586ae Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 10 Feb 2020 03:20:48 +0800 Subject: [PATCH 037/134] Not generating the debug info for generated modules --- src/embed_tests/TestDomainReload.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index c1b7a8d91..4c2756d59 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -119,13 +119,13 @@ static Assembly BuildAssembly(string assemblyName) var provider = CodeDomProvider.CreateProvider("CSharp"); var compilerparams = new CompilerParameters(); var assemblies = from assembly in AppDomain.CurrentDomain.GetAssemblies() - where !assembly.IsDynamic + where !assembly.IsDynamic && !string.IsNullOrEmpty(assembly.Location) select assembly.Location; compilerparams.ReferencedAssemblies.AddRange(assemblies.ToArray()); compilerparams.GenerateExecutable = false; compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = true; + compilerparams.IncludeDebugInformation = false; compilerparams.OutputAssembly = assemblyName; var dir = Path.GetDirectoryName(new StackTrace(true).GetFrame(0).GetFileName()); From 9499c644af4f6111c24e19b3d378155d54e07295 Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 11 Feb 2020 02:32:42 +0800 Subject: [PATCH 038/134] `TestDomainReload` use itself assembly instead of dynamic creation --- src/embed_tests/DomainCode.cs | 159 ----------------------- src/embed_tests/TestDomainReload.cs | 194 ++++++++++++++++++++++------ 2 files changed, 157 insertions(+), 196 deletions(-) delete mode 100644 src/embed_tests/DomainCode.cs diff --git a/src/embed_tests/DomainCode.cs b/src/embed_tests/DomainCode.cs deleted file mode 100644 index 48f4011d4..000000000 --- a/src/embed_tests/DomainCode.cs +++ /dev/null @@ -1,159 +0,0 @@ -using Python.Runtime; -using System; -using System.Diagnostics; -using System.Reflection; - -// -// The code we'll test. All that really matters is -// using GIL { Python.Exec(pyScript); } -// but the rest is useful for debugging. -// -// What matters in the python code is gc.collect and clr.AddReference. -// -// Note that the language version is 2.0, so no $"foo{bar}" syntax. -// -static class PythonRunner -{ - static readonly Action XIncref; - static readonly Action XDecref; - - static PythonRunner() - { - const BindingFlags flags = BindingFlags.Static | BindingFlags.NonPublic; - MethodInfo incMethod = typeof(Runtime).GetMethod("XIncref", flags); - MethodInfo decMethod = typeof(Runtime).GetMethod("XDecref", flags); - - XIncref = (Action)Delegate.CreateDelegate(typeof(Action), incMethod); - XDecref = (Action)Delegate.CreateDelegate(typeof(Action), decMethod); - } - - public static void RunPython() - { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format("[{0} in .NET] In PythonRunner.RunPython", name)); - PythonEngine.Initialize(mode: ShutdownMode.Reload); - using (Py.GIL()) - { - try - { - var pyScript = string.Format("import clr\n" - + "print('[{0} in python] imported clr')\n" - + "clr.AddReference('System')\n" - + "print('[{0} in python] allocated a clr object')\n" - + "import gc\n" - + "gc.collect()\n" - + "print('[{0} in python] collected garbage')\n", - name); - PythonEngine.Exec(pyScript); - } - catch (Exception e) - { - Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); - } - } - PythonEngine.BeginAllowThreads(); - } - - - private static IntPtr _state; - - public static void InitPython(ShutdownMode mode) - { - PythonEngine.Initialize(mode: mode); - _state = PythonEngine.BeginAllowThreads(); - } - - public static void ShutdownPython() - { - PythonEngine.EndAllowThreads(_state); - PythonEngine.Shutdown(); - } - - public static void ShutdownPythonCompletely() - { - PythonEngine.EndAllowThreads(_state); - PythonEngine.ShutdownMode = ShutdownMode.Normal; - PythonEngine.Shutdown(); - } - - public static IntPtr GetTestObject() - { - try - { - Type type = typeof(Python.EmbeddingTest.Domain.MyClass); - string code = string.Format(@" -import clr -clr.AddReference('{0}') - -from Python.EmbeddingTest.Domain import MyClass -obj = MyClass() -obj.Method() -obj.StaticMethod() -", Assembly.GetExecutingAssembly().FullName); - - using (Py.GIL()) - using (var scope = Py.CreateScope()) - { - scope.Exec(code); - using (PyObject obj = scope.Get("obj")) - { - Debug.Assert(obj.AsManagedObject(type).GetType() == type); - // We only needs its Python handle - XIncref(obj.Handle); - return obj.Handle; - } - } - } - catch (Exception e) - { - Debug.WriteLine(e); - throw; - } - } - - public static void RunTestObject(IntPtr handle) - { - try - { - using (Py.GIL()) - { - - using (PyObject obj = new PyObject(handle)) - { - obj.InvokeMethod("Method"); - obj.InvokeMethod("StaticMethod"); - } - } - } - catch (Exception e) - { - Debug.WriteLine(e); - throw; - } - } - - public static void ReleaseTestObject(IntPtr handle) - { - using (Py.GIL()) - { - XDecref(handle); - } - } - - static void OnDomainUnload(object sender, EventArgs e) - { - Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); - } -} - - -namespace Python.EmbeddingTest.Domain -{ - [Serializable] - public class MyClass - { - public void Method() { } - public static void StaticMethod() { } - } -} diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 4c2756d59..044c90bf1 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -54,22 +54,12 @@ class TestDomainReload [Test] public static void DomainReloadAndGC() { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); + RunAssemblyAndUnload("test1"); Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, "On soft-shutdown mode, Python runtime should still running"); - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - - PythonEngine.Shutdown(); + RunAssemblyAndUnload("test2"); } [Test] @@ -77,13 +67,11 @@ public static void CrossDomainObject() { IntPtr handle; Type type = typeof(Proxy); - Assembly assembly = BuildAssembly("test_domain_reload"); { AppDomain domain = CreateDomain("test_domain_reload"); var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( type.Assembly.FullName, type.FullName); - theProxy.InitAssembly(assembly.Location); theProxy.Call("InitPython", ShutdownMode.Reload); handle = (IntPtr)theProxy.Call("GetTestObject"); theProxy.Call("ShutdownPython"); @@ -95,7 +83,6 @@ public static void CrossDomainObject() var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( type.Assembly.FullName, type.FullName); - theProxy.InitAssembly(assembly.Location); theProxy.Call("InitPython", ShutdownMode.Reload); // handle refering a clr object created in previous domain, @@ -129,7 +116,8 @@ static Assembly BuildAssembly(string assemblyName) compilerparams.OutputAssembly = assemblyName; var dir = Path.GetDirectoryName(new StackTrace(true).GetFrame(0).GetFileName()); - var results = provider.CompileAssemblyFromFile(compilerparams, Path.Combine(dir, "DomainCode.cs")); + string DomainCodePath = Path.Combine(dir, "DomainCode.cs"); + var results = provider.CompileAssemblyFromFile(compilerparams, DomainCodePath); if (results.Errors.HasErrors) { var errors = new System.Text.StringBuilder("Compiler Errors:\n"); @@ -152,28 +140,16 @@ static Assembly BuildAssembly(string assemblyName) /// class Proxy : MarshalByRefObject { - Assembly theAssembly = null; - - public void InitAssembly(string assemblyPath) - { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); - } - public void RunPython() { Console.WriteLine("[Proxy] Entering RunPython"); - - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); - + PythonRunner.RunPython(); Console.WriteLine("[Proxy] Leaving RunPython"); } public object Call(string methodName, params object[] args) { - var pythonrunner = theAssembly.GetType("PythonRunner"); + var pythonrunner = typeof(PythonRunner); var method = pythonrunner.GetMethod(methodName); return method.Invoke(null, args); } @@ -183,26 +159,24 @@ public object Call(string methodName, params object[] args) /// Create a domain, run the assembly in it (the RunPython function), /// and unload the domain. /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) + static void RunAssemblyAndUnload(string domainName) { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); + Console.WriteLine($"[Program.Main] === creating domain {domainName}"); - AppDomain domain = CreateDomain(assemblyName); + AppDomain domain = CreateDomain(domainName); // Create a Proxy object in the new domain, where we want the // assembly (and Python .NET) to reside - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); Type type = typeof(Proxy); var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( type.Assembly.FullName, type.FullName); // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); theProxy.RunPython(); - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); + Console.WriteLine($"[Program.Main] Before Domain Unload on {domainName}"); AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); + Console.WriteLine($"[Program.Main] After Domain Unload on {domainName}"); // Validate that the assembly does not exist anymore try @@ -254,5 +228,151 @@ static Assembly ResolveAssembly(object sender, ResolveEventArgs args) return null; } } + + + // + // The code we'll test. All that really matters is + // using GIL { Python.Exec(pyScript); } + // but the rest is useful for debugging. + // + // What matters in the python code is gc.collect and clr.AddReference. + // + // Note that the language version is 2.0, so no $"foo{bar}" syntax. + // + static class PythonRunner + { + public static void RunPython() + { + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + string name = AppDomain.CurrentDomain.FriendlyName; + Console.WriteLine(string.Format("[{0} in .NET] In PythonRunner.RunPython", name)); + PythonEngine.Initialize(mode: ShutdownMode.Reload); + using (Py.GIL()) + { + try + { + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + } + } + PythonEngine.BeginAllowThreads(); + } + + + private static IntPtr _state; + + public static void InitPython(ShutdownMode mode) + { + PythonEngine.Initialize(mode: mode); + _state = PythonEngine.BeginAllowThreads(); + } + + public static void ShutdownPython() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.Shutdown(); + } + + public static void ShutdownPythonCompletely() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.ShutdownMode = ShutdownMode.Normal; + PythonEngine.Shutdown(); + } + + public static IntPtr GetTestObject() + { + try + { + Type type = typeof(Python.EmbeddingTest.Domain.MyClass); + string code = string.Format(@" +import clr +clr.AddReference('{0}') + +from Python.EmbeddingTest.Domain import MyClass +obj = MyClass() +obj.Method() +obj.StaticMethod() +", Assembly.GetExecutingAssembly().FullName); + + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(code); + using (PyObject obj = scope.Get("obj")) + { + Debug.Assert(obj.AsManagedObject(type).GetType() == type); + // We only needs its Python handle + Runtime.Runtime.XIncref(obj.Handle); + return obj.Handle; + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + } + + public static void RunTestObject(IntPtr handle) + { + try + { + using (Py.GIL()) + { + + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + } + + public static void ReleaseTestObject(IntPtr handle) + { + using (Py.GIL()) + { + Runtime.Runtime.XDecref(handle); + } + } + + static void OnDomainUnload(object sender, EventArgs e) + { + Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); + } + } + } + + +namespace Python.EmbeddingTest.Domain +{ + [Serializable] + public class MyClass + { + public void Method() { } + public static void StaticMethod() { } + } +} + + #endif From 2b84394f00d0db083ea82b8ca6d1831d8e9e9b81 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 01:16:20 +0800 Subject: [PATCH 039/134] * Fix refcnt error * Pick `SlotHelper` from #958 --- src/runtime/runtime.cs | 27 +++------------------------ src/runtime/typemanager.cs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5ead02378..7da70ddcd 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -340,30 +340,9 @@ internal static void Initialize(bool initSigs = false) private static IntPtr Get_PyObject_NextNotImplemented() { - IntPtr globals = PyDict_New(); - if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0) - { - XDecref(globals); - throw new PythonException(); - } - const string code = "class A(object): pass"; - IntPtr res = PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); - if (res == IntPtr.Zero) - { - try - { - throw new PythonException(); - } - finally - { - XDecref(globals); - } - } - XDecref(res); - IntPtr A = PyDict_GetItemString(globals, "A"); - IntPtr iternext = Marshal.ReadIntPtr(A, TypeOffset.tp_iternext); - XDecref(globals); - XDecref(A); + IntPtr pyType = SlotHelper.CreateObjectType(); + IntPtr iternext = Marshal.ReadIntPtr(pyType, TypeOffset.tp_iternext); + Runtime.XDecref(pyType); return iternext; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index bb920b74f..c541f9dc2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -820,4 +821,37 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) Marshal.WriteIntPtr(to, offset, fp); } } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = Runtime.PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + Runtime.XDecref(res); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } + } } From 5ade069d280c69986787ee118080ae7e3e7441b3 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 01:56:56 +0800 Subject: [PATCH 040/134] Validate return value --- src/runtime/platform/NativeCodePage.cs | 53 ++++++++++++++++---------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs index 3f89e68ab..f4b595706 100644 --- a/src/runtime/platform/NativeCodePage.cs +++ b/src/runtime/platform/NativeCodePage.cs @@ -244,26 +244,39 @@ public void SetReadExec(IntPtr mappedMemory, int numBytes) /// public static void InitializePlatformData() { - IntPtr op; - IntPtr fn; - IntPtr platformModule = Runtime.PyImport_ImportModule("platform"); - IntPtr emptyTuple = Runtime.PyTuple_New(0); - - fn = Runtime.PyObject_GetAttrString(platformModule, "system"); - op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); - PythonException.ThrowIfIsNull(op); - OperatingSystemName = Runtime.GetManagedString(op); - Runtime.XDecref(op); - Runtime.XDecref(fn); - - fn = Runtime.PyObject_GetAttrString(platformModule, "machine"); - op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = Runtime.GetManagedString(op); - Runtime.XDecref(op); - Runtime.XDecref(fn); - - Runtime.XDecref(emptyTuple); - Runtime.XDecref(platformModule); + IntPtr op = IntPtr.Zero; + IntPtr fn = IntPtr.Zero; + IntPtr platformModule = IntPtr.Zero; + IntPtr emptyTuple = IntPtr.Zero; + try + { + platformModule = Runtime.PyImport_ImportModule("platform"); + PythonException.ThrowIfIsNull(platformModule); + + fn = Runtime.PyObject_GetAttrString(platformModule, "system"); + PythonException.ThrowIfIsNull(fn); + + emptyTuple = Runtime.PyTuple_New(0); + op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); + PythonException.ThrowIfIsNull(op); + + OperatingSystemName = Runtime.GetManagedString(op); + + fn = Runtime.PyObject_GetAttrString(platformModule, "machine"); + PythonException.ThrowIfIsNull(fn); + + op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); + PythonException.ThrowIfIsNull(op); + MachineName = Runtime.GetManagedString(op); + } + finally + { + Runtime.XDecref(op); + Runtime.XDecref(fn); + + Runtime.XDecref(emptyTuple); + Runtime.XDecref(platformModule); + } // Now convert the strings into enum values so we can do switch // statements rather than constant parsing. From f4bb77a1d4c5d8386816203506e908450c90ae52 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 11:26:00 +0800 Subject: [PATCH 041/134] * API for getting the default shutdown mode * Make the reload test won't bother other tests * Clear obsolete code --- .../Python.EmbeddingTest.15.csproj | 4 - src/embed_tests/TestDomainReload.cs | 107 ++++++++---------- src/runtime/pythonengine.cs | 5 +- src/runtime/runtime.cs | 34 +++--- 4 files changed, 67 insertions(+), 83 deletions(-) diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a55f32e53..4f6b2de46 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -65,13 +65,9 @@ $(DefineConstants) - - - - diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 044c90bf1..96a1d4fce 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -1,9 +1,5 @@ using System; -using System.CodeDom.Compiler; -using System.Collections.Generic; using System.Diagnostics; -using System.IO; -using System.Linq; using System.Reflection; using NUnit.Framework; using Python.Runtime; @@ -54,6 +50,7 @@ class TestDomainReload [Test] public static void DomainReloadAndGC() { + Assert.IsFalse(PythonEngine.IsInitialized); RunAssemblyAndUnload("test1"); Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, @@ -65,72 +62,47 @@ public static void DomainReloadAndGC() [Test] public static void CrossDomainObject() { - IntPtr handle; + IntPtr handle = IntPtr.Zero; Type type = typeof(Proxy); { AppDomain domain = CreateDomain("test_domain_reload"); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - theProxy.Call("InitPython", ShutdownMode.Reload); - handle = (IntPtr)theProxy.Call("GetTestObject"); - theProxy.Call("ShutdownPython"); - AppDomain.Unload(domain); + try + { + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + theProxy.Call("InitPython", ShutdownMode.Reload); + handle = (IntPtr)theProxy.Call("GetTestObject"); + theProxy.Call("ShutdownPython"); + } + finally + { + AppDomain.Unload(domain); + } } { AppDomain domain = CreateDomain("test_domain_reload"); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - theProxy.Call("InitPython", ShutdownMode.Reload); - - // handle refering a clr object created in previous domain, - // it should had been deserialized and became callable agian. - theProxy.Call("RunTestObject", handle); - theProxy.Call("ShutdownPythonCompletely"); - AppDomain.Unload(domain); - } - Assert.IsTrue(Runtime.Runtime.Py_IsInitialized() == 0); - } - - /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. - /// - static Assembly BuildAssembly(string assemblyName) - { - var provider = CodeDomProvider.CreateProvider("CSharp"); - var compilerparams = new CompilerParameters(); - var assemblies = from assembly in AppDomain.CurrentDomain.GetAssemblies() - where !assembly.IsDynamic && !string.IsNullOrEmpty(assembly.Location) - select assembly.Location; - compilerparams.ReferencedAssemblies.AddRange(assemblies.ToArray()); - - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; - - var dir = Path.GetDirectoryName(new StackTrace(true).GetFrame(0).GetFileName()); - string DomainCodePath = Path.Combine(dir, "DomainCode.cs"); - var results = provider.CompileAssemblyFromFile(compilerparams, DomainCodePath); - if (results.Errors.HasErrors) - { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) + try + { + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + theProxy.Call("InitPython", ShutdownMode.Reload); + + // handle refering a clr object created in previous domain, + // it should had been deserialized and became callable agian. + theProxy.Call("RunTestObject", handle); + theProxy.Call("ShutdownPythonCompletely"); + } + finally { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); + AppDomain.Unload(domain); } - throw new Exception(errors.ToString()); } - else + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) { - return results.CompiledAssembly; + Assert.IsTrue(Runtime.Runtime.Py_IsInitialized() == 0); } } @@ -246,7 +218,7 @@ public static void RunPython() AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; Console.WriteLine(string.Format("[{0} in .NET] In PythonRunner.RunPython", name)); - PythonEngine.Initialize(mode: ShutdownMode.Reload); + PythonEngine.Initialize(); using (Py.GIL()) { try @@ -287,8 +259,21 @@ public static void ShutdownPython() public static void ShutdownPythonCompletely() { PythonEngine.EndAllowThreads(_state); - PythonEngine.ShutdownMode = ShutdownMode.Normal; + // XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`, + // if it used a another mode(the default mode) in other tests, + // when other tests trying to access these reserved objects, it may cause Domain exception, + // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python. + if (PythonEngine.DefaultShutdownMode != ShutdownMode.Reload) + { + PythonEngine.ShutdownMode = ShutdownMode.Soft; + } PythonEngine.Shutdown(); + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + // Normal mode will shutdown the VM, so it needs to be shutdown + // for avoiding influence with other tests. + Runtime.Runtime.Shutdown(); + } } public static IntPtr GetTestObject() diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 05a31ff68..b18ce56c6 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -17,7 +17,8 @@ public static ShutdownMode ShutdownMode get => Runtime.ShutdownMode; set => Runtime.ShutdownMode = value; } - + + public static ShutdownMode DefaultShutdownMode => Runtime.GetDefaultShutdownMode(); private static DelegateManager delegateManager; private static bool initialized; @@ -157,7 +158,7 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Normal) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 6b70711fe..4b6f37ea8 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -142,23 +142,10 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd if (mode == ShutdownMode.Default) { - if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") - { - mode = ShutdownMode.Soft; - } - #if !NETSTANDARD - else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") - { - mode = ShutdownMode.Reload; - } -#endif - else - { - mode = ShutdownMode.Normal; - } + mode = GetDefaultShutdownMode(); } - ShutdownMode = mode; + if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); @@ -408,6 +395,21 @@ internal static void Shutdown() Py_Finalize(); } + internal static ShutdownMode GetDefaultShutdownMode() + { + if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") + { + return ShutdownMode.Soft; + } +#if !NETSTANDARD + else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") + { + return ShutdownMode.Reload; + } +#endif + return ShutdownMode.Normal; + } + // called *without* the GIL acquired by clr._AtExit internal static int AtExit() { @@ -1984,7 +1986,7 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type¡¯s base class. Return 0 on success, or return -1 and sets an exception on error. + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); From 670bd745be9f13dd2e8fff044b2691641e15dd37 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 11:36:56 +0800 Subject: [PATCH 042/134] In domain test, use soft mode if default mode is normal --- src/embed_tests/TestDomainReload.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 96a1d4fce..1637a0b1a 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -218,7 +218,12 @@ public static void RunPython() AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; Console.WriteLine(string.Format("[{0} in .NET] In PythonRunner.RunPython", name)); - PythonEngine.Initialize(); + var mode = PythonEngine.DefaultShutdownMode; + if (mode == ShutdownMode.Normal) + { + mode = ShutdownMode.Soft; + } + PythonEngine.Initialize(mode: mode); using (Py.GIL()) { try From 3cb56f16bdd6dc978520f630714bf3f24474132a Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 18:11:18 +0800 Subject: [PATCH 043/134] Avoid Domain tests influence other tests when the default is not Soft or Reload --- src/embed_tests/TestDomainReload.cs | 22 +++++++++++++--------- src/runtime/runtime.cs | 10 ++++++++-- src/runtime/runtime_state.cs | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 1637a0b1a..c94c8599c 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -52,11 +52,20 @@ public static void DomainReloadAndGC() { Assert.IsFalse(PythonEngine.IsInitialized); RunAssemblyAndUnload("test1"); - Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, "On soft-shutdown mode, Python runtime should still running"); RunAssemblyAndUnload("test2"); + Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + // The default mode is a normal mode, + // it should shutdown the Python VM avoiding influence other tests. + Runtime.Runtime.PyGILState_Ensure(); + Runtime.Runtime.Py_Finalize(); + } } [Test] @@ -268,17 +277,12 @@ public static void ShutdownPythonCompletely() // if it used a another mode(the default mode) in other tests, // when other tests trying to access these reserved objects, it may cause Domain exception, // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python. - if (PythonEngine.DefaultShutdownMode != ShutdownMode.Reload) + var defaultMode = PythonEngine.DefaultShutdownMode; + if (defaultMode != ShutdownMode.Reload) { - PythonEngine.ShutdownMode = ShutdownMode.Soft; + PythonEngine.ShutdownMode = defaultMode; } PythonEngine.Shutdown(); - if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) - { - // Normal mode will shutdown the VM, so it needs to be shutdown - // for avoiding influence with other tests. - Runtime.Runtime.Shutdown(); - } } public static IntPtr GetTestObject() diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 4b6f37ea8..8a9453dd3 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -6,8 +6,6 @@ using System.Collections.Generic; using Python.Runtime.Platform; using System.Linq; -using System.IO; -using System.Runtime.Serialization.Formatters.Binary; namespace Python.Runtime { @@ -157,6 +155,14 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd { RuntimeState.Save(); } +#if !NETSTANDARD + // XXX: Reload mode may reduct to Soft mode, + // so even on Reload mode it still needs to save the RuntimeState + else if (mode == ShutdownMode.Reload) + { + RuntimeState.Save(); + } +#endif MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } else diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index ae01169ab..0ee192843 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -98,7 +98,7 @@ public static void Restore() private static void ResotreModules(IntPtr dummyGC) { var intialModules = PySys_GetObject("initial_modules"); - Debug.Assert(intialModules != null); + Debug.Assert(intialModules != IntPtr.Zero); var modules = PyImport_GetModuleDict(); foreach (var name in GetModuleNames()) { From 3c9a83c13a839ca66723ef14039f9184b0d0e532 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 13 Feb 2020 21:24:56 +0800 Subject: [PATCH 044/134] Skip non-serializable objects --- src/runtime/runtime_state.cs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 0ee192843..bf7850d01 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -249,16 +249,27 @@ internal static void Stash() TypeManager.StashPush(stack); ClassManager.StashPush(stack); var objs = ManagedType.GetManagedObjects(); - foreach (var obj in objs.Keys) + var saveObjs = new Dictionary(); + foreach (var entry in objs) { + var obj = entry.Key; + if (entry.Value == ManagedType.TrackTypes.Wrapper + && !obj.GetType().IsSerializable) + { + // XXX: Skip non-serializable objects, + // use them after next initialization will raise exceptions. + continue; + } + Debug.Assert(obj.GetType().IsSerializable); obj.Save(); + saveObjs.Add(entry.Key, entry.Value); } - stack.Push(objs); + stack.Push(saveObjs); formatter.Serialize(ms, stack); byte[] data = ms.GetBuffer(); // TODO: use buffer api instead - System.Diagnostics.Debug.Assert(data.Length <= int.MaxValue); + Debug.Assert(data.Length <= int.MaxValue); IntPtr mem = PyMem_Malloc(data.LongLength + IntPtr.Size); Marshal.WriteIntPtr(mem, (IntPtr)data.LongLength); Marshal.Copy(data, 0, mem + IntPtr.Size, data.Length); From df84e290fef15f289149231fa35dd47eac511e47 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 17 Feb 2020 10:46:59 +0800 Subject: [PATCH 045/134] * Reset Exceptions to IntPtr.Zero * Cleanup the code --- src/runtime/exceptions.cs | 6 ++++-- src/runtime/interop.cs | 13 +++---------- src/runtime/runtime.cs | 5 +++-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index c1edaced2..908de9745 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -140,10 +140,12 @@ internal static void Shutdown() foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) { var op = (IntPtr)fi.GetValue(type); - if (op != IntPtr.Zero) + if (op == IntPtr.Zero) { - Runtime.XDecref(op); + continue; } + Runtime.XDecref(op); + fi.SetValue(null, IntPtr.Zero); } Runtime.Py_CLEAR(ref exceptions_module); Runtime.Py_CLEAR(ref warnings_module); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 58464ec7f..58ec82415 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -499,16 +499,9 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { return ThunkInfo.Empty; } - try - { - Delegate d = Delegate.CreateDelegate(dt, method); - var info = new ThunkInfo(d); - return info; - } - catch (Exception) - { - throw; - } + Delegate d = Delegate.CreateDelegate(dt, method); + var info = new ThunkInfo(d); + return info; } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 8a9453dd3..d61ac7b34 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -358,9 +358,7 @@ internal static void Shutdown() } #endif AssemblyManager.Shutdown(); - Exceptions.Shutdown(); ImportHook.Shutdown(); - Finalizer.Shutdown(); #if !NETSTANDARD if (mode != ShutdownMode.Reload) @@ -377,6 +375,9 @@ internal static void Shutdown() MetaType.Release(); PyCLRMetaType = IntPtr.Zero; + Exceptions.Shutdown(); + Finalizer.Shutdown(); + if (mode != ShutdownMode.Normal) { PyGC_Collect(); From 8b516210a5088f730b8fa192dd19db5b0f03b9eb Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 17 Feb 2020 11:30:27 +0800 Subject: [PATCH 046/134] Split RuntimeData into separate file --- src/runtime/Python.Runtime.csproj | 1 + src/runtime/runtime_data.cs | 99 +++++++++++++++++++++++++++++++ src/runtime/runtime_state.cs | 91 ---------------------------- 3 files changed, 100 insertions(+), 91 deletions(-) create mode 100644 src/runtime/runtime_data.cs diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index a00f37440..73f352076 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -137,6 +137,7 @@ + diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs new file mode 100644 index 000000000..420837be8 --- /dev/null +++ b/src/runtime/runtime_data.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using System.Runtime.Serialization.Formatters.Binary; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + class RuntimeData + { + internal static void Stash() + { + var formatter = new BinaryFormatter(); + var ms = new MemoryStream(); + var stack = new Stack(); + MetaType.StashPush(stack); + TypeManager.StashPush(stack); + ClassManager.StashPush(stack); + var objs = ManagedType.GetManagedObjects(); + var saveObjs = new Dictionary(); + foreach (var entry in objs) + { + var obj = entry.Key; + if (entry.Value == ManagedType.TrackTypes.Wrapper + && !obj.GetType().IsSerializable) + { + // XXX: Skip non-serializable objects, + // use them after next initialization will raise exceptions. + continue; + } + Debug.Assert(obj.GetType().IsSerializable); + obj.Save(); + saveObjs.Add(entry.Key, entry.Value); + } + stack.Push(saveObjs); + formatter.Serialize(ms, stack); + + byte[] data = ms.GetBuffer(); + // TODO: use buffer api instead + Debug.Assert(data.Length <= int.MaxValue); + IntPtr mem = PyMem_Malloc(data.LongLength + IntPtr.Size); + Marshal.WriteIntPtr(mem, (IntPtr)data.LongLength); + Marshal.Copy(data, 0, mem + IntPtr.Size, data.Length); + + IntPtr capsule = PySys_GetObject("clr_data"); + if (capsule != IntPtr.Zero) + { + IntPtr oldData = PyCapsule_GetPointer(capsule, null); + PyMem_Free(oldData); + PyCapsule_SetPointer(capsule, IntPtr.Zero); + } + capsule = PyCapsule_New(mem, null, IntPtr.Zero); + PySys_SetObject("clr_data", capsule); + XDecref(capsule); + } + + internal static void StashPop() + { + IntPtr capsule = PySys_GetObject("clr_data"); + if (capsule == IntPtr.Zero) + { + return; + } + IntPtr mem = PyCapsule_GetPointer(capsule, null); + int length = (int)Marshal.ReadIntPtr(mem); + byte[] data = new byte[length]; + Marshal.Copy(mem + IntPtr.Size, data, 0, length); + var ms = new MemoryStream(data); + var formatter = new BinaryFormatter(); + + var stack = (Stack)formatter.Deserialize(ms); + + var loadObjs = (IDictionary)stack.Pop(); + foreach (var entry in loadObjs) + { + ManagedType obj = entry.Key; + obj.Load(); + } + Debug.Assert(ManagedType.GetManagedObjects().Count == loadObjs.Count); + ClassManager.StashPop(stack); + TypeManager.StashPop(stack); + PyCLRMetaType = MetaType.StashPop(stack); + } + + public static bool HasStashData() + { + return PySys_GetObject("clr_data") != IntPtr.Zero; + } + + public static void ClearStash() + { + PySys_SetObject("clr_data", IntPtr.Zero); + } + } +} diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index bf7850d01..ca3fecbb2 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -1,10 +1,7 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; -using System.Runtime.Serialization.Formatters.Binary; using static Python.Runtime.Runtime; @@ -236,92 +233,4 @@ private static unsafe IntPtr GetNextGCNode(IntPtr node) return ((PyGC_Head*)node)->gc.gc_next; } } - - - class RuntimeData - { - internal static void Stash() - { - var formatter = new BinaryFormatter(); - var ms = new MemoryStream(); - var stack = new Stack(); - MetaType.StashPush(stack); - TypeManager.StashPush(stack); - ClassManager.StashPush(stack); - var objs = ManagedType.GetManagedObjects(); - var saveObjs = new Dictionary(); - foreach (var entry in objs) - { - var obj = entry.Key; - if (entry.Value == ManagedType.TrackTypes.Wrapper - && !obj.GetType().IsSerializable) - { - // XXX: Skip non-serializable objects, - // use them after next initialization will raise exceptions. - continue; - } - Debug.Assert(obj.GetType().IsSerializable); - obj.Save(); - saveObjs.Add(entry.Key, entry.Value); - } - stack.Push(saveObjs); - formatter.Serialize(ms, stack); - - byte[] data = ms.GetBuffer(); - // TODO: use buffer api instead - Debug.Assert(data.Length <= int.MaxValue); - IntPtr mem = PyMem_Malloc(data.LongLength + IntPtr.Size); - Marshal.WriteIntPtr(mem, (IntPtr)data.LongLength); - Marshal.Copy(data, 0, mem + IntPtr.Size, data.Length); - - IntPtr capsule = PySys_GetObject("clr_data"); - if (capsule != IntPtr.Zero) - { - IntPtr oldData = PyCapsule_GetPointer(capsule, null); - PyMem_Free(oldData); - PyCapsule_SetPointer(capsule, IntPtr.Zero); - } - capsule = PyCapsule_New(mem, null, IntPtr.Zero); - PySys_SetObject("clr_data", capsule); - XDecref(capsule); - } - - internal static void StashPop() - { - IntPtr capsule = PySys_GetObject("clr_data"); - if (capsule == IntPtr.Zero) - { - return; - } - IntPtr mem = PyCapsule_GetPointer(capsule, null); - int length = (int)Marshal.ReadIntPtr(mem); - byte[] data = new byte[length]; - Marshal.Copy(mem + IntPtr.Size, data, 0, length); - var ms = new MemoryStream(data); - var formatter = new BinaryFormatter(); - - var stack = (Stack)formatter.Deserialize(ms); - - var loadObjs = (IDictionary)stack.Pop(); - foreach (var entry in loadObjs) - { - ManagedType obj = entry.Key; - obj.Load(); - } - Debug.Assert(ManagedType.GetManagedObjects().Count == loadObjs.Count); - ClassManager.StashPop(stack); - TypeManager.StashPop(stack); - PyCLRMetaType = MetaType.StashPop(stack); - } - - public static bool HasStashData() - { - return PySys_GetObject("clr_data") != IntPtr.Zero; - } - - public static void ClearStash() - { - PySys_SetObject("clr_data", IntPtr.Zero); - } - } } From 97c8c2a200bc1246f7ed0fa034300b81fa6dfe48 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 21 Feb 2020 11:51:10 +0800 Subject: [PATCH 047/134] Extract InitPyMembers method --- src/runtime/runtime.cs | 64 ++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d61ac7b34..dcf3df4b1 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -179,6 +179,40 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd ClassDerivedObject.Reset(); TypeManager.Initialize(); + InitPyMembers(); + + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + NativeCodePageHelper.InitializePlatformData(); + + // Initialize modules that depend on the runtime class. + AssemblyManager.Initialize(); +#if !NETSTANDARD + if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) + { + RuntimeData.StashPop(); + } + else +#endif + { + PyCLRMetaType = MetaType.Initialize(); // Steal a reference + } + Exceptions.Initialize(); + ImportHook.Initialize(); + + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + IntPtr path = PySys_GetObject("path"); + IntPtr item = PyString_FromString(rtdir); + PyList_Append(path, item); + XDecref(item); + AssemblyManager.UpdatePath(); + } + + private static void InitPyMembers() + { IntPtr op; { var builtins = GetBuiltins(); @@ -300,36 +334,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd PyModuleType = PyObject_Type(sys); XDecref(sys); } - - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - NativeCodePageHelper.InitializePlatformData(); - - // Initialize modules that depend on the runtime class. - AssemblyManager.Initialize(); -#if !NETSTANDARD - if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) - { - RuntimeData.StashPop(); - RuntimeData.ClearStash(); - } - else -#endif - { - PyCLRMetaType = MetaType.Initialize(); // Steal a reference - } - Exceptions.Initialize(); - ImportHook.Initialize(); - - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - IntPtr path = PySys_GetObject("path"); - IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); - XDecref(item); - AssemblyManager.UpdatePath(); } private static IntPtr Get_PyObject_NextNotImplemented() From 39f47c81bbbafc501c98e7d59517484eca4e7a57 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 24 Feb 2020 13:02:08 +0800 Subject: [PATCH 048/134] * Classify runtime data * External interface for custom storing wrapper objects --- src/runtime/classmanager.cs | 8 +- src/runtime/clrobject.cs | 16 ++ src/runtime/interop.cs | 4 +- src/runtime/metatype.cs | 8 +- src/runtime/runtime.cs | 17 +- src/runtime/runtime_data.cs | 333 +++++++++++++++++++++++++++++++----- src/runtime/typemanager.cs | 18 +- 7 files changed, 336 insertions(+), 68 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 3b9922a3d..0c7fff5af 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -82,14 +82,14 @@ private static int OnVisit(IntPtr ob, IntPtr arg) } - internal static void StashPush(Stack stack) + internal static void StashPush(RuntimeDataStorage storage) { - stack.Push(cache); + storage.PushValue(cache); } - internal static void StashPop(Stack stack) + internal static void StashPop(RuntimeDataStorage storage) { - cache = (Dictionary)stack.Pop(); + cache = storage.PopValue>(); } /// diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index b004eb23b..a8af80f67 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -35,6 +35,10 @@ internal CLRObject(object ob, IntPtr tp) Exceptions.SetArgsAndCause(py); } + protected CLRObject() + { + } + internal static CLRObject GetInstance(object ob, IntPtr pyType) { @@ -70,6 +74,18 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } + internal static CLRObject Restore(object ob, IntPtr pyHandle) + { + CLRObject co = new CLRObject() + { + inst = ob, + pyHandle = pyHandle, + tpHandle = Runtime.PyObject_TYPE(pyHandle) + }; + co.Load(); + return co; + } + protected override void OnSave() { base.OnSave(); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 58ec82415..5a39a512d 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -564,7 +564,7 @@ public ThunkInfo(Delegate target) } } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + [StructLayout(LayoutKind.Sequential)] struct PyGC_Node { public IntPtr gc_next; @@ -572,7 +572,7 @@ struct PyGC_Node public IntPtr gc_refs; } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + [StructLayout(LayoutKind.Sequential)] struct PyGC_Head { public PyGC_Node gc; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index b54c2eb90..95dd8b9b4 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -40,15 +40,15 @@ public static void Release() _metaSlotsHodler = null; } - internal static void StashPush(Stack stack) + internal static void StashPush(RuntimeDataStorage storage) { Runtime.XIncref(PyCLRMetaType); - stack.Push(PyCLRMetaType); + storage.PushValue(PyCLRMetaType); } - internal static IntPtr StashPop(Stack stack) + internal static IntPtr StashPop(RuntimeDataStorage storage) { - PyCLRMetaType = (IntPtr)stack.Pop(); + PyCLRMetaType = storage.PopValue(); _metaSlotsHodler = new SlotsHolder(PyCLRMetaType); TypeManager.InitializeSlots(PyCLRMetaType, typeof(MetaType), _metaSlotsHodler); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index dcf3df4b1..66c23c87b 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -206,7 +206,10 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); IntPtr path = PySys_GetObject("path"); IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); + if (PySequence_Contains(path, item) == 0) + { + PyList_Append(path, item); + } XDecref(item); AssemblyManager.UpdatePath(); } @@ -364,12 +367,7 @@ internal static void Shutdown() AssemblyManager.Shutdown(); ImportHook.Shutdown(); -#if !NETSTANDARD - if (mode != ShutdownMode.Reload) - { - ClearClrModules(); - } -#endif + ClearClrModules(); RemoveClrRootModule(); MoveClrInstancesOnwershipToPython(); @@ -454,8 +452,7 @@ private static void ClearClrModules() var item = PyList_GetItem(items, i); var name = PyTuple_GetItem(item, 0); var module = PyTuple_GetItem(item, 1); - var clrModule = ManagedType.GetManagedObject(module); - if (clrModule != null) + if (ManagedType.IsManagedType(module)) { PyDict_DelItem(modules, name); } @@ -486,8 +483,8 @@ private static void PyDictTryDelItem(IntPtr dict, string key) private static void MoveClrInstancesOnwershipToPython() { - var copyObjs = ManagedType.GetManagedObjects().ToArray(); var objs = ManagedType.GetManagedObjects(); + var copyObjs = objs.ToArray(); foreach (var entry in copyObjs) { ManagedType obj = entry.Key; diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 420837be8..a4cab5a20 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -1,50 +1,70 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Diagnostics; using System.IO; +using System.Linq; using System.Runtime.InteropServices; +using System.Runtime.Serialization; using System.Runtime.Serialization.Formatters.Binary; using static Python.Runtime.Runtime; namespace Python.Runtime { - class RuntimeData + public static class RuntimeData { - internal static void Stash() + private static Type _formatterType; + public static Type FormatterType { - var formatter = new BinaryFormatter(); - var ms = new MemoryStream(); - var stack = new Stack(); - MetaType.StashPush(stack); - TypeManager.StashPush(stack); - ClassManager.StashPush(stack); - var objs = ManagedType.GetManagedObjects(); - var saveObjs = new Dictionary(); - foreach (var entry in objs) + get => _formatterType; + set { - var obj = entry.Key; - if (entry.Value == ManagedType.TrackTypes.Wrapper - && !obj.GetType().IsSerializable) + if (!typeof(IFormatter).IsAssignableFrom(value)) { - // XXX: Skip non-serializable objects, - // use them after next initialization will raise exceptions. - continue; + throw new ArgumentException("Not a type implemented IFormatter"); } - Debug.Assert(obj.GetType().IsSerializable); - obj.Save(); - saveObjs.Add(entry.Key, entry.Value); + _formatterType = value; } - stack.Push(saveObjs); - formatter.Serialize(ms, stack); + } + + public static ICLRObjectStorer WrappersStorer { get; set; } + + internal static void Stash() + { + var metaStorage = new RuntimeDataStorage(); + MetaType.StashPush(metaStorage); + + var typeStorage = new RuntimeDataStorage(); + TypeManager.StashPush(typeStorage); + + var clsStorage = new RuntimeDataStorage(); + ClassManager.StashPush(clsStorage); + var moduleStorage = new RuntimeDataStorage(); + StashPushModules(moduleStorage); + + var objStorage = new RuntimeDataStorage(); + StashPushObjects(objStorage); + + var runtimeStorage = new RuntimeDataStorage(); + runtimeStorage.AddValue("meta", metaStorage); + runtimeStorage.AddValue("types", typeStorage); + runtimeStorage.AddValue("classes", clsStorage); + runtimeStorage.AddValue("modules", moduleStorage); + runtimeStorage.AddValue("objs", objStorage); + + IFormatter formatter = CreateFormatter(); + var ms = new MemoryStream(); + formatter.Serialize(ms, runtimeStorage); + + Debug.Assert(ms.Length <= int.MaxValue); byte[] data = ms.GetBuffer(); // TODO: use buffer api instead - Debug.Assert(data.Length <= int.MaxValue); - IntPtr mem = PyMem_Malloc(data.LongLength + IntPtr.Size); - Marshal.WriteIntPtr(mem, (IntPtr)data.LongLength); - Marshal.Copy(data, 0, mem + IntPtr.Size, data.Length); + IntPtr mem = PyMem_Malloc(ms.Length + IntPtr.Size); + Marshal.WriteIntPtr(mem, (IntPtr)ms.Length); + Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); IntPtr capsule = PySys_GetObject("clr_data"); if (capsule != IntPtr.Zero) @@ -58,7 +78,20 @@ internal static void Stash() XDecref(capsule); } + internal static void StashPop() + { + try + { + StashPopImpl(); + } + finally + { + ClearStash(); + } + } + + private static void StashPopImpl() { IntPtr capsule = PySys_GetObject("clr_data"); if (capsule == IntPtr.Zero) @@ -70,20 +103,14 @@ internal static void StashPop() byte[] data = new byte[length]; Marshal.Copy(mem + IntPtr.Size, data, 0, length); var ms = new MemoryStream(data); - var formatter = new BinaryFormatter(); - - var stack = (Stack)formatter.Deserialize(ms); + var formatter = CreateFormatter(); + var storage = (RuntimeDataStorage)formatter.Deserialize(ms); - var loadObjs = (IDictionary)stack.Pop(); - foreach (var entry in loadObjs) - { - ManagedType obj = entry.Key; - obj.Load(); - } - Debug.Assert(ManagedType.GetManagedObjects().Count == loadObjs.Count); - ClassManager.StashPop(stack); - TypeManager.StashPop(stack); - PyCLRMetaType = MetaType.StashPop(stack); + StashPopModules(storage.GetStorage("modules")); + StashPopObjects(storage.GetStorage("objs")); + ClassManager.StashPop(storage.GetStorage("classes")); + TypeManager.StashPop(storage.GetStorage("types")); + PyCLRMetaType = MetaType.StashPop(storage.GetStorage("meta")); } public static bool HasStashData() @@ -95,5 +122,233 @@ public static void ClearStash() { PySys_SetObject("clr_data", IntPtr.Zero); } + + private static void StashPushObjects(RuntimeDataStorage storage) + { + var objs = ManagedType.GetManagedObjects(); + var extensionObjs = new List(); + var wrappers = new Dictionary>(); + var serializeObjs = new CLRWrapperCollection(); + foreach (var entry in objs) + { + var obj = entry.Key; + switch (entry.Value) + { + case ManagedType.TrackTypes.Extension: + Debug.Assert(obj.GetType().IsSerializable); + obj.Save(); + extensionObjs.Add(obj); + break; + case ManagedType.TrackTypes.Wrapper: + // Wrapper must be the CLRObject + var clrObj = (CLRObject)obj; + object inst = clrObj.inst; + CLRMappedItem item; + List mappedObjs; + if (!serializeObjs.TryGetValue(inst, out item)) + { + item = new CLRMappedItem(inst) + { + Handles = new List() + }; + serializeObjs.Add(item); + + Debug.Assert(!wrappers.ContainsKey(inst)); + mappedObjs = new List(); + wrappers.Add(inst, mappedObjs); + } + else + { + mappedObjs = wrappers[inst]; + } + item.Handles.Add(clrObj.pyHandle); + mappedObjs.Add(clrObj); + break; + default: + break; + } + } + + var wrapperStorage = new RuntimeDataStorage(); + WrappersStorer?.Store(serializeObjs, wrapperStorage); + + var internalStores = new List(); + foreach (var item in serializeObjs) + { + if (!item.Stored) + { + if (!item.Instance.GetType().IsSerializable) + { + continue; + } + internalStores.AddRange(wrappers[item.Instance]); + } + foreach (var clrObj in wrappers[item.Instance]) + { + clrObj.Save(); + } + } + storage.AddValue("internalStores", internalStores); + storage.AddValue("extensions", extensionObjs); + storage.AddValue("wrappers", wrapperStorage); + } + + private static void StashPopObjects(RuntimeDataStorage storage) + { + var extensions = storage.GetValue>("extensions"); + var internalStores = storage.GetValue>("internalStores"); + foreach (var obj in Enumerable.Union(extensions, internalStores)) + { + obj.Load(); + } + if (WrappersStorer != null) + { + var wrapperStorage = storage.GetStorage("wrappers"); + var handle2Obj = WrappersStorer.Restore(wrapperStorage); + foreach (var item in handle2Obj) + { + object obj = item.Instance; + foreach (var handle in item.Handles) + { + CLRObject.Restore(obj, handle); + } + } + } + } + + private static void StashPushModules(RuntimeDataStorage storage) + { + var pyModules = PyImport_GetModuleDict(); + var items = PyDict_Items(pyModules); + long length = PyList_Size(items); + var modules = new Dictionary(); ; + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item, 0); + var module = PyTuple_GetItem(item, 1); + if (ManagedType.IsManagedType(module)) + { + XIncref(name); + XIncref(module); + modules.Add(name, module); + } + } + XDecref(items); + storage.AddValue("modules", modules); + } + + private static void StashPopModules(RuntimeDataStorage storage) + { + var modules = storage.GetValue>("modules"); + var pyMoudles = PyImport_GetModuleDict(); + foreach (var item in modules) + { + int res = PyDict_SetItem(pyMoudles, item.Key, item.Value); + PythonException.ThrowIfIsNotZero(res); + XDecref(item.Key); + XDecref(item.Value); + } + modules.Clear(); + } + + private static IFormatter CreateFormatter() + { + return FormatterType != null ? + (IFormatter)Activator.CreateInstance(FormatterType) + : new BinaryFormatter(); + } + } + + + [Serializable] + public class RuntimeDataStorage + { + private Stack _stack; + private Dictionary _namedValues; + + public T AddValue(string name, T value) + { + if (_namedValues == null) + { + _namedValues = new Dictionary(); + } + _namedValues.Add(name, value); + return value; + } + + public object GetValue(string name) + { + return _namedValues[name]; + } + + public T GetValue(string name) + { + return (T)GetValue(name); + } + + public RuntimeDataStorage GetStorage(string name) + { + return GetValue(name); + } + + public T PushValue(T value) + { + if (_stack == null) + { + _stack = new Stack(); + } + _stack.Push(value); + return value; + } + + public object PopValue() + { + return _stack.Pop(); + } + + public T PopValue() + { + return (T)PopValue(); + } + } + + + public class CLRMappedItem + { + public object Instance { get; private set; } + public IList Handles { get; set; } + public bool Stored { get; set; } + + public CLRMappedItem(object instance) + { + Instance = instance; + } + } + + + public interface ICLRObjectStorer + { + ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); + CLRWrapperCollection Restore(RuntimeDataStorage storage); + } + + + public class CLRWrapperCollection : KeyedCollection + { + public bool TryGetValue(object key, out CLRMappedItem value) + { + if (Dictionary == null) + { + value = null; + return false; + } + return Dictionary.TryGetValue(key, out value); + } + + protected override object GetKeyForItem(CLRMappedItem item) + { + return item.Instance; + } } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 6475fa919..48716a967 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -65,19 +65,19 @@ internal static void RemoveTypes() _slotsHolders.Clear(); } - internal static void StashPush(Stack stack) + internal static void StashPush(RuntimeDataStorage storage) { foreach (var tpHandle in cache.Values) { Runtime.XIncref(tpHandle); } - stack.Push(cache); + storage.PushValue(cache); } - internal static void StashPop(Stack stack) + internal static void StashPop(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); - cache = (Dictionary)stack.Pop(); + cache = storage.PopValue>(); foreach (var entry in cache) { Type type = entry.Key; @@ -235,11 +235,11 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.BaseType; - flags |= TypeFlags.HaveGC; + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.BaseType + | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); // Leverage followup initialization from the Python runtime. Note From aa63f0bcbcc1af0aa0488c13a2a3d1adaf97f8c6 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 27 Feb 2020 17:03:00 +0800 Subject: [PATCH 049/134] * Stash for ImportHook * Refactor Save/Load * Fix refcnt error for moduleobject.cs --- src/embed_tests/TestDomainReload.cs | 39 +++++++++++++++++------------ src/runtime/classbase.cs | 7 ++++++ src/runtime/classmanager.cs | 14 ++++++++++- src/runtime/eventbinding.cs | 6 ----- src/runtime/exceptions.cs | 6 +---- src/runtime/extensiontype.cs | 6 ----- src/runtime/importhook.cs | 18 ++++++++++++- src/runtime/managedtype.cs | 2 ++ src/runtime/moduleobject.cs | 6 +++++ src/runtime/runtime.cs | 9 ++++--- src/runtime/runtime_data.cs | 18 ++++++++++++- src/runtime/typemanager.cs | 13 +++++++--- 12 files changed, 102 insertions(+), 42 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index c94c8599c..68e92e22c 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -233,26 +233,33 @@ public static void RunPython() mode = ShutdownMode.Soft; } PythonEngine.Initialize(mode: mode); - using (Py.GIL()) + try { - try - { - var pyScript = string.Format("import clr\n" - + "print('[{0} in python] imported clr')\n" - + "clr.AddReference('System')\n" - + "print('[{0} in python] allocated a clr object')\n" - + "import gc\n" - + "gc.collect()\n" - + "print('[{0} in python] collected garbage')\n", - name); - PythonEngine.Exec(pyScript); - } - catch (Exception e) + using (Py.GIL()) { - Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + try + { + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + throw; + } } } - PythonEngine.BeginAllowThreads(); + finally + { + PythonEngine.BeginAllowThreads(); + } } diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 3f10849fd..9a8b0db74 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -303,5 +303,12 @@ public static int tp_clear(IntPtr ob) self.tpHandle = IntPtr.Zero; return 0; } + + protected override void OnLoad() + { + base.OnLoad(); + gcHandle = AllocGCHandle(); + Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + } } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0c7fff5af..ee9006d1e 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -56,12 +56,14 @@ internal static void RemoveClasses() cls.CallTypeTraverse(OnVisit, visitedPtr); // XXX: Force release instance resources but not dealloc itself. cls.CallTypeClear(); + cls.DecrRefCount(); } } finally { visitedHandle.Free(); } + cache.Clear(); } private static int OnVisit(IntPtr ob, IntPtr arg) @@ -81,15 +83,25 @@ private static int OnVisit(IntPtr ob, IntPtr arg) return 0; } - internal static void StashPush(RuntimeDataStorage storage) { storage.PushValue(cache); + foreach (var cls in cache.Values) + { + // This incref is for cache to hold the cls, + // thus no need for decreasing it at StashPop. + Runtime.XIncref(cls.pyHandle); + cls.Save(); + } } internal static void StashPop(RuntimeDataStorage storage) { cache = storage.PopValue>(); + foreach (var cls in cache.Values) + { + cls.Load(); + } } /// diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index 5dbace9d9..581095185 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -128,11 +128,5 @@ public static int tp_clear(IntPtr ob) Runtime.Py_CLEAR(ref self.target); return 0; } - - protected override void OnSave() - { - base.OnSave(); - Runtime.XIncref(target); - } } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 908de9745..b0c540782 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -89,15 +89,11 @@ internal static Exception ToException(IntPtr ob) /// /// Readability of the Exceptions class improvements as we look toward version 2.7 ... /// - public class Exceptions + public static class Exceptions { internal static IntPtr warnings_module; internal static IntPtr exceptions_module; - private Exceptions() - { - } - /// /// Initialization performed on startup of the Python runtime. /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index fc839644e..75ca04205 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -98,12 +98,6 @@ public static void tp_dealloc(IntPtr ob) self.Dealloc(); } - protected override void OnSave() - { - base.OnSave(); - Runtime.XIncref(pyHandle); - } - protected override void OnLoad() { base.OnLoad(); diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index aa3bbab6d..e6e6c252a 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// /// Implements the "import hook" used to integrate Python with the CLR. /// - internal class ImportHook + internal static class ImportHook { private static IntPtr py_import; private static CLRModule root; @@ -130,6 +130,22 @@ internal static void Shutdown() CLRModule.Reset(); } + internal static void StashPush(RuntimeDataStorage storage) + { + Runtime.XIncref(py_clr_module); + Runtime.XIncref(root.pyHandle); + storage.AddValue("py_clr_module", py_clr_module); + storage.AddValue("root", root.pyHandle); + } + + internal static void StashPop(RuntimeDataStorage storage) + { + InitImport(); + storage.GetValue("py_clr_module", out py_clr_module); + var rootHandle = storage.GetValue("root"); + root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + } + /// /// Return the clr python module (new reference) /// diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 99e897570..9fc1a6424 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -202,12 +202,14 @@ protected void TypeClear() internal void Save() { + Runtime.XIncref(pyHandle); OnSave(); } internal void Load() { OnLoad(); + Runtime.XDecref(pyHandle); } protected virtual void OnSave() { } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8661d74ae..c6a4a5ca0 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -103,6 +103,7 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } @@ -118,6 +119,7 @@ public ManagedType GetAttribute(string name, bool guess) } c = ClassManager.GetClass(type); StoreAttribute(name, c); + c.DecrRefCount(); return c; } @@ -132,6 +134,7 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } @@ -144,6 +147,7 @@ public ManagedType GetAttribute(string name, bool guess) } c = ClassManager.GetClass(type); StoreAttribute(name, c); + c.DecrRefCount(); return c; } } @@ -239,6 +243,7 @@ internal void InitializeModuleMembers() mi[0] = method; var m = new ModuleFunctionObject(type, name, mi, allow_threads); StoreAttribute(name, m); + m.DecrRefCount(); } } @@ -251,6 +256,7 @@ internal void InitializeModuleMembers() string name = property.Name; var p = new ModulePropertyObject(property); StoreAttribute(name, p); + p.DecrRefCount(); } } type = type.BaseType; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 66c23c87b..5124a46e7 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -197,9 +197,9 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd #endif { PyCLRMetaType = MetaType.Initialize(); // Steal a reference + ImportHook.Initialize(); } Exceptions.Initialize(); - ImportHook.Initialize(); // Need to add the runtime directory to sys.path so that we // can find built-in assemblies like System.Data, et. al. @@ -497,13 +497,16 @@ private static void MoveClrInstancesOnwershipToPython() obj.CallTypeClear(); // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), // thus just be safe to give it back to GC chain. - PyObject_GC_Track(obj.pyHandle); + if (!_PyObject_GC_IS_TRACKED(obj.pyHandle)) + { + PyObject_GC_Track(obj.pyHandle); + } } if (obj.gcHandle.IsAllocated) { obj.gcHandle.Free(); } - obj.gcHandle = new GCHandle(); + obj.gcHandle = default; } ManagedType.ClearTrackedObjects(); } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index a4cab5a20..96e8ea85b 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -36,6 +36,9 @@ internal static void Stash() var metaStorage = new RuntimeDataStorage(); MetaType.StashPush(metaStorage); + var importStorage = new RuntimeDataStorage(); + ImportHook.StashPush(importStorage); + var typeStorage = new RuntimeDataStorage(); TypeManager.StashPush(typeStorage); @@ -50,6 +53,7 @@ internal static void Stash() var runtimeStorage = new RuntimeDataStorage(); runtimeStorage.AddValue("meta", metaStorage); + runtimeStorage.AddValue("import", importStorage); runtimeStorage.AddValue("types", typeStorage); runtimeStorage.AddValue("classes", clsStorage); runtimeStorage.AddValue("modules", moduleStorage); @@ -106,10 +110,11 @@ private static void StashPopImpl() var formatter = CreateFormatter(); var storage = (RuntimeDataStorage)formatter.Deserialize(ms); - StashPopModules(storage.GetStorage("modules")); StashPopObjects(storage.GetStorage("objs")); + StashPopModules(storage.GetStorage("modules")); ClassManager.StashPop(storage.GetStorage("classes")); TypeManager.StashPop(storage.GetStorage("types")); + ImportHook.StashPop(storage.GetStorage("import")); PyCLRMetaType = MetaType.StashPop(storage.GetStorage("meta")); } @@ -287,6 +292,12 @@ public T GetValue(string name) return (T)GetValue(name); } + public T GetValue(string name, out T value) + { + value = GetValue(name); + return value; + } + public RuntimeDataStorage GetStorage(string name) { return GetValue(name); @@ -311,6 +322,11 @@ public T PopValue() { return (T)PopValue(); } + + public T PopValue(out T value) + { + return value = (T)PopValue(); + } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 48716a967..b552e2cae 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -22,6 +22,7 @@ internal class TypeManager private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; private static Dictionary cache = new Dictionary(); private static readonly Dictionary _slotsHolders = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); // Slots which must be set private static readonly string[] _requiredSlots = new string[] @@ -62,6 +63,7 @@ internal static void RemoveTypes() Runtime.XDecref(tpHandle); } cache.Clear(); + _slotsImpls.Clear(); _slotsHolders.Clear(); } @@ -71,19 +73,21 @@ internal static void StashPush(RuntimeDataStorage storage) { Runtime.XIncref(tpHandle); } - storage.PushValue(cache); + storage.AddValue("cache", cache); + storage.AddValue("slots", _slotsImpls); } internal static void StashPop(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); - cache = storage.PopValue>(); + storage.GetValue("slots", out _slotsImpls); + storage.GetValue("cache", out cache); foreach (var entry in cache) { Type type = entry.Key; IntPtr handle = entry.Value; SlotsHolder holder = CreateSolotsHolder(handle); - InitializeSlots(handle, type, holder); + InitializeSlots(handle, _slotsImpls[type], holder); // FIXME: mp_length_slot.CanAssgin(clrType) } } @@ -107,6 +111,7 @@ internal static IntPtr GetTypeHandle(Type type) } handle = CreateType(type); cache[type] = handle; + _slotsImpls.Add(type, type); return handle; } @@ -127,6 +132,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) } handle = CreateType(obj, type); cache[type] = handle; + _slotsImpls.Add(type, obj.GetType()); return handle; } @@ -711,6 +717,7 @@ private static void InitMethods(IntPtr pytype, Type type) mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + //m.DecrRefCount(); addedMethods.Add(method_name); } } From 35cbe55b285923bc7253d545d3ae6f530703f4b3 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 27 Feb 2020 18:26:33 +0800 Subject: [PATCH 050/134] Apply Reference type usage --- src/runtime/runtime.cs | 10 +++++----- src/runtime/runtime_data.cs | 9 +++++---- src/runtime/runtime_state.cs | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2f255c995..d3faa5f24 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -446,18 +446,18 @@ private static void ClearClrModules() { var modules = PyImport_GetModuleDict(); var items = PyDict_Items(modules); - long length = PyList_Size(items); + long length = PyList_Size(items.DangerousGetAddress()); for (long i = 0; i < length; i++) { - var item = PyList_GetItem(items, i); - var name = PyTuple_GetItem(item, 0); - var module = PyTuple_GetItem(item, 1); + var item = PyList_GetItem(items.DangerousGetAddress(), i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); if (ManagedType.IsManagedType(module)) { PyDict_DelItem(modules, name); } } - XDecref(items); + items.Dispose(); } private static void RemoveClrRootModule() diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 96e8ea85b..f8cc1f682 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -224,14 +224,15 @@ private static void StashPopObjects(RuntimeDataStorage storage) private static void StashPushModules(RuntimeDataStorage storage) { var pyModules = PyImport_GetModuleDict(); - var items = PyDict_Items(pyModules); + var itemsRef = PyDict_Items(pyModules); + var items = itemsRef.DangerousGetAddress(); long length = PyList_Size(items); var modules = new Dictionary(); ; for (long i = 0; i < length; i++) { var item = PyList_GetItem(items, i); - var name = PyTuple_GetItem(item, 0); - var module = PyTuple_GetItem(item, 1); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); if (ManagedType.IsManagedType(module)) { XIncref(name); @@ -239,7 +240,7 @@ private static void StashPushModules(RuntimeDataStorage storage) modules.Add(name, module); } } - XDecref(items); + itemsRef.Dispose(); storage.AddValue("modules", modules); } diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index ca3fecbb2..9c3823315 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -152,7 +152,7 @@ public static IEnumerable PyGCGetObjects() for (long i = 0; i < length; i++) { var obj = PyList_GetItem(objs, i); - yield return obj; + yield return obj.DangerousGetAddress(); } XDecref(objs); XDecref(gc); @@ -166,7 +166,7 @@ public static IEnumerable GetModuleNames() for (int i = 0; i < length; i++) { var name = PyList_GetItem(names, i); - yield return name; + yield return name.DangerousGetAddress(); } } From f23cae63b9944fd65a706456b9ad090ad1e94bb2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 2 Mar 2020 12:07:00 +0800 Subject: [PATCH 051/134] * Manipulate refcnt in Push/Pop objects * Add several Serializable mark --- src/embed_tests/TestFinalizer.cs | 2 +- src/runtime/arrayobject.cs | 1 + src/runtime/classderived.cs | 1 + src/runtime/classmanager.cs | 3 +-- src/runtime/exceptions.cs | 1 + src/runtime/generictype.cs | 1 + src/runtime/interfaceobject.cs | 1 + src/runtime/managedtype.cs | 2 -- src/runtime/pyobject.cs | 4 ++-- src/runtime/runtime_data.cs | 17 ++++++++++++++--- 10 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 650ee5686..53497d7cd 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -214,7 +214,7 @@ public void ValidateRefCount() { if (!Finalizer.Instance.RefCountValidationEnabled) { - Assert.Pass("Only run with FINALIZER_CHECK"); + Assert.Ignore("Only run with FINALIZER_CHECK"); } IntPtr ptr = IntPtr.Zero; bool called = false; diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 1ef318473..cf7067ace 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// the same as a ClassObject, except that it provides sequence semantics /// to support natural array usage (indexing) from Python. /// + [Serializable] internal class ArrayObject : ClassBase { internal ArrayObject(Type tp) : base(tp) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index f9c019cfe..ab2bda3bf 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -22,6 +22,7 @@ public interface IPythonDerivedType { } + [Serializable] internal class ClassDerivedObject : ClassObject { private static Dictionary assemblyBuilders; diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index ee9006d1e..0ef6f5e42 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -69,11 +69,10 @@ internal static void RemoveClasses() private static int OnVisit(IntPtr ob, IntPtr arg) { var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; - if (visited.Contains(ob)) + if (!visited.Add(ob)) { return 0; } - visited.Add(ob); var clrObj = ManagedType.GetManagedObject(ob); if (clrObj != null) { diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index b0c540782..1af91d1b8 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -15,6 +15,7 @@ namespace Python.Runtime /// it subclasses System.Object. Instead TypeManager.CreateType() uses /// Python's exception.Exception class as base class for System.Exception. /// + [Serializable] internal class ExceptionClassObject : ClassObject { internal ExceptionClassObject(Type tp) : base(tp) diff --git a/src/runtime/generictype.cs b/src/runtime/generictype.cs index eeae801d2..76d2e9a5d 100644 --- a/src/runtime/generictype.cs +++ b/src/runtime/generictype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// generic types. Both are essentially factories for creating closed /// types based on the required generic type parameters. /// + [Serializable] internal class GenericType : ClassBase { internal GenericType(Type tp) : base(tp) diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 616ced6bd..536c8796f 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// Each of those type objects is associated with an instance of this /// class, which provides the implementation for the Python type. /// + [Serializable] internal class InterfaceObject : ClassBase { internal ConstructorInfo ctor; diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 9fc1a6424..99e897570 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -202,14 +202,12 @@ protected void TypeClear() internal void Save() { - Runtime.XIncref(pyHandle); OnSave(); } internal void Load() { OnLoad(); - Runtime.XDecref(pyHandle); } protected virtual void OnSave() { } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 58359e5c5..da6c73b18 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -20,8 +20,8 @@ public interface IPyDisposable : IDisposable /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - //[Serializable] - public class PyObject : DynamicObject, IEnumerable, IPyDisposable + [Serializable] + public partial class PyObject : DynamicObject, IEnumerable, IPyDisposable { #if TRACE_ALLOC /// diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index f8cc1f682..83bdb23ba 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -110,12 +110,17 @@ private static void StashPopImpl() var formatter = CreateFormatter(); var storage = (RuntimeDataStorage)formatter.Deserialize(ms); - StashPopObjects(storage.GetStorage("objs")); + var objs = StashPopObjects(storage.GetStorage("objs")); StashPopModules(storage.GetStorage("modules")); ClassManager.StashPop(storage.GetStorage("classes")); TypeManager.StashPop(storage.GetStorage("types")); ImportHook.StashPop(storage.GetStorage("import")); PyCLRMetaType = MetaType.StashPop(storage.GetStorage("meta")); + + foreach (var item in objs) + { + XDecref(item.pyHandle); + } } public static bool HasStashData() @@ -137,6 +142,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) foreach (var entry in objs) { var obj = entry.Key; + XIncref(obj.pyHandle); switch (entry.Value) { case ManagedType.TrackTypes.Extension: @@ -190,6 +196,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) } foreach (var clrObj in wrappers[item.Instance]) { + XIncref(clrObj.pyHandle); clrObj.Save(); } } @@ -198,13 +205,15 @@ private static void StashPushObjects(RuntimeDataStorage storage) storage.AddValue("wrappers", wrapperStorage); } - private static void StashPopObjects(RuntimeDataStorage storage) + private static IEnumerable StashPopObjects(RuntimeDataStorage storage) { var extensions = storage.GetValue>("extensions"); var internalStores = storage.GetValue>("internalStores"); + var storedObjs = new List(); foreach (var obj in Enumerable.Union(extensions, internalStores)) { obj.Load(); + storedObjs.Add(obj); } if (WrappersStorer != null) { @@ -215,10 +224,12 @@ private static void StashPopObjects(RuntimeDataStorage storage) object obj = item.Instance; foreach (var handle in item.Handles) { - CLRObject.Restore(obj, handle); + var co = CLRObject.Restore(obj, handle); + storedObjs.Add(co); } } } + return storedObjs; } private static void StashPushModules(RuntimeDataStorage storage) From 183f9d822c17b1cba7f3538b8d4394b7ef1cae44 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 8 Mar 2020 15:51:00 +0800 Subject: [PATCH 052/134] Load cache of ModuleObject after reload --- src/runtime/fieldobject.cs | 1 + src/runtime/moduleobject.cs | 37 +++++++++++++++++++++++++++++++++++-- src/runtime/runtime.cs | 6 +++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index c4675d723..86b93dd1b 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR fields. /// + [Serializable] internal class FieldObject : ExtensionType { private FieldInfo info; diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index ca6abd05d..4a1a15ca8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -17,6 +17,10 @@ internal class ModuleObject : ExtensionType [NonSerialized] private Dictionary cache; + [NonSerialized] + // FIXME: Used by reload mode, remove it after implement a delay load handler. + private bool _cacheInited; + internal string moduleName; internal IntPtr dict; protected string _namespace; @@ -29,6 +33,7 @@ public ModuleObject(string name) } moduleName = name; cache = new Dictionary(); + _cacheInited = true; _namespace = name; // Use the filename from any of the assemblies just so there's something for @@ -72,6 +77,11 @@ public ModuleObject(string name) /// public ManagedType GetAttribute(string name, bool guess) { + if (!_cacheInited) + { + // XXX: Used by reload mode. + SetupCacheByDict(); + } ManagedType cached = null; cache.TryGetValue(name, out cached); if (cached != null) @@ -116,7 +126,6 @@ public ManagedType GetAttribute(string name, bool guess) { c = ClassManager.GetClass(type); StoreAttribute(name, c); - c.DecrRefCount(); return c; } @@ -140,7 +149,6 @@ public ManagedType GetAttribute(string name, bool guess) { c = ClassManager.GetClass(type); StoreAttribute(name, c); - c.DecrRefCount(); return c; } } @@ -364,9 +372,34 @@ protected override void OnSave() protected override void OnLoad() { base.OnLoad(); + // XXX: Set the cache after all objects loaded. cache = new Dictionary(); + _cacheInited = false; SetObjectDict(pyHandle, dict); } + + private void SetupCacheByDict() + { + System.Diagnostics.Debug.Assert(!_cacheInited); + _cacheInited = true; + IntPtr key, value; + IntPtr pos; + while (Runtime.PyDict_Next(dict, out pos, out key, out value) != 0) + { + ManagedType obj = GetManagedObject(value); + if (obj == null) + { + continue; + } + string name = Runtime.GetManagedString(key); + if (cache.ContainsKey(name)) + { + continue; + } + Runtime.XIncref(value); + cache.Add(name, obj); + } + } } /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d3faa5f24..4cedabbf9 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -490,6 +490,7 @@ private static void MoveClrInstancesOnwershipToPython() ManagedType obj = entry.Key; if (!objs.ContainsKey(obj)) { + System.Diagnostics.Debug.Assert(obj.gcHandle == default); continue; } if (entry.Value == ManagedType.TrackTypes.Extension) @@ -1675,6 +1676,9 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_New(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDictProxy_New(IntPtr dict); @@ -2004,7 +2008,7 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type¡¯s base class. Return 0 on success, or return -1 and sets an exception on error. + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); From 9d57a82732053f408b8d115d1035177d5c077bef Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 8 Mar 2020 16:21:24 +0800 Subject: [PATCH 053/134] Add temp tests reference by https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 --- src/embed_tests/TestDomainReload.cs | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 68e92e22c..9e5f67d82 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -1,6 +1,7 @@ using System; using System.Diagnostics; using System.Reflection; +using System.Runtime.InteropServices; using NUnit.Framework; using Python.Runtime; @@ -115,6 +116,71 @@ public static void CrossDomainObject() } } + + #region Tempary tests + // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 + [Test] + public void CrossReleaseBuiltinType() + { + try + { + var numRef = CreateNumReference(); + GC.Collect(); + GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue + Finalizer.Instance.Collect(forceDispose: true); + // ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`, + // but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead. + Assert.False(numRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + [Test] + public void CrossReleaseCustomType() + { + try + { + var objRef = CreateConcreateObject(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + Finalizer.Instance.Collect(forceDispose: true); + Assert.False(objRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + private static WeakReference CreateNumReference() + { + PythonEngine.Initialize(); + var num = 3216757418.ToPython(); + Assert.AreEqual(num.Refcount, 1); + WeakReference numRef = new WeakReference(num, false); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + num = null; + return numRef; + } + + private static WeakReference CreateConcreateObject() + { + PythonEngine.Initialize(); + var obj = new Domain.MyClass().ToPython(); + Assert.AreEqual(obj.Refcount, 1); + WeakReference numRef = new WeakReference(obj, false); + PythonEngine.Shutdown(); + PythonEngine.Initialize(); + obj = null; + return numRef; + } + + #endregion Tempary tests + /// /// This is a magic incantation required to run code in an application /// domain other than the current one. @@ -305,6 +371,8 @@ from Python.EmbeddingTest.Domain import MyClass obj = MyClass() obj.Method() obj.StaticMethod() +obj.Property = 1 +obj.Field = 10 ", Assembly.GetExecutingAssembly().FullName); using (Py.GIL()) @@ -333,11 +401,27 @@ public static void RunTestObject(IntPtr handle) { using (Py.GIL()) { + IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); + IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); using (PyObject obj = new PyObject(handle)) { obj.InvokeMethod("Method"); obj.InvokeMethod("StaticMethod"); + + using (var scope = Py.CreateScope()) + { + scope.Set("obj", obj); + scope.Exec(@" +obj.Method() +obj.StaticMethod() +obj.Property += 1 +obj.Field += 10 +"); + } + var clrObj = obj.As(); + Assert.AreEqual(clrObj.Property, 2); + Assert.AreEqual(clrObj.Field, 20); } } } @@ -370,6 +454,8 @@ namespace Python.EmbeddingTest.Domain [Serializable] public class MyClass { + public int Property { get; set; } + public int Field; public void Method() { } public static void StaticMethod() { } } From 08fad26dda7fb52d98dce548fddf2302eb99fd3a Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 9 Mar 2020 23:44:59 +0800 Subject: [PATCH 054/134] * Fix refcnt error of MethodBinding * Custom storage context --- src/runtime/classbase.cs | 21 ++++++++++-- src/runtime/classmanager.cs | 18 +++++++--- src/runtime/clrobject.cs | 13 ++++--- src/runtime/extensiontype.cs | 4 +-- src/runtime/managedtype.cs | 12 +++---- src/runtime/methodbinding.cs | 7 ++++ src/runtime/methodobject.cs | 12 ++++++- src/runtime/moduleobject.cs | 51 +++++----------------------- src/runtime/runtime_data.cs | 66 ++++++++++++++++++++++++++++++------ 9 files changed, 128 insertions(+), 76 deletions(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 9a8b0db74..5bae9b350 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -304,9 +305,25 @@ public static int tp_clear(IntPtr ob) return 0; } - protected override void OnLoad() + protected override void OnSave(PyObjectSerializeContext context) { - base.OnLoad(); + base.OnSave(context); + if (pyHandle != tpHandle) + { + IntPtr dict = GetObjectDict(pyHandle); + Runtime.XIncref(dict); + context.Storage.AddValue("dict", dict); + } + } + + protected override void OnLoad(PyObjectSerializeContext context) + { + base.OnLoad(context); + if (pyHandle != tpHandle) + { + IntPtr dict = context.Storage.GetValue("dict"); + SetObjectDict(pyHandle, dict); + } gcHandle = AllocGCHandle(); Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0ef6f5e42..a7d2d74e2 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -84,23 +84,31 @@ private static int OnVisit(IntPtr ob, IntPtr arg) internal static void StashPush(RuntimeDataStorage storage) { - storage.PushValue(cache); + var contexts = storage.AddValue("contexts", + new Dictionary()); + storage.AddValue("cache", cache); foreach (var cls in cache.Values) { // This incref is for cache to hold the cls, // thus no need for decreasing it at StashPop. Runtime.XIncref(cls.pyHandle); - cls.Save(); + var context = contexts[cls.pyHandle] = new PyObjectSerializeContext(); + cls.Save(context); } } - internal static void StashPop(RuntimeDataStorage storage) + internal static Dictionary StashPop(RuntimeDataStorage storage) { - cache = storage.PopValue>(); + cache = storage.GetValue>("cache"); + var contexts = storage.GetValue >("contexts"); + var loadedObjs = new Dictionary(); foreach (var cls in cache.Values) { - cls.Load(); + var context = contexts[cls.pyHandle]; + cls.Load(context); + loadedObjs.Add(cls, context); } + return loadedObjs; } /// diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index a8af80f67..f5fe45f5a 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -74,7 +74,7 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } - internal static CLRObject Restore(object ob, IntPtr pyHandle) + internal static CLRObject Restore(object ob, IntPtr pyHandle, PyObjectSerializeContext context) { CLRObject co = new CLRObject() { @@ -82,22 +82,21 @@ internal static CLRObject Restore(object ob, IntPtr pyHandle) pyHandle = pyHandle, tpHandle = Runtime.PyObject_TYPE(pyHandle) }; - co.Load(); + co.Load(context); return co; } - protected override void OnSave() + protected override void OnSave(PyObjectSerializeContext context) { - base.OnSave(); + base.OnSave(context); Runtime.XIncref(pyHandle); } - protected override void OnLoad() + protected override void OnLoad(PyObjectSerializeContext context) { - base.OnLoad(); + base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); - Runtime.XDecref(pyHandle); } } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 75ca04205..79d78268b 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -98,9 +98,9 @@ public static void tp_dealloc(IntPtr ob) self.Dealloc(); } - protected override void OnLoad() + protected override void OnLoad(PyObjectSerializeContext context) { - base.OnLoad(); + base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Extension); Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); Runtime.PyObject_GC_UnTrack(pyHandle); diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 99e897570..dae7ebee7 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -200,18 +200,18 @@ protected void TypeClear() ClearObjectDict(pyHandle); } - internal void Save() + internal void Save(PyObjectSerializeContext context) { - OnSave(); + OnSave(context); } - internal void Load() + internal void Load(PyObjectSerializeContext context) { - OnLoad(); + OnLoad(context); } - protected virtual void OnSave() { } - protected virtual void OnLoad() { } + protected virtual void OnSave(PyObjectSerializeContext context) { } + protected virtual void OnLoad(PyObjectSerializeContext context) { } protected static void ClearObjectDict(IntPtr ob) { diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index c14e592d5..1a87c4d31 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -254,5 +254,12 @@ public static int tp_clear(IntPtr ob) self.ClearMembers(); return 0; } + + protected override void OnSave(PyObjectSerializeContext context) + { + base.OnSave(context); + Runtime.XIncref(target); + Runtime.XIncref(targetType); + } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 14fb0cd19..1e44f3270 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -18,7 +18,7 @@ internal class MethodObject : ExtensionType internal MethodBinding unbound; internal MethodBinder binder; internal bool is_static = false; - [NonSerialized] + internal IntPtr doc; internal Type type; @@ -221,5 +221,15 @@ public static int tp_clear(IntPtr ob) ClearObjectDict(ob); return 0; } + + protected override void OnSave(PyObjectSerializeContext context) + { + base.OnSave(context); + if (unbound != null) + { + Runtime.XIncref(unbound.pyHandle); + } + Runtime.XIncref(doc); + } } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 4a1a15ca8..1001cde13 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -14,13 +14,8 @@ namespace Python.Runtime [Serializable] internal class ModuleObject : ExtensionType { - [NonSerialized] private Dictionary cache; - [NonSerialized] - // FIXME: Used by reload mode, remove it after implement a delay load handler. - private bool _cacheInited; - internal string moduleName; internal IntPtr dict; protected string _namespace; @@ -33,7 +28,6 @@ public ModuleObject(string name) } moduleName = name; cache = new Dictionary(); - _cacheInited = true; _namespace = name; // Use the filename from any of the assemblies just so there's something for @@ -63,7 +57,7 @@ public ModuleObject(string name) Runtime.XDecref(pydocstring); Runtime.XIncref(dict); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); + SetObjectDict(pyHandle, dict); InitializeModuleMembers(); } @@ -77,11 +71,6 @@ public ModuleObject(string name) /// public ManagedType GetAttribute(string name, bool guess) { - if (!_cacheInited) - { - // XXX: Used by reload mode. - SetupCacheByDict(); - } ManagedType cached = null; cache.TryGetValue(name, out cached); if (cached != null) @@ -360,46 +349,24 @@ public static int tp_clear(IntPtr ob) return 0; } - protected override void OnSave() + protected override void OnSave(PyObjectSerializeContext context) { - base.OnSave(); + base.OnSave(context); System.Diagnostics.Debug.Assert(dict == GetObjectDict(pyHandle)); + foreach (var attr in cache.Values) + { + Runtime.XIncref(attr.pyHandle); + } // Decref twice in tp_clear, equilibrate them. Runtime.XIncref(dict); Runtime.XIncref(dict); } - protected override void OnLoad() + protected override void OnLoad(PyObjectSerializeContext context) { - base.OnLoad(); - // XXX: Set the cache after all objects loaded. - cache = new Dictionary(); - _cacheInited = false; + base.OnLoad(context); SetObjectDict(pyHandle, dict); } - - private void SetupCacheByDict() - { - System.Diagnostics.Debug.Assert(!_cacheInited); - _cacheInited = true; - IntPtr key, value; - IntPtr pos; - while (Runtime.PyDict_Next(dict, out pos, out key, out value) != 0) - { - ManagedType obj = GetManagedObject(value); - if (obj == null) - { - continue; - } - string name = Runtime.GetManagedString(key); - if (cache.ContainsKey(name)) - { - continue; - } - Runtime.XIncref(value); - cache.Add(name, obj); - } - } } /// diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 83bdb23ba..19eb80a18 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -82,7 +82,6 @@ internal static void Stash() XDecref(capsule); } - internal static void StashPop() { try @@ -112,14 +111,19 @@ private static void StashPopImpl() var objs = StashPopObjects(storage.GetStorage("objs")); StashPopModules(storage.GetStorage("modules")); - ClassManager.StashPop(storage.GetStorage("classes")); + var clsObjs = ClassManager.StashPop(storage.GetStorage("classes")); TypeManager.StashPop(storage.GetStorage("types")); ImportHook.StashPop(storage.GetStorage("import")); PyCLRMetaType = MetaType.StashPop(storage.GetStorage("meta")); foreach (var item in objs) { - XDecref(item.pyHandle); + item.Value.ExecutePostActions(); + XDecref(item.Key.pyHandle); + } + foreach (var item in clsObjs) + { + item.Value.ExecutePostActions(); } } @@ -139,6 +143,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) var extensionObjs = new List(); var wrappers = new Dictionary>(); var serializeObjs = new CLRWrapperCollection(); + var contexts = new Dictionary(); foreach (var entry in objs) { var obj = entry.Key; @@ -147,7 +152,9 @@ private static void StashPushObjects(RuntimeDataStorage storage) { case ManagedType.TrackTypes.Extension: Debug.Assert(obj.GetType().IsSerializable); - obj.Save(); + var context = new PyObjectSerializeContext(); + contexts[obj.pyHandle] = context; + obj.Save(context); extensionObjs.Add(obj); break; case ManagedType.TrackTypes.Wrapper: @@ -197,23 +204,28 @@ private static void StashPushObjects(RuntimeDataStorage storage) foreach (var clrObj in wrappers[item.Instance]) { XIncref(clrObj.pyHandle); - clrObj.Save(); + var context = new PyObjectSerializeContext(); + contexts[clrObj.pyHandle] = context; + clrObj.Save(context); } } storage.AddValue("internalStores", internalStores); storage.AddValue("extensions", extensionObjs); storage.AddValue("wrappers", wrapperStorage); + storage.AddValue("contexts", contexts); } - private static IEnumerable StashPopObjects(RuntimeDataStorage storage) + private static Dictionary StashPopObjects(RuntimeDataStorage storage) { var extensions = storage.GetValue>("extensions"); var internalStores = storage.GetValue>("internalStores"); - var storedObjs = new List(); + var contexts = storage.GetValue >("contexts"); + var storedObjs = new Dictionary(); foreach (var obj in Enumerable.Union(extensions, internalStores)) { - obj.Load(); - storedObjs.Add(obj); + var context = contexts[obj.pyHandle]; + obj.Load(context); + storedObjs.Add(obj, context); } if (WrappersStorer != null) { @@ -224,8 +236,9 @@ private static IEnumerable StashPopObjects(RuntimeDataStorage stora object obj = item.Instance; foreach (var handle in item.Handles) { - var co = CLRObject.Restore(obj, handle); - storedObjs.Add(co); + var context = contexts[handle]; + var co = CLRObject.Restore(obj, handle, context); + storedObjs.Add(co, context); } } } @@ -342,6 +355,37 @@ public T PopValue(out T value) } + [Serializable] + class PyObjectSerializeContext + { + private RuntimeDataStorage _storage; + public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); + + /// + /// Actions after loaded. + /// + [NonSerialized] + private List _postActions; + public List PostActions => _postActions ?? (_postActions = new List()); + + public void AddPostAction(Action action) + { + PostActions.Add(action); + } + + public void ExecutePostActions() + { + if (_postActions == null) + { + return; + } + foreach (var action in _postActions) + { + action(); + } + } + } + public class CLRMappedItem { public object Instance { get; private set; } From bcfdcc7001cc87817b92005e6ac373214bc42d04 Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 10 Mar 2020 12:05:13 +0800 Subject: [PATCH 055/134] Test for class object on crossed domain --- src/embed_tests/TestDomainReload.cs | 328 ++++++++++++++++++---------- 1 file changed, 216 insertions(+), 112 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 9e5f67d82..5f3c189d2 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -5,6 +5,7 @@ using NUnit.Framework; using Python.Runtime; +using PyRuntime = Python.Runtime.Runtime; // // This test case is disabled on .NET Standard because it doesn't have all the // APIs we use. We could work around that, but .NET Core doesn't implement @@ -17,6 +18,12 @@ namespace Python.EmbeddingTest { class TestDomainReload { + abstract class CrossCaller : MarshalByRefObject + { + public abstract ValueType Execte(ValueType arg); + } + + /// /// Test that the python runtime can survive a C# domain reload without crashing. /// @@ -53,71 +60,198 @@ public static void DomainReloadAndGC() { Assert.IsFalse(PythonEngine.IsInitialized); RunAssemblyAndUnload("test1"); - Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, + Assert.That(PyRuntime.Py_IsInitialized() != 0, "On soft-shutdown mode, Python runtime should still running"); RunAssemblyAndUnload("test2"); - Assert.That(Runtime.Runtime.Py_IsInitialized() != 0, + Assert.That(PyRuntime.Py_IsInitialized() != 0, "On soft-shutdown mode, Python runtime should still running"); if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) { // The default mode is a normal mode, // it should shutdown the Python VM avoiding influence other tests. - Runtime.Runtime.PyGILState_Ensure(); - Runtime.Runtime.Py_Finalize(); + PyRuntime.PyGILState_Ensure(); + PyRuntime.Py_Finalize(); } } - [Test] - public static void CrossDomainObject() + #region CrossDomainObject + + class CrossDomianObjectStep1 : CrossCaller { - IntPtr handle = IntPtr.Zero; - Type type = typeof(Proxy); + public override ValueType Execte(ValueType arg) { - AppDomain domain = CreateDomain("test_domain_reload"); try { - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - theProxy.Call("InitPython", ShutdownMode.Reload); - handle = (IntPtr)theProxy.Call("GetTestObject"); - theProxy.Call("ShutdownPython"); + Type type = typeof(Python.EmbeddingTest.Domain.MyClass); + string code = string.Format(@" +import clr +clr.AddReference('{0}') + +from Python.EmbeddingTest.Domain import MyClass +obj = MyClass() +obj.Method() +obj.StaticMethod() +obj.Property = 1 +obj.Field = 10 +", Assembly.GetExecutingAssembly().FullName); + + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(code); + using (PyObject obj = scope.Get("obj")) + { + Debug.Assert(obj.AsManagedObject(type).GetType() == type); + // We only needs its Python handle + PyRuntime.XIncref(obj.Handle); + return obj.Handle; + } + } } - finally + catch (Exception e) { - AppDomain.Unload(domain); + Debug.WriteLine(e); + throw; } } + } + + class CrossDomianObjectStep2 : CrossCaller + { + public override ValueType Execte(ValueType arg) { - AppDomain domain = CreateDomain("test_domain_reload"); + // handle refering a clr object created in previous domain, + // it should had been deserialized and became callable agian. + IntPtr handle = (IntPtr)arg; try { - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - theProxy.Call("InitPython", ShutdownMode.Reload); + using (Py.GIL()) + { + IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); + IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); - // handle refering a clr object created in previous domain, - // it should had been deserialized and became callable agian. - theProxy.Call("RunTestObject", handle); - theProxy.Call("ShutdownPythonCompletely"); + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + + using (var scope = Py.CreateScope()) + { + scope.Set("obj", obj); + scope.Exec(@" +obj.Method() +obj.StaticMethod() +obj.Property += 1 +obj.Field += 10 +"); + } + var clrObj = obj.As(); + Assert.AreEqual(clrObj.Property, 2); + Assert.AreEqual(clrObj.Field, 20); + } + } } - finally + catch (Exception e) { - AppDomain.Unload(domain); + Debug.WriteLine(e); + throw; } + return 0; } - if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + } + + [Test] + public static void CrossDomainObject() + { + RunDomainReloadSteps(); + } + + #endregion + + #region TestClassReference + + class ReloadClassRefStep1 : CrossCaller + { + public override ValueType Execte(ValueType arg) + { + const string code = @" +from Python.EmbeddingTest.Domain import MyClass + +def test_obj_call(): + obj = MyClass() + obj.Method() + obj.StaticMethod() + obj.Property = 1 + obj.Field = 10 + +test_obj_call() +"; + const string name = "test_domain_reload_mod"; + using (Py.GIL()) + { + IntPtr module = PyRuntime.PyModule_New(name); + Assert.That(module != IntPtr.Zero); + IntPtr globals = PyRuntime.PyObject_GetAttrString(module, "__dict__"); + Assert.That(globals != IntPtr.Zero); + try + { + int res = PyRuntime.PyDict_SetItemString(globals, "__builtins__", + PyRuntime.PyEval_GetBuiltins()); + PythonException.ThrowIfIsNotZero(res); + + PythonEngine.Exec(code, globals); + IntPtr modules = PyRuntime.PyImport_GetModuleDict(); + res = PyRuntime.PyDict_SetItemString(modules, name, modules); + PythonException.ThrowIfIsNotZero(res); + } + catch + { + PyRuntime.XDecref(module); + throw; + } + finally + { + PyRuntime.XDecref(globals); + } + return module; + } + } + } + + class ReloadClassRefStep2 : CrossCaller + { + public override ValueType Execte(ValueType arg) { - Assert.IsTrue(Runtime.Runtime.Py_IsInitialized() == 0); + var module = (IntPtr)arg; + using (Py.GIL()) + { + var test_obj_call = PyRuntime.PyObject_GetAttrString(module, "test_obj_call"); + PythonException.ThrowIfIsNull(test_obj_call); + var args = PyRuntime.PyTuple_New(0); + var res = PyRuntime.PyObject_CallObject(test_obj_call, args); + PythonException.ThrowIfIsNull(res); + + PyRuntime.XDecref(args); + PyRuntime.XDecref(res); + } + return 0; } } + [Test] + public void TestClassReference() + { + RunDomainReloadSteps(); + } + + #endregion + #region Tempary tests + // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 [Test] public void CrossReleaseBuiltinType() @@ -274,6 +408,58 @@ static Assembly ResolveAssembly(object sender, ResolveEventArgs args) return null; } + + static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : CrossCaller + { + ValueType arg = null; + Type type = typeof(Proxy); + { + AppDomain domain = CreateDomain("test_domain_reload"); + try + { + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = (T1)domain.CreateInstanceAndUnwrap( + typeof(T1).Assembly.FullName, + typeof(T1).FullName); + arg = caller.Execte(arg); + + theProxy.Call("ShutdownPython"); + } + finally + { + AppDomain.Unload(domain); + } + } + + { + AppDomain domain = CreateDomain("test_domain_reload"); + try + { + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = (T2)domain.CreateInstanceAndUnwrap( + typeof(T2).Assembly.FullName, + typeof(T2).FullName); + caller.Execte(arg); + theProxy.Call("ShutdownPythonCompletely"); + } + finally + { + AppDomain.Unload(domain); + } + } + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0); + } + } } @@ -358,88 +544,6 @@ public static void ShutdownPythonCompletely() PythonEngine.Shutdown(); } - public static IntPtr GetTestObject() - { - try - { - Type type = typeof(Python.EmbeddingTest.Domain.MyClass); - string code = string.Format(@" -import clr -clr.AddReference('{0}') - -from Python.EmbeddingTest.Domain import MyClass -obj = MyClass() -obj.Method() -obj.StaticMethod() -obj.Property = 1 -obj.Field = 10 -", Assembly.GetExecutingAssembly().FullName); - - using (Py.GIL()) - using (var scope = Py.CreateScope()) - { - scope.Exec(code); - using (PyObject obj = scope.Get("obj")) - { - Debug.Assert(obj.AsManagedObject(type).GetType() == type); - // We only needs its Python handle - Runtime.Runtime.XIncref(obj.Handle); - return obj.Handle; - } - } - } - catch (Exception e) - { - Debug.WriteLine(e); - throw; - } - } - - public static void RunTestObject(IntPtr handle) - { - try - { - using (Py.GIL()) - { - IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); - IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); - - using (PyObject obj = new PyObject(handle)) - { - obj.InvokeMethod("Method"); - obj.InvokeMethod("StaticMethod"); - - using (var scope = Py.CreateScope()) - { - scope.Set("obj", obj); - scope.Exec(@" -obj.Method() -obj.StaticMethod() -obj.Property += 1 -obj.Field += 10 -"); - } - var clrObj = obj.As(); - Assert.AreEqual(clrObj.Property, 2); - Assert.AreEqual(clrObj.Field, 20); - } - } - } - catch (Exception e) - { - Debug.WriteLine(e); - throw; - } - } - - public static void ReleaseTestObject(IntPtr handle) - { - using (Py.GIL()) - { - Runtime.Runtime.XDecref(handle); - } - } - static void OnDomainUnload(object sender, EventArgs e) { Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); From 66ab7192222e382faf6cff808a23964eb846fc34 Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 10 Mar 2020 13:14:13 +0800 Subject: [PATCH 056/134] Multi times for running cross dispose --- src/embed_tests/TestDomainReload.cs | 73 +++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 5f3c189d2..05b449237 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.Reflection; using System.Runtime.InteropServices; @@ -256,37 +257,81 @@ public void TestClassReference() [Test] public void CrossReleaseBuiltinType() { + void ExecTest() + { + try + { + var numRef = CreateNumReference(); + GC.Collect(); + GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue + Finalizer.Instance.Collect(forceDispose: true); + // ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`, + // but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead. + Assert.False(numRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) + { + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; try { - var numRef = CreateNumReference(); - GC.Collect(); - GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue - Finalizer.Instance.Collect(forceDispose: true); - // ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`, - // but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead. - Assert.False(numRef.IsAlive); + for (int i = 0; i < 10; i++) + { + ExecTest(); + } } finally { - PythonEngine.Shutdown(); + Finalizer.Instance.ErrorHandler -= ErrorHandler; } + Assert.AreEqual(errorArgs.Count, 0); } [Test] public void CrossReleaseCustomType() { + void ExecTest() + { + try + { + var objRef = CreateConcreateObject(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + Finalizer.Instance.Collect(forceDispose: true); + Assert.False(objRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) + { + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; try { - var objRef = CreateConcreateObject(); - GC.Collect(); - GC.WaitForPendingFinalizers(); - Finalizer.Instance.Collect(forceDispose: true); - Assert.False(objRef.IsAlive); + for (int i = 0; i < 10; i++) + { + ExecTest(); + } } finally { - PythonEngine.Shutdown(); + Finalizer.Instance.ErrorHandler -= ErrorHandler; } + Assert.AreEqual(errorArgs.Count, 0); } private static WeakReference CreateNumReference() From 8e3c0284d77805194abb3908fe9046e75baf6e9b Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 10 Mar 2020 15:44:47 +0800 Subject: [PATCH 057/134] Apply Reference type usage --- src/runtime/typemanager.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index b552e2cae..43181735b 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -891,7 +891,8 @@ public static IntPtr CreateObjectType() throw new PythonException(); } const string code = "class A(object): pass"; - IntPtr res = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + IntPtr res = resRef.DangerousGetAddress(); if (res == IntPtr.Zero) { try @@ -903,7 +904,7 @@ public static IntPtr CreateObjectType() Runtime.XDecref(globals); } } - Runtime.XDecref(res); + resRef.Dispose(); IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); Debug.Assert(A != IntPtr.Zero); Runtime.XIncref(A); From cb65af358a62d0e16cadaa0334e01ba249195d89 Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 16 Jun 2020 16:21:03 +0800 Subject: [PATCH 058/134] Manually merge - remove redundant code --- src/runtime/runtime.cs | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 8c59555f0..7d51e4088 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -418,35 +418,6 @@ internal static ShutdownMode GetDefaultShutdownMode() } #endif return ShutdownMode.Normal; -#else - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - OperatingSystem = OperatingSystemType.Linux; - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - OperatingSystem = OperatingSystemType.Darwin; - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - OperatingSystem = OperatingSystemType.Windows; - else - OperatingSystem = OperatingSystemType.Other; - - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X86: - Machine = MachineType.i386; - break; - case Architecture.X64: - Machine = MachineType.x86_64; - break; - case Architecture.Arm: - Machine = MachineType.armv7l; - break; - case Architecture.Arm64: - Machine = MachineType.aarch64; - break; - default: - Machine = MachineType.Other; - break; - } -#endif } // called *without* the GIL acquired by clr._AtExit @@ -2058,7 +2029,7 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type¡¯s base class. Return 0 on success, or return -1 and sets an exception on error. + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); From 498fc8cd89ad13623f56670961474a35deb7c51a Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 16 Jun 2020 16:25:36 +0800 Subject: [PATCH 059/134] Manually merge - ManagedDataOffsets --- src/runtime/interop.cs | 42 ++++++++++++++++++++++++++------------ src/runtime/managedtype.cs | 4 ++-- src/runtime/typemanager.cs | 6 +++--- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 1e2d97868..f5d4a750f 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -5,7 +5,6 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; -using System.Collections.Generic; using System.Linq; namespace Python.Runtime @@ -70,15 +69,20 @@ public ModulePropertyAttribute() } } - internal static partial class ManagedDataOffsets + internal static partial class TypeOffset { - static class Helper - { - public static int magic; - public static readonly Dictionary NameMapping = new Dictionary(); - } + public static int magic() => ManagedDataOffsets.Magic; + } - static TypeOffset() + internal static class ManagedDataOffsets + { + public static int Magic { get; private set; } + public static readonly Dictionary NameMapping = new Dictionary(); + + public static readonly int ob_data; + public static readonly int ob_dict; + + static ManagedDataOffsets() { Type type = typeof(TypeOffset); FieldInfo[] fields = type.GetFields(); @@ -88,18 +92,30 @@ static TypeOffset() int offset = i * size; FieldInfo fi = fields[i]; fi.SetValue(null, offset); - Helper.NameMapping[fi.Name] = offset; + NameMapping[fi.Name] = offset; } // XXX: Use the members after PyHeapTypeObject as magic slot - Helper.magic = members; + Magic = TypeOffset.members; } - public static int magic() => Helper.magic; - public static int GetSlotOffset(string name) { - return Helper.NameMapping[name]; + return NameMapping[name]; } + + private static int BaseOffset(IntPtr type) + { + Debug.Assert(type != IntPtr.Zero); + int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); + Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + return typeSize; + } + + public static int DataOffset(IntPtr type) + { + return BaseOffset(type) + ob_data; + } + public static int DictOffset(IntPtr type) { return BaseOffset(type) + ob_dict; diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 336c1b9ab..7aba72a4f 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -226,12 +226,12 @@ protected static void ClearObjectDict(IntPtr ob) protected static IntPtr GetObjectDict(IntPtr ob) { - return Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(ob)); } protected static void SetObjectDict(IntPtr ob, IntPtr value) { - Marshal.WriteIntPtr(ob, ObjectOffset.DictOffset(ob), value); + Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(ob), value); } } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 0f3aced01..2307077a2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -638,7 +638,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo { continue; } - var offset = TypeOffset.GetSlotOffset(slot); + var offset = ManagedDataOffsets.GetSlotOffset(slot); Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } } @@ -656,7 +656,7 @@ internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHo /// Can override the slot when it existed static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) { - var offset = TypeOffset.GetSlotOffset(name); + var offset = ManagedDataOffsets.GetSlotOffset(name); if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) { return; @@ -693,7 +693,7 @@ static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, Slots static bool IsSlotSet(IntPtr type, string name) { - int offset = TypeOffset.GetSlotOffset(name); + int offset = ManagedDataOffsets.GetSlotOffset(name); return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } From cc2219e8bc1e9ebe543ed22418cb8b34359f36bb Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 16 Jun 2020 16:33:03 +0800 Subject: [PATCH 060/134] Manually merge - capi prototype --- src/runtime/runtime.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7d51e4088..58c7ec655 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -899,9 +899,9 @@ public static extern int Py_Main( [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] #if PYTHON2 - internal static extern NewReference PyRun_String(string code, IntPtr st, IntPtr globals, IntPtr locals); + internal static extern NewReference PyRun_String(string code, RunFlagType st, IntPtr globals, IntPtr locals); #else - internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, IntPtr st, IntPtr globals, IntPtr locals); + internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, RunFlagType st, IntPtr globals, IntPtr locals); #endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] From 8c8d66ec57a576c43466743920701538ef91d9e2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 24 Jun 2020 16:36:03 +0800 Subject: [PATCH 061/134] * Move fields of ManagedDataOffsets into nested type * Remove assert condition which is `typeSize <= ExceptionOffset.Size()` cause of the type size of `PyHeapTypeObject` derived from clr type may over `ExceptionOffset.Size()` --- src/runtime/interop.cs | 49 ++++++++++++++++++++++++++++---------- src/runtime/managedtype.cs | 6 +++-- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index f5d4a750f..f1db8c8dc 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -71,6 +71,19 @@ public ModulePropertyAttribute() internal static partial class TypeOffset { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fields = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fields.Length; i++) + { + int offset = i * size; + FieldInfo fi = fields[i]; + fi.SetValue(null, offset); + } + } + public static int magic() => ManagedDataOffsets.Magic; } @@ -79,23 +92,33 @@ internal static class ManagedDataOffsets public static int Magic { get; private set; } public static readonly Dictionary NameMapping = new Dictionary(); - public static readonly int ob_data; - public static readonly int ob_dict; + static class DataOffsets + { + public static readonly int ob_data; + public static readonly int ob_dict; + + static DataOffsets() + { + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fields.Length; i++) + { + fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } + } + } static ManagedDataOffsets() { Type type = typeof(TypeOffset); - FieldInfo[] fields = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fields.Length; i++) + foreach (FieldInfo fi in type.GetFields()) { - int offset = i * size; - FieldInfo fi = fields[i]; - fi.SetValue(null, offset); - NameMapping[fi.Name] = offset; + NameMapping[fi.Name] = (int)fi.GetValue(null); } // XXX: Use the members after PyHeapTypeObject as magic slot Magic = TypeOffset.members; + + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + size = fields.Length * IntPtr.Size; } public static int GetSlotOffset(string name) @@ -107,20 +130,22 @@ private static int BaseOffset(IntPtr type) { Debug.Assert(type != IntPtr.Zero); int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + Debug.Assert(typeSize > 0); return typeSize; } public static int DataOffset(IntPtr type) { - return BaseOffset(type) + ob_data; + return BaseOffset(type) + DataOffsets.ob_data; } public static int DictOffset(IntPtr type) { - return BaseOffset(type) + ob_dict; + return BaseOffset(type) + DataOffsets.ob_dict; } + public static int ob_data => DataOffsets.ob_data; + public static int ob_dict => DataOffsets.ob_dict; public static int Size { get { return size; } } private static readonly int size; diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 7aba72a4f..a3b50c52f 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -226,12 +226,14 @@ protected static void ClearObjectDict(IntPtr ob) protected static IntPtr GetObjectDict(IntPtr ob) { - return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(ob)); + IntPtr type = Runtime.PyObject_TYPE(ob); + return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); } protected static void SetObjectDict(IntPtr ob, IntPtr value) { - Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(ob), value); + IntPtr type = Runtime.PyObject_TYPE(ob); + Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); } } } From 4f00165874d5094c9e5c4a7b80865e7f1944707e Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 24 Jun 2020 16:43:22 +0800 Subject: [PATCH 062/134] Avoid mess up the debug info of runtime module by emit IL code --- src/runtime/classderived.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 7f8462897..e55e89240 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -135,7 +135,7 @@ internal static Type CreateDerivedType(string name, if (null == assemblyName) { - assemblyName = Assembly.GetExecutingAssembly().FullName; + assemblyName = "Python.Runtime.Dynamic"; } ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); From da7c1500fdc9d67258cf56a34338c2cf79511980 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 24 Jun 2020 22:40:22 +0800 Subject: [PATCH 063/134] * Fix syntax error * Test soft shutdown on Python 3.8 --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 716c2ea35..5ee100f6a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,10 +28,10 @@ environment: - PYTHON_VERSION: 3.7 - PYTHON_VERSION: 3.6 - PYTHON_VERSION: 3.5 - -matrix: - PYTHON_VERSION: 3.7 PYTHONNET_SOFT_SHUTDOWN: 1 + - PYTHON_VERSION: 3.8 + PYTHONNET_SOFT_SHUTDOWN: 1 init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% From a8840b25756b087a21247ac853d4159d6f43d24d Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 2 Jul 2020 15:01:20 +0800 Subject: [PATCH 064/134] Drop Python 2.7 on CI --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5ee100f6a..7ea0226f1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,8 +16,6 @@ environment: matrix: - PYTHON_VERSION: 3.8 - - PYTHON_VERSION: 2.7 - BUILD_OPTS: --xplat - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 From dec7a74e9bc593294c1a2b1cd6614b984d9ea017 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 11 Jul 2020 09:56:31 +0800 Subject: [PATCH 065/134] Fix refcnt error --- src/runtime/runtime_state.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 9c3823315..5ab2e632d 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -146,6 +146,7 @@ private static void RestoreObjects(IntPtr dummyGC) public static IEnumerable PyGCGetObjects() { var gc = PyImport_ImportModule("gc"); + PythonException.ThrowIfIsNull(gc); var get_objects = PyObject_GetAttrString(gc, "get_objects"); var objs = PyObject_CallObject(get_objects, IntPtr.Zero); var length = PyList_Size(objs); @@ -168,6 +169,7 @@ public static IEnumerable GetModuleNames() var name = PyList_GetItem(names, i); yield return name.DangerousGetAddress(); } + XDecref(names); } private static void AddObjPtrToSet(IntPtr set, IntPtr obj) From e877b3328cb347c961ea6cb002557b1bdcd473e1 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 11 Jul 2020 16:06:54 +0800 Subject: [PATCH 066/134] Get platform information without import `platform` --- src/embed_tests/TestRuntime.cs | 1 + src/runtime/platform/NativeCodePage.cs | 52 ++------------------------ 2 files changed, 4 insertions(+), 49 deletions(-) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 38878205c..c55813700 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -23,6 +23,7 @@ public void SetUp() /// /// Test fails on platforms we haven't implemented yet. /// + [Ignore("Temparary test")] [Test] public static void PlatformCache() { diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs index 97cc7e9c2..28fdb3e16 100644 --- a/src/runtime/platform/NativeCodePage.cs +++ b/src/runtime/platform/NativeCodePage.cs @@ -246,55 +246,9 @@ public void SetReadExec(IntPtr mappedMemory, int numBytes) /// public static void InitializePlatformData() { - IntPtr op = IntPtr.Zero; - IntPtr fn = IntPtr.Zero; - IntPtr platformModule = IntPtr.Zero; - IntPtr emptyTuple = IntPtr.Zero; - try - { - platformModule = Runtime.PyImport_ImportModule("platform"); - PythonException.ThrowIfIsNull(platformModule); - - fn = Runtime.PyObject_GetAttrString(platformModule, "system"); - PythonException.ThrowIfIsNull(fn); - - emptyTuple = Runtime.PyTuple_New(0); - op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); - PythonException.ThrowIfIsNull(op); - - OperatingSystemName = Runtime.GetManagedString(op); - - fn = Runtime.PyObject_GetAttrString(platformModule, "machine"); - PythonException.ThrowIfIsNull(fn); - - op = Runtime.PyObject_Call(fn, emptyTuple, IntPtr.Zero); - PythonException.ThrowIfIsNull(op); - MachineName = Runtime.GetManagedString(op); - } - finally - { - Runtime.XDecref(op); - Runtime.XDecref(fn); - - Runtime.XDecref(emptyTuple); - Runtime.XDecref(platformModule); - } - - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) - { - OSType = OperatingSystemType.Other; - } - OperatingSystem = OSType; - - MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) - { - MType = MachineType.Other; - } - Machine = MType; + // FIXME: arch, non-windows + Machine = Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; + OperatingSystem = Runtime.IsWindows ? OperatingSystemType.Windows : OperatingSystemType.Linux; } internal static IMemoryMapper CreateMemoryMapper() From 6d738bf53a1e4b47f8d575858c5112b60fe587d2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 12 Jul 2020 01:30:28 +0800 Subject: [PATCH 067/134] Run callbacks registered by `atexit` at Shutdown on soft-shutdown mode --- src/embed_tests/pyinitialize.cs | 41 +++++++++++++++++++++++++++++++++ src/runtime/pythonexception.cs | 5 ++++ src/runtime/runtime.cs | 34 ++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 69ed127bd..30e4ebc9b 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -136,5 +136,46 @@ public void ShutdownHandlers() // Wrong: (4 * 2) + 1 + 1 + 1 = 11 Assert.That(shutdown_count, Is.EqualTo(12)); } + + [Test] + public static void TestRunExitFuncs() + { + if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal) + { + // If the runtime using the normal mode, + // callback registered by atexit will be called after we release the clr information, + // thus there's no chance we can check it here. + Assert.Ignore("Skip on normal mode"); + } + Runtime.Runtime.Initialize(); + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + string msg = e.ToString(); + Runtime.Runtime.Shutdown(); + + if (e.IsMatches(Exceptions.ImportError)) + { + Assert.Ignore("no atexit module"); + } + else + { + Assert.Fail(msg); + } + return; + } + bool called = false; + Action callback = () => + { + called = true; + }; + atexit.InvokeMethod("register", callback.ToPython()); + Runtime.Runtime.Shutdown(); + Assert.True(called); + } } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 8efdccc91..8f75501af 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -187,6 +187,11 @@ public string Format() return res; } + public bool IsMatches(IntPtr exc) + { + return Runtime.PyErr_GivenExceptionMatches(PyType, exc) != 0; + } + /// /// Dispose Method /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 0df7a9b26..e7d702e66 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -317,6 +317,10 @@ internal static void Shutdown() PyGILState_Ensure(); var mode = ShutdownMode; + if (mode != ShutdownMode.Normal) + { + RunExitFuncs(); + } #if !NETSTANDARD if (mode == ShutdownMode.Reload) { @@ -388,6 +392,34 @@ internal static int AtExit() return 0; } + private static void RunExitFuncs() + { + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + if (!e.IsMatches(Exceptions.ImportError)) + { + throw; + } + e.Dispose(); + // The runtime may not provided `atexit` module. + return; + } + try + { + atexit.InvokeMethod("_run_exitfuncs").Dispose(); + } + catch (PythonException e) + { + Console.Error.WriteLine(e); + e.Dispose(); + } + } + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) { // XXX: For current usages, value should not be null. @@ -1849,7 +1881,7 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type¡¯s base class. Return 0 on success, or return -1 and sets an exception on error. + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); From ff5edc30c0844d584ad2f84c4697a49a01138073 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 12 Jul 2020 03:53:02 +0800 Subject: [PATCH 068/134] Use named shutdown-mode value by environment value instead of simple switch --- .travis.yml | 4 ++-- appveyor.yml | 4 ++-- src/runtime/runtime.cs | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index c164cd603..00bc4c4c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,8 @@ env: matrix: - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" - - PYTHONNET_SOFT_SHUTDOWN="1" BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ - - PYTHONNET_SOFT_SHUTDOWN="1" BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" + - PYTHONNET_SHUTDOWN_MODE="Soft" BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ + - PYTHONNET_SHUTDOWN_MODE="Soft" BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so diff --git a/appveyor.yml b/appveyor.yml index 7ea0226f1..2af08c4cd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,9 +27,9 @@ environment: - PYTHON_VERSION: 3.6 - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.7 - PYTHONNET_SOFT_SHUTDOWN: 1 + PYTHONNET_SHUTDOWN_MODE: Soft - PYTHON_VERSION: 3.8 - PYTHONNET_SOFT_SHUTDOWN: 1 + PYTHONNET_SHUTDOWN_MODE: Soft init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e7d702e66..fc4f20f67 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -369,16 +369,16 @@ internal static void Shutdown() internal static ShutdownMode GetDefaultShutdownMode() { - if (Environment.GetEnvironmentVariable("PYTHONNET_SOFT_SHUTDOWN") == "1") + string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); + if (modeEvn == null) { - return ShutdownMode.Soft; + return ShutdownMode.Normal; } -#if !NETSTANDARD - else if (Environment.GetEnvironmentVariable("PYTHONNET_RELOAD_SHUTDOWN") == "1") + ShutdownMode mode; + if (Enum.TryParse(modeEvn, true, out mode)) { - return ShutdownMode.Reload; + return mode; } -#endif return ShutdownMode.Normal; } From 5ac75ba7f3653e43ce7e2c2c74c37d6589b9f564 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 15 Jul 2020 16:22:21 +0800 Subject: [PATCH 069/134] Remove dependency on importing `platform`. #891 --- src/embed_tests/TestRuntime.cs | 1 - src/runtime/platform/NativeCodePage.cs | 8 +- src/runtime/platform/Types.cs | 169 +++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 5 deletions(-) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index c55813700..38878205c 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -23,7 +23,6 @@ public void SetUp() /// /// Test fails on platforms we haven't implemented yet. /// - [Ignore("Temparary test")] [Test] public static void PlatformCache() { diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs index 28fdb3e16..ab2ee3bcf 100644 --- a/src/runtime/platform/NativeCodePage.cs +++ b/src/runtime/platform/NativeCodePage.cs @@ -15,7 +15,7 @@ class NativeCodePageHelper /// Gets the operating system as reported by python's platform.system(). /// [Obsolete] - public static string OperatingSystemName { get; private set; } + public static string OperatingSystemName => PythonEngine.Platform; /// /// Gets the machine architecture as reported by python's platform.machine(). @@ -246,9 +246,9 @@ public void SetReadExec(IntPtr mappedMemory, int numBytes) /// public static void InitializePlatformData() { - // FIXME: arch, non-windows - Machine = Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; - OperatingSystem = Runtime.IsWindows ? OperatingSystemType.Windows : OperatingSystemType.Linux; + MachineName = SystemInfo.GetArchitecture(); + Machine = SystemInfo.GetMachineType(); + OperatingSystem = SystemInfo.GetSystemType(); } internal static IMemoryMapper CreateMemoryMapper() diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs index 62be0e421..15235da5a 100644 --- a/src/runtime/platform/Types.cs +++ b/src/runtime/platform/Types.cs @@ -1,3 +1,6 @@ +using System; +using System.Runtime.InteropServices; + namespace Python.Runtime.Platform { public enum MachineType @@ -20,4 +23,170 @@ public enum OperatingSystemType Linux, Other } + + + static class SystemInfo + { + public static MachineType GetMachineType() + { + return Runtime.IsWindows ? GetMachineType_Windows() : GetMachineType_Unix(); + } + + public static string GetArchitecture() + { + return Runtime.IsWindows ? GetArchName_Windows() : GetArchName_Unix(); + } + + public static OperatingSystemType GetSystemType() + { + if (Runtime.IsWindows) + { + return OperatingSystemType.Windows; + } + switch (PythonEngine.Platform) + { + case "linux": + return OperatingSystemType.Linux; + + case "darwin": + return OperatingSystemType.Darwin; + + default: + return OperatingSystemType.Other; + } + } + + #region WINDOWS + + static string GetArchName_Windows() + { + // https://docs.microsoft.com/en-us/windows/win32/winprog64/wow64-implementation-details + return Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE"); + } + + static MachineType GetMachineType_Windows() + { + if (Runtime.Is32Bit) + { + return MachineType.i386; + } + switch (GetArchName_Windows()) + { + case "AMD64": + return MachineType.x86_64; + case "ARM64": + return MachineType.aarch64; + default: + return MachineType.Other; + } + } + + #endregion + + #region UNIX + + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_linux + { + const int NameLength = 65; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + + // GNU extension + fixed byte domainname[NameLength]; /* NIS or YP domain name */ + } + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_darwin + { + const int NameLength = 256; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + } + + [DllImport("libc")] + static extern int uname(IntPtr buf); + + + static unsafe string GetArchName_Unix() + { + switch (GetSystemType()) + { + case OperatingSystemType.Linux: + { + var buf = stackalloc utsname_linux[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + case OperatingSystemType.Darwin: + { + var buf = stackalloc utsname_darwin[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + default: + return null; + } + } + + static unsafe MachineType GetMachineType_Unix() + { + switch (GetArchName_Unix()) + { + case "x86_64": + case "em64t": + return Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; + case "i386": + case "i686": + return MachineType.i386; + + case "armv7l": + return MachineType.armv7l; + case "armv8": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.armv8; + case "aarch64": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.aarch64; + + default: + return MachineType.Other; + } + } + + #endregion + } } From 65cb22e973892d904862936e103b9e8a84ccb651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 14 Aug 2020 13:25:09 -0400 Subject: [PATCH 070/134] Don't call exit functions on soft or reload shutdown This fixes the exit functions being called during soft shutdown or reload shutdown. Also a small improvement to readability in Runtime.Shutdown. --- src/runtime/runtime.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index fc4f20f67..28a299dae 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -317,7 +317,7 @@ internal static void Shutdown() PyGILState_Ensure(); var mode = ShutdownMode; - if (mode != ShutdownMode.Normal) + if (mode == ShutdownMode.Normal) { RunExitFuncs(); } @@ -361,10 +361,12 @@ internal static void Shutdown() // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. } PyEval_SaveThread(); - return; } - ResetPyMembers(); - Py_Finalize(); + else + { + ResetPyMembers(); + Py_Finalize(); + } } internal static ShutdownMode GetDefaultShutdownMode() From 73865d45bb60db49bf635fccb64a2ecf190e4f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 11:49:38 -0400 Subject: [PATCH 071/134] Adding to AUTHORS.MD --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index c6e590523..c4c3d0599 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -33,6 +33,7 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) From 1a75f51037928959f03133bec516e9f71bc604e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 14:33:36 -0400 Subject: [PATCH 072/134] Code review fixes --- src/runtime/runtime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 28a299dae..b95d4af1e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -317,7 +317,7 @@ internal static void Shutdown() PyGILState_Ensure(); var mode = ShutdownMode; - if (mode == ShutdownMode.Normal) + if (mode == ShutdownMode.Soft) { RunExitFuncs(); } From d9d5562950f77bfdd5ce5e1e14a4aa014dc7ce2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 12:06:32 -0400 Subject: [PATCH 073/134] Remove unused code Addresses comment: amos402: Hummm, seems I only use it in my previous version, I will check it latter. https://github.com/pythonnet/pythonnet/pull/958/#discussion_r343815280 --- src/runtime/classmanager.cs | 5 ----- src/runtime/typemanager.cs | 5 ----- 2 files changed, 10 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index a7d2d74e2..08fe7e6c7 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -39,11 +39,6 @@ public static void Reset() cache = new Dictionary(128); } - public static IList GetManagedTypes() - { - return cache.Values.ToArray(); // Make a copy. - } - internal static void RemoveClasses() { var visited = new HashSet(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e83ea6df0..6e4c87b92 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -41,11 +41,6 @@ internal static void Initialize() Runtime.XDecref(type); } - public static IList GetManagedTypes() - { - return cache.Values.ToArray(); - } - internal static void RemoveTypes() { foreach (var tpHandle in cache.Values) From 9b62a61de5698b6eddb668a1a6822842489d15b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 13:17:32 -0400 Subject: [PATCH 074/134] Fixes some typos in TestDomainReload Addresses comments: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390639971 https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390640042 https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390643563 --- src/embed_tests/TestDomainReload.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 05b449237..563a0fab6 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -21,7 +21,7 @@ class TestDomainReload { abstract class CrossCaller : MarshalByRefObject { - public abstract ValueType Execte(ValueType arg); + public abstract ValueType Execute(ValueType arg); } @@ -79,9 +79,9 @@ public static void DomainReloadAndGC() #region CrossDomainObject - class CrossDomianObjectStep1 : CrossCaller + class CrossDomainObjectStep1 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { try { @@ -120,9 +120,9 @@ from Python.EmbeddingTest.Domain import MyClass } - class CrossDomianObjectStep2 : CrossCaller + class CrossDomainObjectStep2 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { // handle refering a clr object created in previous domain, // it should had been deserialized and became callable agian. @@ -167,7 +167,7 @@ public override ValueType Execte(ValueType arg) [Test] public static void CrossDomainObject() { - RunDomainReloadSteps(); + RunDomainReloadSteps(); } #endregion @@ -176,7 +176,7 @@ public static void CrossDomainObject() class ReloadClassRefStep1 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { const string code = @" from Python.EmbeddingTest.Domain import MyClass @@ -224,7 +224,7 @@ def test_obj_call(): class ReloadClassRefStep2 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { var module = (IntPtr)arg; using (Py.GIL()) @@ -470,7 +470,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro var caller = (T1)domain.CreateInstanceAndUnwrap( typeof(T1).Assembly.FullName, typeof(T1).FullName); - arg = caller.Execte(arg); + arg = caller.Execute(arg); theProxy.Call("ShutdownPython"); } @@ -492,7 +492,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro var caller = (T2)domain.CreateInstanceAndUnwrap( typeof(T2).Assembly.FullName, typeof(T2).FullName); - caller.Execte(arg); + caller.Execute(arg); theProxy.Call("ShutdownPythonCompletely"); } finally From 4f0420e77b187ffe1b7af07a43a4025cb26717db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 14:15:11 -0400 Subject: [PATCH 075/134] Adds missing assert Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390644837 --- src/embed_tests/TestDomainReload.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 563a0fab6..f9a8d321f 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -133,6 +133,7 @@ public override ValueType Execute(ValueType arg) { IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); + Assert.That(tp_clear, Is.Not.Null); using (PyObject obj = new PyObject(handle)) { From 4ab9f1ca0fa3c83cf610ab1f52ad22e13a8e234e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 14:16:32 -0400 Subject: [PATCH 076/134] Adds code comments to CrossDomainObject test Addresses comment: lostmu: Add a comment explaining the intent of this method or extract this code into a separate method with clear name. https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390644281 --- src/embed_tests/TestDomainReload.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index f9a8d321f..959eac600 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -85,6 +85,7 @@ public override ValueType Execute(ValueType arg) { try { + // Create a C# user-defined object in Python. Asssing some values. Type type = typeof(Python.EmbeddingTest.Domain.MyClass); string code = string.Format(@" import clr @@ -165,6 +166,11 @@ public override ValueType Execute(ValueType arg) } } + /// + /// Create a C# custom object in a domain, in python code. + /// Unload the domain, create a new domain. + /// Make sure the C# custom object created in the previous domain has been re-created + /// [Test] public static void CrossDomainObject() { From 32bcb3a018602b7a97a42cea6676751e0e6e0d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 14:58:05 -0400 Subject: [PATCH 077/134] Adds documentation to TestDomainReload.TestClassReference Addresses comment: lostmu: BTW, it would be helpful to add comments with equivalent Python code like [...] https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390650653 --- src/embed_tests/TestDomainReload.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 959eac600..bd587c65f 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -200,19 +200,26 @@ def test_obj_call(): const string name = "test_domain_reload_mod"; using (Py.GIL()) { + // Create a new module IntPtr module = PyRuntime.PyModule_New(name); Assert.That(module != IntPtr.Zero); IntPtr globals = PyRuntime.PyObject_GetAttrString(module, "__dict__"); Assert.That(globals != IntPtr.Zero); try { + // import builtins + // module.__dict__[__builtins__] = builtins int res = PyRuntime.PyDict_SetItemString(globals, "__builtins__", PyRuntime.PyEval_GetBuiltins()); PythonException.ThrowIfIsNotZero(res); + // Execute the code in the module's scope PythonEngine.Exec(code, globals); + // import sys + // modules = sys.modules IntPtr modules = PyRuntime.PyImport_GetModuleDict(); - res = PyRuntime.PyDict_SetItemString(modules, name, modules); + // modules[name] = module + res = PyRuntime.PyDict_SetItemString(modules, name, module); PythonException.ThrowIfIsNotZero(res); } catch @@ -251,6 +258,11 @@ public override ValueType Execute(ValueType arg) [Test] + /// + /// Create a new Python module, define a function in it. + /// Unload the domain, load a new one. + /// Make sure the function (and module) still exists. + /// public void TestClassReference() { RunDomainReloadSteps(); From 0077ea81bff24c7601dec12577fe3b5705a1987c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 16:02:02 -0400 Subject: [PATCH 078/134] Adds numbering to domain names in TestDomainReload Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390655505 --- src/embed_tests/TestDomainReload.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index bd587c65f..30eeae05a 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -478,7 +478,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro ValueType arg = null; Type type = typeof(Proxy); { - AppDomain domain = CreateDomain("test_domain_reload"); + AppDomain domain = CreateDomain("test_domain_reload_1"); try { var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( @@ -500,7 +500,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro } { - AppDomain domain = CreateDomain("test_domain_reload"); + AppDomain domain = CreateDomain("test_domain_reload_2"); try { var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( From 38ea0b62298e47b56a21b453345a534acc787010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 16:15:29 -0400 Subject: [PATCH 079/134] Use Console.WriteLine formatting overload Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390656145 --- src/embed_tests/TestDomainReload.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 30eeae05a..9538d6444 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -542,7 +542,7 @@ public static void RunPython() { AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format("[{0} in .NET] In PythonRunner.RunPython", name)); + Console.WriteLine("[{0} in .NET] In PythonRunner.RunPython", name); var mode = PythonEngine.DefaultShutdownMode; if (mode == ShutdownMode.Normal) { From 06a656e5e52b50a920437fb144ded6b574c50d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 19 Aug 2020 15:58:45 -0400 Subject: [PATCH 080/134] Inline called-once methods in TestDomainReload Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390653941 --- src/embed_tests/TestDomainReload.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 9538d6444..257bc0634 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -280,7 +280,12 @@ void ExecTest() { try { + PythonEngine.Initialize(); var numRef = CreateNumReference(); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + Assert.True(numRef.IsAlive); + GC.Collect(); GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue Finalizer.Instance.Collect(forceDispose: true); @@ -321,7 +326,11 @@ void ExecTest() { try { + PythonEngine.Initialize(); var objRef = CreateConcreateObject(); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + Assert.True(objRef.IsAlive); GC.Collect(); GC.WaitForPendingFinalizers(); Finalizer.Instance.Collect(forceDispose: true); @@ -355,25 +364,17 @@ void ErrorHandler(object sender, Finalizer.ErrorArgs e) private static WeakReference CreateNumReference() { - PythonEngine.Initialize(); var num = 3216757418.ToPython(); Assert.AreEqual(num.Refcount, 1); WeakReference numRef = new WeakReference(num, false); - PythonEngine.Shutdown(); // <- "run" 1 ends - PythonEngine.Initialize(); // <- "run" 2 starts - num = null; return numRef; } private static WeakReference CreateConcreateObject() { - PythonEngine.Initialize(); var obj = new Domain.MyClass().ToPython(); Assert.AreEqual(obj.Refcount, 1); WeakReference numRef = new WeakReference(obj, false); - PythonEngine.Shutdown(); - PythonEngine.Initialize(); - obj = null; return numRef; } From 09f828196d869f772ccbcf81fbfe08086481511d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 11:47:37 -0400 Subject: [PATCH 081/134] Simplify PythonRunner.RunPython Addresses comments: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390656459 https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390657130 --- src/embed_tests/TestDomainReload.cs | 47 +++++++++++------------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 257bc0634..87b842358 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -417,9 +417,11 @@ static void RunAssemblyAndUnload(string domainName) type.Assembly.FullName, type.FullName); + theProxy.Call("InitPython", ShutdownMode.Soft); // From now on use the Proxy to call into the new assembly theProxy.RunPython(); + theProxy.Call("ShutdownPython"); Console.WriteLine($"[Program.Main] Before Domain Unload on {domainName}"); AppDomain.Unload(domain); Console.WriteLine($"[Program.Main] After Domain Unload on {domainName}"); @@ -544,38 +546,25 @@ public static void RunPython() AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; string name = AppDomain.CurrentDomain.FriendlyName; Console.WriteLine("[{0} in .NET] In PythonRunner.RunPython", name); - var mode = PythonEngine.DefaultShutdownMode; - if (mode == ShutdownMode.Normal) + using (Py.GIL()) { - mode = ShutdownMode.Soft; - } - PythonEngine.Initialize(mode: mode); - try - { - using (Py.GIL()) + try { - try - { - var pyScript = string.Format("import clr\n" - + "print('[{0} in python] imported clr')\n" - + "clr.AddReference('System')\n" - + "print('[{0} in python] allocated a clr object')\n" - + "import gc\n" - + "gc.collect()\n" - + "print('[{0} in python] collected garbage')\n", - name); - PythonEngine.Exec(pyScript); - } - catch (Exception e) - { - Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); - throw; - } + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + throw; } - } - finally - { - PythonEngine.BeginAllowThreads(); } } From 7ec9a6c677cd346afd3b75fe8022f51e4e076809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 12:00:46 -0400 Subject: [PATCH 082/134] Release the GIL Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390659969 --- src/embed_tests/TestRuntime.cs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 38878205c..707d2a467 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -40,16 +40,27 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { - if (Runtime.Runtime.Py_IsInitialized() == 1) + IntPtr state = IntPtr.Zero; + try { - Runtime.Runtime.PyGILState_Ensure(); + if (Runtime.Runtime.Py_IsInitialized() == 1) + { + state = Runtime.Runtime.PyGILState_Ensure(); + } + Runtime.Runtime.Py_Finalize(); + Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); + Runtime.Runtime.Py_Initialize(); + Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); + Runtime.Runtime.Py_Finalize(); + Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); + } + finally + { + if(state != IntPtr.Zero) + { + Runtime.Runtime.PyGILState_Release(state); + } } - Runtime.Runtime.Py_Finalize(); - Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); - Runtime.Runtime.Py_Initialize(); - Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); - Runtime.Runtime.Py_Finalize(); - Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); } [Test] From 802a43abc47c703927ef59da1a51149155ba565a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 13:29:47 -0400 Subject: [PATCH 083/134] BorrowReference instead of increasing the refcount Addresses comments: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390660744 https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390660804 --- src/embed_tests/pyinitialize.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 30e4ebc9b..6e539609b 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -24,10 +24,12 @@ public static void StartAndStopTwice() public static void LoadDefaultArgs() { using (new PythonEngine()) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Runtime.Runtime.XIncref(argv.Handle); - Assert.AreNotEqual(0, argv.Length()); + var argvref = new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")); + using(var argv = new PyList(argvref.DangerousGetAddress())) + { + Assert.AreNotEqual(0, argv.Length()); + } } } @@ -36,11 +38,13 @@ public static void LoadSpecificArgs() { var args = new[] { "test1", "test2" }; using (new PythonEngine(args)) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Runtime.Runtime.XIncref(argv.Handle); - Assert.AreEqual(args[0], argv[0].ToString()); - Assert.AreEqual(args[1], argv[1].ToString()); + var argvref = new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")); + using(var argv = new PyList(argvref.DangerousGetAddress())) + { + Assert.AreEqual(args[0], argv[0].ToString()); + Assert.AreEqual(args[1], argv[1].ToString()); + } } } From 0fdf969edc762d5b249b7894afbc0c8f471e5900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 14:18:00 -0400 Subject: [PATCH 084/134] Add IsTypeObject to Managed type Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390663522 --- src/runtime/classbase.cs | 2 +- src/runtime/managedtype.cs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5bae9b350..0c091475c 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -297,7 +297,7 @@ public static void tp_dealloc(IntPtr ob) public static int tp_clear(IntPtr ob) { ManagedType self = GetManagedObject(ob); - if (self.pyHandle != self.tpHandle) + if (!self.IsTypeObject()) { ClearObjectDict(ob); } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index a3b50c52f..ac321b375 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -136,6 +136,11 @@ internal static bool IsManagedType(IntPtr ob) return false; } + public bool IsTypeObject() + { + return pyHandle == tpHandle; + } + internal static IDictionary GetManagedObjects() { return _managedObjs; From 3a8c72d7d770bdee435f72aa646a26827a4711d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 14:31:48 -0400 Subject: [PATCH 085/134] Rename PyObjectSerializeContext to InterDomainContext Addresses Comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390664306 --- src/runtime/classbase.cs | 4 ++-- src/runtime/classmanager.cs | 10 +++++----- src/runtime/clrobject.cs | 6 +++--- src/runtime/extensiontype.cs | 2 +- src/runtime/managedtype.cs | 8 ++++---- src/runtime/methodbinding.cs | 2 +- src/runtime/methodobject.cs | 2 +- src/runtime/moduleobject.cs | 4 ++-- src/runtime/runtime_data.cs | 14 +++++++------- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 0c091475c..f26079de2 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -305,7 +305,7 @@ public static int tp_clear(IntPtr ob) return 0; } - protected override void OnSave(PyObjectSerializeContext context) + protected override void OnSave(InterDomainContext context) { base.OnSave(context); if (pyHandle != tpHandle) @@ -316,7 +316,7 @@ protected override void OnSave(PyObjectSerializeContext context) } } - protected override void OnLoad(PyObjectSerializeContext context) + protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); if (pyHandle != tpHandle) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 08fe7e6c7..0adc61589 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -80,23 +80,23 @@ private static int OnVisit(IntPtr ob, IntPtr arg) internal static void StashPush(RuntimeDataStorage storage) { var contexts = storage.AddValue("contexts", - new Dictionary()); + new Dictionary()); storage.AddValue("cache", cache); foreach (var cls in cache.Values) { // This incref is for cache to hold the cls, // thus no need for decreasing it at StashPop. Runtime.XIncref(cls.pyHandle); - var context = contexts[cls.pyHandle] = new PyObjectSerializeContext(); + var context = contexts[cls.pyHandle] = new InterDomainContext(); cls.Save(context); } } - internal static Dictionary StashPop(RuntimeDataStorage storage) + internal static Dictionary StashPop(RuntimeDataStorage storage) { cache = storage.GetValue>("cache"); - var contexts = storage.GetValue >("contexts"); - var loadedObjs = new Dictionary(); + var contexts = storage.GetValue >("contexts"); + var loadedObjs = new Dictionary(); foreach (var cls in cache.Values) { var context = contexts[cls.pyHandle]; diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 3d29e04d4..0b62fecba 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -73,7 +73,7 @@ internal static IntPtr GetInstHandle(object ob) return co.pyHandle; } - internal static CLRObject Restore(object ob, IntPtr pyHandle, PyObjectSerializeContext context) + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) { CLRObject co = new CLRObject() { @@ -85,13 +85,13 @@ internal static CLRObject Restore(object ob, IntPtr pyHandle, PyObjectSerializeC return co; } - protected override void OnSave(PyObjectSerializeContext context) + protected override void OnSave(InterDomainContext context) { base.OnSave(context); Runtime.XIncref(pyHandle); } - protected override void OnLoad(PyObjectSerializeContext context) + protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 79d78268b..003e27a56 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -98,7 +98,7 @@ public static void tp_dealloc(IntPtr ob) self.Dealloc(); } - protected override void OnLoad(PyObjectSerializeContext context) + protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); GCHandle gc = AllocGCHandle(TrackTypes.Extension); diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index ac321b375..1190fb871 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -205,18 +205,18 @@ protected void TypeClear() ClearObjectDict(pyHandle); } - internal void Save(PyObjectSerializeContext context) + internal void Save(InterDomainContext context) { OnSave(context); } - internal void Load(PyObjectSerializeContext context) + internal void Load(InterDomainContext context) { OnLoad(context); } - protected virtual void OnSave(PyObjectSerializeContext context) { } - protected virtual void OnLoad(PyObjectSerializeContext context) { } + protected virtual void OnSave(InterDomainContext context) { } + protected virtual void OnLoad(InterDomainContext context) { } protected static void ClearObjectDict(IntPtr ob) { diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 1a87c4d31..d10a5840b 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -255,7 +255,7 @@ public static int tp_clear(IntPtr ob) return 0; } - protected override void OnSave(PyObjectSerializeContext context) + protected override void OnSave(InterDomainContext context) { base.OnSave(context); Runtime.XIncref(target); diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 1e44f3270..9624c6d6e 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -222,7 +222,7 @@ public static int tp_clear(IntPtr ob) return 0; } - protected override void OnSave(PyObjectSerializeContext context) + protected override void OnSave(InterDomainContext context) { base.OnSave(context); if (unbound != null) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 1001cde13..af4578e62 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -349,7 +349,7 @@ public static int tp_clear(IntPtr ob) return 0; } - protected override void OnSave(PyObjectSerializeContext context) + protected override void OnSave(InterDomainContext context) { base.OnSave(context); System.Diagnostics.Debug.Assert(dict == GetObjectDict(pyHandle)); @@ -362,7 +362,7 @@ protected override void OnSave(PyObjectSerializeContext context) Runtime.XIncref(dict); } - protected override void OnLoad(PyObjectSerializeContext context) + protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); SetObjectDict(pyHandle, dict); diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 19eb80a18..3c1229618 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -143,7 +143,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) var extensionObjs = new List(); var wrappers = new Dictionary>(); var serializeObjs = new CLRWrapperCollection(); - var contexts = new Dictionary(); + var contexts = new Dictionary(); foreach (var entry in objs) { var obj = entry.Key; @@ -152,7 +152,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) { case ManagedType.TrackTypes.Extension: Debug.Assert(obj.GetType().IsSerializable); - var context = new PyObjectSerializeContext(); + var context = new InterDomainContext(); contexts[obj.pyHandle] = context; obj.Save(context); extensionObjs.Add(obj); @@ -204,7 +204,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) foreach (var clrObj in wrappers[item.Instance]) { XIncref(clrObj.pyHandle); - var context = new PyObjectSerializeContext(); + var context = new InterDomainContext(); contexts[clrObj.pyHandle] = context; clrObj.Save(context); } @@ -215,12 +215,12 @@ private static void StashPushObjects(RuntimeDataStorage storage) storage.AddValue("contexts", contexts); } - private static Dictionary StashPopObjects(RuntimeDataStorage storage) + private static Dictionary StashPopObjects(RuntimeDataStorage storage) { var extensions = storage.GetValue>("extensions"); var internalStores = storage.GetValue>("internalStores"); - var contexts = storage.GetValue >("contexts"); - var storedObjs = new Dictionary(); + var contexts = storage.GetValue >("contexts"); + var storedObjs = new Dictionary(); foreach (var obj in Enumerable.Union(extensions, internalStores)) { var context = contexts[obj.pyHandle]; @@ -356,7 +356,7 @@ public T PopValue(out T value) [Serializable] - class PyObjectSerializeContext + class InterDomainContext { private RuntimeDataStorage _storage; public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); From b52bc016be767bba4453147032d828efd3da7af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 15:04:17 -0400 Subject: [PATCH 086/134] Rename StashPush/Pop methods Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390669351 --- src/runtime/classmanager.cs | 6 +++--- src/runtime/importhook.cs | 4 ++-- src/runtime/metatype.cs | 4 ++-- src/runtime/runtime.cs | 2 +- src/runtime/runtime_data.cs | 38 ++++++++++++++++++------------------- src/runtime/typemanager.cs | 4 ++-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0adc61589..b63beb620 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -77,7 +77,7 @@ private static int OnVisit(IntPtr ob, IntPtr arg) return 0; } - internal static void StashPush(RuntimeDataStorage storage) + internal static void SaveRuntimeData(RuntimeDataStorage storage) { var contexts = storage.AddValue("contexts", new Dictionary()); @@ -85,14 +85,14 @@ internal static void StashPush(RuntimeDataStorage storage) foreach (var cls in cache.Values) { // This incref is for cache to hold the cls, - // thus no need for decreasing it at StashPop. + // thus no need for decreasing it at RestoreRuntimeData. Runtime.XIncref(cls.pyHandle); var context = contexts[cls.pyHandle] = new InterDomainContext(); cls.Save(context); } } - internal static Dictionary StashPop(RuntimeDataStorage storage) + internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) { cache = storage.GetValue>("cache"); var contexts = storage.GetValue >("contexts"); diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 1af3867f2..e99e7e9bb 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -121,7 +121,7 @@ internal static void Shutdown() CLRModule.Reset(); } - internal static void StashPush(RuntimeDataStorage storage) + internal static void SaveRuntimeData(RuntimeDataStorage storage) { Runtime.XIncref(py_clr_module); Runtime.XIncref(root.pyHandle); @@ -129,7 +129,7 @@ internal static void StashPush(RuntimeDataStorage storage) storage.AddValue("root", root.pyHandle); } - internal static void StashPop(RuntimeDataStorage storage) + internal static void RestoreRuntimeData(RuntimeDataStorage storage) { InitImport(); storage.GetValue("py_clr_module", out py_clr_module); diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 95dd8b9b4..f7afd5d6d 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -40,13 +40,13 @@ public static void Release() _metaSlotsHodler = null; } - internal static void StashPush(RuntimeDataStorage storage) + internal static void SaveRuntimeData(RuntimeDataStorage storage) { Runtime.XIncref(PyCLRMetaType); storage.PushValue(PyCLRMetaType); } - internal static IntPtr StashPop(RuntimeDataStorage storage) + internal static IntPtr RestoreRuntimeData(RuntimeDataStorage storage) { PyCLRMetaType = storage.PopValue(); _metaSlotsHodler = new SlotsHolder(PyCLRMetaType); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b95d4af1e..ffca3d291 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -172,7 +172,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd #if !NETSTANDARD if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) { - RuntimeData.StashPop(); + RuntimeData.RestoreRuntimeData(); } else #endif diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 3c1229618..ae7ed1890 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -34,22 +34,22 @@ public static Type FormatterType internal static void Stash() { var metaStorage = new RuntimeDataStorage(); - MetaType.StashPush(metaStorage); + MetaType.SaveRuntimeData(metaStorage); var importStorage = new RuntimeDataStorage(); - ImportHook.StashPush(importStorage); + ImportHook.SaveRuntimeData(importStorage); var typeStorage = new RuntimeDataStorage(); - TypeManager.StashPush(typeStorage); + TypeManager.SaveRuntimeData(typeStorage); var clsStorage = new RuntimeDataStorage(); - ClassManager.StashPush(clsStorage); + ClassManager.SaveRuntimeData(clsStorage); var moduleStorage = new RuntimeDataStorage(); - StashPushModules(moduleStorage); + SaveRuntimeDataModules(moduleStorage); var objStorage = new RuntimeDataStorage(); - StashPushObjects(objStorage); + SaveRuntimeDataObjects(objStorage); var runtimeStorage = new RuntimeDataStorage(); runtimeStorage.AddValue("meta", metaStorage); @@ -82,11 +82,11 @@ internal static void Stash() XDecref(capsule); } - internal static void StashPop() + internal static void RestoreRuntimeData() { try { - StashPopImpl(); + RestoreRuntimeDataImpl(); } finally { @@ -94,7 +94,7 @@ internal static void StashPop() } } - private static void StashPopImpl() + private static void RestoreRuntimeDataImpl() { IntPtr capsule = PySys_GetObject("clr_data"); if (capsule == IntPtr.Zero) @@ -109,12 +109,12 @@ private static void StashPopImpl() var formatter = CreateFormatter(); var storage = (RuntimeDataStorage)formatter.Deserialize(ms); - var objs = StashPopObjects(storage.GetStorage("objs")); - StashPopModules(storage.GetStorage("modules")); - var clsObjs = ClassManager.StashPop(storage.GetStorage("classes")); - TypeManager.StashPop(storage.GetStorage("types")); - ImportHook.StashPop(storage.GetStorage("import")); - PyCLRMetaType = MetaType.StashPop(storage.GetStorage("meta")); + var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); + RestoreRuntimeDataModules(storage.GetStorage("modules")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); + TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + ImportHook.RestoreRuntimeData(storage.GetStorage("import")); + PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); foreach (var item in objs) { @@ -137,7 +137,7 @@ public static void ClearStash() PySys_SetObject("clr_data", IntPtr.Zero); } - private static void StashPushObjects(RuntimeDataStorage storage) + private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) { var objs = ManagedType.GetManagedObjects(); var extensionObjs = new List(); @@ -215,7 +215,7 @@ private static void StashPushObjects(RuntimeDataStorage storage) storage.AddValue("contexts", contexts); } - private static Dictionary StashPopObjects(RuntimeDataStorage storage) + private static Dictionary RestoreRuntimeDataObjects(RuntimeDataStorage storage) { var extensions = storage.GetValue>("extensions"); var internalStores = storage.GetValue>("internalStores"); @@ -245,7 +245,7 @@ private static Dictionary StashPopObjects(Runti return storedObjs; } - private static void StashPushModules(RuntimeDataStorage storage) + private static void SaveRuntimeDataModules(RuntimeDataStorage storage) { var pyModules = PyImport_GetModuleDict(); var itemsRef = PyDict_Items(pyModules); @@ -268,7 +268,7 @@ private static void StashPushModules(RuntimeDataStorage storage) storage.AddValue("modules", modules); } - private static void StashPopModules(RuntimeDataStorage storage) + private static void RestoreRuntimeDataModules(RuntimeDataStorage storage) { var modules = storage.GetValue>("modules"); var pyMoudles = PyImport_GetModuleDict(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 6e4c87b92..3a7a366c5 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -62,7 +62,7 @@ internal static void RemoveTypes() _slotsHolders.Clear(); } - internal static void StashPush(RuntimeDataStorage storage) + internal static void SaveRuntimeData(RuntimeDataStorage storage) { foreach (var tpHandle in cache.Values) { @@ -72,7 +72,7 @@ internal static void StashPush(RuntimeDataStorage storage) storage.AddValue("slots", _slotsImpls); } - internal static void StashPop(RuntimeDataStorage storage) + internal static void RestoreRuntimeData(RuntimeDataStorage storage) { Debug.Assert(cache == null || cache.Count == 0); storage.GetValue("slots", out _slotsImpls); From bfbf2c3e6d356ca0400e51169f2559c268fa6957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 15:06:52 -0400 Subject: [PATCH 087/134] Rename ClassManager.RemoveClasses To `DisposePythonWrappersForClrTypes` Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390670768 --- src/runtime/classmanager.cs | 2 +- src/runtime/runtime.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index b63beb620..8311555f8 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -39,7 +39,7 @@ public static void Reset() cache = new Dictionary(128); } - internal static void RemoveClasses() + internal static void DisposePythonWrappersForClrTypes() { var visited = new HashSet(); var visitedHandle = GCHandle.Alloc(visited); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ffca3d291..e25400d9e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -334,7 +334,7 @@ internal static void Shutdown() RemoveClrRootModule(); MoveClrInstancesOnwershipToPython(); - ClassManager.RemoveClasses(); + ClassManager.DisposePythonWrappersForClrTypes(); TypeManager.RemoveTypes(); MetaType.Release(); From 883c4cebc5a8fa7f684290aade4d419aef88de07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 20 Aug 2020 15:21:40 -0400 Subject: [PATCH 088/134] Rename ClassManager.OnVisit To `TraverseTypeClear` to emphasis it's used for type traversal (similar to tp_traverse) Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390671251 --- src/runtime/classmanager.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 8311555f8..8fc9949fb 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -48,7 +48,7 @@ internal static void DisposePythonWrappersForClrTypes() { foreach (var cls in cache.Values) { - cls.CallTypeTraverse(OnVisit, visitedPtr); + cls.CallTypeTraverse(TraverseTypeClear, visitedPtr); // XXX: Force release instance resources but not dealloc itself. cls.CallTypeClear(); cls.DecrRefCount(); @@ -61,7 +61,7 @@ internal static void DisposePythonWrappersForClrTypes() cache.Clear(); } - private static int OnVisit(IntPtr ob, IntPtr arg) + private static int TraverseTypeClear(IntPtr ob, IntPtr arg) { var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; if (!visited.Add(ob)) @@ -71,7 +71,7 @@ private static int OnVisit(IntPtr ob, IntPtr arg) var clrObj = ManagedType.GetManagedObject(ob); if (clrObj != null) { - clrObj.CallTypeTraverse(OnVisit, arg); + clrObj.CallTypeTraverse(TraverseTypeClear, arg); clrObj.CallTypeClear(); } return 0; From 7e0d56d33162f7fa1681351aee41c449c2ff139b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 21 Aug 2020 11:20:01 -0400 Subject: [PATCH 089/134] fixup! Inline called-once methods in TestDomainReload --- src/embed_tests/TestDomainReload.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 87b842358..e70c71236 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -282,9 +282,9 @@ void ExecTest() { PythonEngine.Initialize(); var numRef = CreateNumReference(); + Assert.True(numRef.IsAlive); PythonEngine.Shutdown(); // <- "run" 1 ends PythonEngine.Initialize(); // <- "run" 2 starts - Assert.True(numRef.IsAlive); GC.Collect(); GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue @@ -328,9 +328,9 @@ void ExecTest() { PythonEngine.Initialize(); var objRef = CreateConcreateObject(); + Assert.True(objRef.IsAlive); PythonEngine.Shutdown(); // <- "run" 1 ends PythonEngine.Initialize(); // <- "run" 2 starts - Assert.True(objRef.IsAlive); GC.Collect(); GC.WaitForPendingFinalizers(); Finalizer.Instance.Collect(forceDispose: true); From 82034dcb503bae528eb1c112017015f9d0410532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 21 Aug 2020 11:28:15 -0400 Subject: [PATCH 090/134] Change refcount logic in ImportHook.Shutdown If the shutdown mode is Reload, don't decref. Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390685930 --- src/runtime/importhook.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index e99e7e9bb..535cd5bbc 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -107,24 +107,25 @@ internal static void Shutdown() } RestoreImport(); - bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; - Runtime.XDecref(py_clr_module); - py_clr_module = IntPtr.Zero; - if (shouldFreeDef) +#if !NETSTANDARD + if(Runtime.ShutdownMode != ShutdownMode.Reload) +#endif { - ReleaseModuleDef(); + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; + if (shouldFreeDef) + { + ReleaseModuleDef(); + } + Runtime.XDecref(root.pyHandle); } - - Runtime.XDecref(root.pyHandle); root = null; CLRModule.Reset(); } internal static void SaveRuntimeData(RuntimeDataStorage storage) { - Runtime.XIncref(py_clr_module); - Runtime.XIncref(root.pyHandle); storage.AddValue("py_clr_module", py_clr_module); storage.AddValue("root", root.pyHandle); } From 4ba50a77d8415e8a14876fc9638192d78223a1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 21 Aug 2020 11:40:49 -0400 Subject: [PATCH 091/134] Remove TODOS tp_clear and tp_traverse always return 0. No error check to do Addresses comments: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390688176 https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390688190 --- src/runtime/managedtype.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 1190fb871..171d2f2cd 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -176,7 +176,6 @@ internal void CallTypeClear() return; } var clearFunc = (Interop.InquiryFunc)Marshal.GetDelegateForFunctionPointer(clearPtr, typeof(Interop.InquiryFunc)); - // TODO: Handle errors base on return value clearFunc(pyHandle); } @@ -196,7 +195,6 @@ internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) } var traverseFunc = (Interop.ObjObjArgFunc)Marshal.GetDelegateForFunctionPointer(traversePtr, typeof(Interop.ObjObjArgFunc)); var visiPtr = Marshal.GetFunctionPointerForDelegate(visitproc); - // TODO: Handle errors base on return value traverseFunc(pyHandle, visiPtr, arg); } From 1ecdce889f6c59176bff06214993db4e161a395c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 21 Aug 2020 12:52:54 -0400 Subject: [PATCH 092/134] Incref a valid pointer Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390689745 --- src/runtime/methodbinding.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index d10a5840b..3b287a35b 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -22,11 +22,11 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) Runtime.XIncref(target); this.target = target; - Runtime.XIncref(targetType); if (targetType == IntPtr.Zero) { targetType = Runtime.PyObject_Type(target); } + Runtime.XIncref(targetType); this.targetType = targetType; this.info = null; From b35f4411f9faa53ee7858fca0073e556d2a274cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 21 Aug 2020 13:37:22 -0400 Subject: [PATCH 093/134] Use Py_CLEAR Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390690092 --- src/runtime/methodobject.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 9624c6d6e..eb3ce8a18 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -113,8 +113,7 @@ internal bool IsStatic() private void ClearMembers() { - Runtime.XDecref(doc); - doc = IntPtr.Zero; + Runtime.Py_CLEAR(ref doc); if (unbound != null) { Runtime.XDecref(unbound.pyHandle); From ce8ee9029fd95cdd230c2552fdf10f1840c09a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 21 Aug 2020 15:05:24 -0400 Subject: [PATCH 094/134] Fixes GIL grabbing during init and shutdown Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390694298 --- src/runtime/runtime.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e25400d9e..5cc3bb77e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -125,6 +125,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } ShutdownMode = mode; + IntPtr state = IntPtr.Zero; if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); @@ -148,7 +149,8 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } else { - PyGILState_Ensure(); + PyEval_InitThreads(); + state = PyGILState_Ensure(); MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } @@ -193,6 +195,10 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } XDecref(item); AssemblyManager.UpdatePath(); + if (state != IntPtr.Zero) + { + PyGILState_Release(state); + } } private static void InitPyMembers() @@ -314,7 +320,7 @@ internal static void Shutdown() } _isInitialized = false; - PyGILState_Ensure(); + var state = PyGILState_Ensure(); var mode = ShutdownMode; if (mode == ShutdownMode.Soft) @@ -360,7 +366,7 @@ internal static void Shutdown() { // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. } - PyEval_SaveThread(); + PyGILState_Release(state); } else { From 4c4bcb083ba41737dcf0819230298a12c98ea954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 26 Aug 2020 13:05:58 -0400 Subject: [PATCH 095/134] Add a parameter to specify the shutdown mode. Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390659648 --- src/runtime/pythonengine.cs | 21 +++++++++++++++++---- src/runtime/runtime.cs | 9 +++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 119befd48..c37edc1ea 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -189,9 +189,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // Make sure we clean up properly on app domain unload. AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - // The global scope gets used implicitly quite early on, remember // to clear it out when we shut down. AddShutdownHandler(PyScopeManager.Global.Clear); @@ -318,7 +315,8 @@ public static IntPtr InitExt() /// Python runtime can no longer be used in the current process /// after calling the Shutdown method. /// - public static void Shutdown() + /// The ShutdownMode to use when shutting down the Runtime + public static void Shutdown(ShutdownMode mode) { if (!initialized) { @@ -330,11 +328,26 @@ public static void Shutdown() PyScopeManager.Global.Clear(); ExecuteShutdownHandlers(); + // Remember to shut down the runtime. + Runtime.Shutdown(mode); PyObjectConversions.Reset(); initialized = false; } + /// + /// Shutdown Method + /// + /// + /// Shutdown and release resources held by the Python runtime. The + /// Python runtime can no longer be used in the current process + /// after calling the Shutdown method. + /// + public static void Shutdown() + { + Shutdown(Runtime.ShutdownMode); + } + /// /// Called when the engine is shut down. /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5cc3bb77e..10854151e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -312,7 +312,7 @@ private static IntPtr Get_PyObject_NextNotImplemented() return iternext; } - internal static void Shutdown() + internal static void Shutdown(ShutdownMode mode) { if (Py_IsInitialized() == 0 || !_isInitialized) { @@ -322,7 +322,6 @@ internal static void Shutdown() var state = PyGILState_Ensure(); - var mode = ShutdownMode; if (mode == ShutdownMode.Soft) { RunExitFuncs(); @@ -375,6 +374,12 @@ internal static void Shutdown() } } + internal static void Shutdown() + { + var mode = ShutdownMode; + Shutdown(mode); + } + internal static ShutdownMode GetDefaultShutdownMode() { string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); From f575bd329913c0791aa52b906ef5b6fc0865251e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 26 Aug 2020 14:00:46 -0400 Subject: [PATCH 096/134] Add typeoffset.cs Move the partial class definition of TypeOffset from interop.cs to its own file. Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390686331 --- src/runtime/Python.Runtime.csproj | 375 +++++++++++++++--------------- src/runtime/interop.cs | 18 -- src/runtime/typeoffset.cs | 23 ++ 3 files changed, 211 insertions(+), 205 deletions(-) create mode 100644 src/runtime/typeoffset.cs diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 410e3d9b3..ec97cc299 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,188 +1,189 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index fa3d81373..fecebaca6 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -69,24 +69,6 @@ public ModulePropertyAttribute() } } - internal static partial class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fields = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fields.Length; i++) - { - int offset = i * size; - FieldInfo fi = fields[i]; - fi.SetValue(null, offset); - } - } - - public static int magic() => ManagedDataOffsets.Magic; - } - internal static class ManagedDataOffsets { public static int Magic { get; private set; } diff --git a/src/runtime/typeoffset.cs b/src/runtime/typeoffset.cs new file mode 100644 index 000000000..2f2cafc3b --- /dev/null +++ b/src/runtime/typeoffset.cs @@ -0,0 +1,23 @@ +using System; +using System.Reflection; + +namespace Python.Runtime +{ + internal static partial class TypeOffset + { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fields = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fields.Length; i++) + { + int offset = i * size; + FieldInfo fi = fields[i]; + fi.SetValue(null, offset); + } + } + + public static int magic() => ManagedDataOffsets.Magic; + } +} From d1799aad7aa116013aab666715ab9ad4347882af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 26 Aug 2020 14:21:03 -0400 Subject: [PATCH 097/134] Extract utility method in TestDomainReload Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390645914 --- src/embed_tests/TestDomainReload.cs | 30 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index e70c71236..3556df0f6 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -400,6 +400,15 @@ public object Call(string methodName, params object[] args) return method.Invoke(null, args); } } + + static T CreateInstanceInstanceAndUnwrap(AppDomain domain) + { + Type type = typeof(T); + var theProxy = (T)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + return theProxy; + } /// /// Create a domain, run the assembly in it (the RunPython function), @@ -412,10 +421,7 @@ static void RunAssemblyAndUnload(string domainName) AppDomain domain = CreateDomain(domainName); // Create a Proxy object in the new domain, where we want the // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); + var theProxy = CreateInstanceInstanceAndUnwrap(domain); theProxy.Call("InitPython", ShutdownMode.Soft); // From now on use the Proxy to call into the new assembly @@ -484,14 +490,10 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro AppDomain domain = CreateDomain("test_domain_reload_1"); try { - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); + var theProxy = CreateInstanceInstanceAndUnwrap(domain); theProxy.Call("InitPython", ShutdownMode.Reload); - var caller = (T1)domain.CreateInstanceAndUnwrap( - typeof(T1).Assembly.FullName, - typeof(T1).FullName); + var caller = CreateInstanceInstanceAndUnwrap(domain); arg = caller.Execute(arg); theProxy.Call("ShutdownPython"); @@ -506,14 +508,10 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro AppDomain domain = CreateDomain("test_domain_reload_2"); try { - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); + var theProxy = CreateInstanceInstanceAndUnwrap(domain); theProxy.Call("InitPython", ShutdownMode.Reload); - var caller = (T2)domain.CreateInstanceAndUnwrap( - typeof(T2).Assembly.FullName, - typeof(T2).FullName); + var caller = CreateInstanceInstanceAndUnwrap(domain); caller.Execute(arg); theProxy.Call("ShutdownPythonCompletely"); } From d2408b9f75484ebba21e40acca0c4da909697bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 26 Aug 2020 15:50:31 -0400 Subject: [PATCH 098/134] Add cleanup to ClassInfo Addresses comment: https://github.com/pythonnet/pythonnet/pull/958/#discussion_r390671982 --- src/runtime/classmanager.cs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 8fc9949fb..40cccbf79 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -212,8 +212,6 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; Runtime.PyDict_SetItemString(dict, name, item.pyHandle); - // info.members are already useless - item.DecrRefCount(); } // If class has constructors, generate an __doc__ attribute. @@ -463,7 +461,7 @@ private static ClassInfo GetClassInfo(Type type) } - internal class ClassInfo + internal class ClassInfo : IDisposable { public Indexer indexer; public Hashtable members; @@ -473,5 +471,28 @@ internal ClassInfo() members = new Hashtable(); indexer = null; } + + ~ClassInfo() + { + Dispose(); + } + + private bool disposed = false; + + public void Dispose() + { + if (!disposed) + { + disposed = true; + foreach(var member in members) + { + var item = (ManagedType)member; + if (item != null) + { + item.DecrRefCount(); + } + } + } + } } } From fd2b66228dd13993f807d787c67be63273c9eca1 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 27 Aug 2020 10:27:03 +0800 Subject: [PATCH 099/134] Update comment of `ClassManager.RemoveClasses` --- src/runtime/classmanager.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index a7d2d74e2..62fbb7570 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -53,8 +53,11 @@ internal static void RemoveClasses() { foreach (var cls in cache.Values) { + // XXX: Force to release instance's managed resources + // but not dealloc itself immediately. + // These managed resources should preserve vacant shells + // since others may still referencing it. cls.CallTypeTraverse(OnVisit, visitedPtr); - // XXX: Force release instance resources but not dealloc itself. cls.CallTypeClear(); cls.DecrRefCount(); } From 2b7bcacb63b25fb760e38bc558e40cedd3351933 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 27 Aug 2020 10:28:46 +0800 Subject: [PATCH 100/134] Fix typo errors --- src/embed_tests/TestDomainReload.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 05b449237..9815ff725 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -21,7 +21,7 @@ class TestDomainReload { abstract class CrossCaller : MarshalByRefObject { - public abstract ValueType Execte(ValueType arg); + public abstract ValueType Execute(ValueType arg); } @@ -81,7 +81,7 @@ public static void DomainReloadAndGC() class CrossDomianObjectStep1 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { try { @@ -120,9 +120,9 @@ from Python.EmbeddingTest.Domain import MyClass } - class CrossDomianObjectStep2 : CrossCaller + class CrossDomainObjectStep2 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { // handle refering a clr object created in previous domain, // it should had been deserialized and became callable agian. @@ -167,7 +167,7 @@ public override ValueType Execte(ValueType arg) [Test] public static void CrossDomainObject() { - RunDomainReloadSteps(); + RunDomainReloadSteps(); } #endregion @@ -176,7 +176,7 @@ public static void CrossDomainObject() class ReloadClassRefStep1 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { const string code = @" from Python.EmbeddingTest.Domain import MyClass @@ -224,7 +224,7 @@ def test_obj_call(): class ReloadClassRefStep2 : CrossCaller { - public override ValueType Execte(ValueType arg) + public override ValueType Execute(ValueType arg) { var module = (IntPtr)arg; using (Py.GIL()) @@ -470,7 +470,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro var caller = (T1)domain.CreateInstanceAndUnwrap( typeof(T1).Assembly.FullName, typeof(T1).FullName); - arg = caller.Execte(arg); + arg = caller.Execute(arg); theProxy.Call("ShutdownPython"); } @@ -492,7 +492,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro var caller = (T2)domain.CreateInstanceAndUnwrap( typeof(T2).Assembly.FullName, typeof(T2).FullName); - caller.Execte(arg); + caller.Execute(arg); theProxy.Call("ShutdownPythonCompletely"); } finally From ba1df6e214a1662c720c3261a9c5b7d0eb103fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 27 Aug 2020 10:53:13 -0400 Subject: [PATCH 101/134] fixup! Add cleanup to ClassInfo --- src/runtime/classmanager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 56d1c4ab5..72eafd747 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -487,9 +487,9 @@ public void Dispose() if (!disposed) { disposed = true; - foreach(var member in members) + foreach(DictionaryEntry member in members) { - var item = (ManagedType)member; + var item = (ManagedType)member.Value; if (item != null) { item.DecrRefCount(); From 639ba1fee517b19b6e1658b864d128f424f63d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 27 Aug 2020 14:49:52 -0400 Subject: [PATCH 102/134] Add a new PyList constructor To build from a BorrowedReference. Assumes Ownership. --- src/embed_tests/pyinitialize.cs | 6 ++---- src/runtime/pylist.cs | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 6e539609b..9b036c472 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -25,8 +25,7 @@ public static void LoadDefaultArgs() { using (new PythonEngine()) { - var argvref = new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")); - using(var argv = new PyList(argvref.DangerousGetAddress())) + using(var argv = new PyList(new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")))) { Assert.AreNotEqual(0, argv.Length()); } @@ -39,8 +38,7 @@ public static void LoadSpecificArgs() var args = new[] { "test1", "test2" }; using (new PythonEngine(args)) { - var argvref = new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")); - using(var argv = new PyList(argvref.DangerousGetAddress())) + using(var argv = new PyList(new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")))) { Assert.AreEqual(args[0], argv[0].ToString()); Assert.AreEqual(args[1], argv[1].ToString()); diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 347cc3000..3ccee9b9b 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -80,6 +80,25 @@ public PyList(PyObject[] items) } } + /// + /// Constructor to make a PyList from a BorrowedReference. + /// The list assumes ownership of the reference. + /// + /// The borrowed reference + internal PyList(BorrowedReference r) + { + IntPtr addr = r.DangerousGetAddress(); + if(!Runtime.PyList_Check(addr)) + { + throw new ArgumentException("object is not a list"); + } + + obj = addr; + // Take ownership. + Runtime.XIncref(addr); + + } + /// /// IsListType Method From 7e5ab52167541b99d0874474f305d33441953485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 27 Aug 2020 14:51:38 -0400 Subject: [PATCH 103/134] Revert previous commit "Add cleaunp to class info" And change the scope of the ClassInfo class and add some documentation. --- src/runtime/classmanager.cs | 45 ++++++++++++------------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 72eafd747..1d569d2c8 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -215,6 +215,8 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + // Decref the item now that it's been used. + item.DecrRefCount(); } // If class has constructors, generate an __doc__ attribute. @@ -461,41 +463,22 @@ private static ClassInfo GetClassInfo(Type type) return ci; } - } - - - internal class ClassInfo : IDisposable - { - public Indexer indexer; - public Hashtable members; - - internal ClassInfo() - { - members = new Hashtable(); - indexer = null; - } - - ~ClassInfo() + + /// + /// This class owns references to PyObjects in the `members` member. + /// The caller has responsibility to DECREF them. + /// + private class ClassInfo { - Dispose(); - } - - private bool disposed = false; + public Indexer indexer; + public Hashtable members; - public void Dispose() - { - if (!disposed) + internal ClassInfo() { - disposed = true; - foreach(DictionaryEntry member in members) - { - var item = (ManagedType)member.Value; - if (item != null) - { - item.DecrRefCount(); - } - } + members = new Hashtable(); + indexer = null; } } } + } From 8075f4827e6cb53124a17cebb49f168ffb254fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 27 Aug 2020 14:52:29 -0400 Subject: [PATCH 104/134] Add corrections to the GIL acquiring/releasing In Runtime's Initialize and Shutdown. --- src/runtime/runtime.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 10854151e..417f757c3 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -111,6 +111,8 @@ public class Runtime /// /// Initialize the runtime... /// + /// When calling this method after a soft shutdown or a domain reload, + /// this method acquires and releases the GIL. internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { if (_isInitialized) @@ -149,7 +151,8 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } else { - PyEval_InitThreads(); + // When initializing more than once (like on soft shutdown and domain + // reload), the GIL might not be acquired by the current thread. state = PyGILState_Ensure(); MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } @@ -366,6 +369,8 @@ internal static void Shutdown(ShutdownMode mode) // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. } PyGILState_Release(state); + // Then release the GIL for good. + PyEval_SaveThread(); } else { From eb8cb8ad735fc8d3461014bc0ed80ed9f377576e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 27 Aug 2020 16:01:31 -0400 Subject: [PATCH 105/134] Factor out ExtensionType's Gc setup mechanic To avoid code duplication. --- src/runtime/extensiontype.cs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 003e27a56..a5f0f1219 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -29,19 +29,24 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); + // Steals a ref to tpHandle. + tpHandle = tp; + pyHandle = py; + + SetupGc(); + } + + void SetupGc () + { GCHandle gc = AllocGCHandle(TrackTypes.Extension); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most // concrete extension types, so untrack the object to save calls // from Python into the managed runtime that are pure overhead. - Runtime.PyObject_GC_UnTrack(py); - - // Steals a ref to tpHandle. - tpHandle = tp; - pyHandle = py; + Runtime.PyObject_GC_UnTrack(pyHandle); } @@ -101,9 +106,7 @@ public static void tp_dealloc(IntPtr ob) protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); - GCHandle gc = AllocGCHandle(TrackTypes.Extension); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); - Runtime.PyObject_GC_UnTrack(pyHandle); + SetupGc(); } } } From 308f0f2195e541e156ff55813b656ff07a320c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 28 Aug 2020 09:26:34 -0400 Subject: [PATCH 106/134] Revert "Release the GIL" This reverts commit 7ec9a6c677cd346afd3b75fe8022f51e4e076809. --- src/embed_tests/TestRuntime.cs | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 707d2a467..38878205c 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -40,27 +40,16 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { - IntPtr state = IntPtr.Zero; - try + if (Runtime.Runtime.Py_IsInitialized() == 1) { - if (Runtime.Runtime.Py_IsInitialized() == 1) - { - state = Runtime.Runtime.PyGILState_Ensure(); - } - Runtime.Runtime.Py_Finalize(); - Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); - Runtime.Runtime.Py_Initialize(); - Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); - Runtime.Runtime.Py_Finalize(); - Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); - } - finally - { - if(state != IntPtr.Zero) - { - Runtime.Runtime.PyGILState_Release(state); - } + Runtime.Runtime.PyGILState_Ensure(); } + Runtime.Runtime.Py_Finalize(); + Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); + Runtime.Runtime.Py_Initialize(); + Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); + Runtime.Runtime.Py_Finalize(); + Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); } [Test] From 1ae0bfe2c349f3321c0a10fdcf9f987a8035f76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 28 Aug 2020 09:29:22 -0400 Subject: [PATCH 107/134] fixup! Incref a valid pointer --- src/runtime/methodbinding.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 3b287a35b..011d8217d 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -26,7 +26,11 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) { targetType = Runtime.PyObject_Type(target); } - Runtime.XIncref(targetType); + else + { + Runtime.XIncref(targetType); + } + this.targetType = targetType; this.info = null; From 07aefe65750cd791b333411a5904f32e30ff6a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 28 Aug 2020 09:40:41 -0400 Subject: [PATCH 108/134] Revert "Add typeoffset.cs" This reverts commit f575bd329913c0791aa52b906ef5b6fc0865251e. --- src/runtime/Python.Runtime.csproj | 375 +++++++++++++++--------------- src/runtime/interop.cs | 18 ++ src/runtime/typeoffset.cs | 23 -- 3 files changed, 205 insertions(+), 211 deletions(-) delete mode 100644 src/runtime/typeoffset.cs diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index ec97cc299..410e3d9b3 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,189 +1,188 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index fecebaca6..fa3d81373 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -69,6 +69,24 @@ public ModulePropertyAttribute() } } + internal static partial class TypeOffset + { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fields = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fields.Length; i++) + { + int offset = i * size; + FieldInfo fi = fields[i]; + fi.SetValue(null, offset); + } + } + + public static int magic() => ManagedDataOffsets.Magic; + } + internal static class ManagedDataOffsets { public static int Magic { get; private set; } diff --git a/src/runtime/typeoffset.cs b/src/runtime/typeoffset.cs deleted file mode 100644 index 2f2cafc3b..000000000 --- a/src/runtime/typeoffset.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Reflection; - -namespace Python.Runtime -{ - internal static partial class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fields = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fields.Length; i++) - { - int offset = i * size; - FieldInfo fi = fields[i]; - fi.SetValue(null, offset); - } - } - - public static int magic() => ManagedDataOffsets.Magic; - } -} From b20367453f41771affe044ab7aac62f0cc9ae28e Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 31 Aug 2020 10:21:58 +0800 Subject: [PATCH 109/134] Revert local project setting --- src/runtime/Python.Runtime.15.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 4a0c95edd..d753a5dff 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -90,7 +90,7 @@ $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - TRACE;DEBUG;XPLAT;;PYTHON3;PYTHON38;UCS2;FINALIZER_CHECK;WINDOWS + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG From 19d137951721221904183088f847a3c3874f0f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Sep 2020 13:34:21 -0400 Subject: [PATCH 110/134] A better fix for releasing a re-acquiring the GIL On shutdown and startup. The main python thread always havea state. After releasing the state on shutdown, restore the state of the main thread (thus also locking the GIL). --- src/runtime/runtime.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 417f757c3..ac13524b6 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -151,9 +151,10 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } else { - // When initializing more than once (like on soft shutdown and domain - // reload), the GIL might not be acquired by the current thread. - state = PyGILState_Ensure(); + // If we're coming back from a domain reload or a soft shutdown, + // we have previously released the thread state. Restore the main + // thread state here. + PyEval_RestoreThread(PyGILState_GetThisThreadState()); MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } @@ -198,10 +199,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } XDecref(item); AssemblyManager.UpdatePath(); - if (state != IntPtr.Zero) - { - PyGILState_Release(state); - } } private static void InitPyMembers() From 25a4064752a2708322dc9867caa7531b0a93ef60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Sep 2020 13:35:50 -0400 Subject: [PATCH 111/134] Add validation for downgrading the shutdown mode --- src/runtime/runtime.cs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ac13524b6..e6325cac0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -312,6 +312,34 @@ private static IntPtr Get_PyObject_NextNotImplemented() return iternext; } + /// + /// Tries to downgrade the shutdown mode, if possible. + /// The only possibles downgrades are: + /// Soft -> Normal + /// Reload -> Soft + /// Reload -> Normal + /// + /// The desired shutdown mode + /// The `mode` parameter if the downgrade is supported, the ShutdownMode + /// set at initialization otherwise. + static ShutdownMode TryDowngradeShutdown(ShutdownMode mode) + { + if ( + mode == Runtime.ShutdownMode + || mode == ShutdownMode.Normal +#if !NETSTANDARD + || (mode == ShutdownMode.Soft && Runtime.ShutdownMode == ShutdownMode.Reload) +#endif + ) + { + return mode; + } + else // we can't downgrade + { + return Runtime.ShutdownMode; + } + } + internal static void Shutdown(ShutdownMode mode) { if (Py_IsInitialized() == 0 || !_isInitialized) @@ -320,6 +348,11 @@ internal static void Shutdown(ShutdownMode mode) } _isInitialized = false; + // If the shutdown mode specified is not the the same as the one specified + // during Initialization, we need to validate it; we can only downgrade, + // not upgrade the shutdown mode. + mode = TryDowngradeShutdown(mode); + var state = PyGILState_Ensure(); if (mode == ShutdownMode.Soft) From 8c133e3c19dd4700b6a876ffc4944412cb3a12db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Sep 2020 14:13:42 -0400 Subject: [PATCH 112/134] Fixes for the merge --- src/embed_tests/pyinitialize.cs | 4 ++-- src/runtime/pylist.cs | 20 -------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 9b036c472..c774680dd 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -25,7 +25,7 @@ public static void LoadDefaultArgs() { using (new PythonEngine()) { - using(var argv = new PyList(new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")))) + using(var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { Assert.AreNotEqual(0, argv.Length()); } @@ -38,7 +38,7 @@ public static void LoadSpecificArgs() var args = new[] { "test1", "test2" }; using (new PythonEngine(args)) { - using(var argv = new PyList(new BorrowedReference(Runtime.Runtime.PySys_GetObject("argv")))) + using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { Assert.AreEqual(args[0], argv[0].ToString()); Assert.AreEqual(args[1], argv[1].ToString()); diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index a30d4afb8..df574140b 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -85,26 +85,6 @@ public PyList(PyObject[] items) } } - /// - /// Constructor to make a PyList from a BorrowedReference. - /// The list assumes ownership of the reference. - /// - /// The borrowed reference - internal PyList(BorrowedReference r) - { - IntPtr addr = r.DangerousGetAddress(); - if(!Runtime.PyList_Check(addr)) - { - throw new ArgumentException("object is not a list"); - } - - obj = addr; - // Take ownership. - Runtime.XIncref(addr); - - } - - /// /// IsListType Method /// From d9b21a50bed9a2b6b78a633c68948e8b74827376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Sep 2020 15:30:13 -0400 Subject: [PATCH 113/134] Revert "Change refcount logic in ImportHook.Shutdown" This reverts commit 82034dcb503bae528eb1c112017015f9d0410532. --- src/runtime/importhook.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7b3051c8a..dfa26be0c 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -108,25 +108,24 @@ internal static void Shutdown() } RestoreImport(); + bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; -#if !NETSTANDARD - if(Runtime.ShutdownMode != ShutdownMode.Reload) -#endif + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; + if (shouldFreeDef) { - Runtime.XDecref(py_clr_module); - py_clr_module = IntPtr.Zero; - if (shouldFreeDef) - { - ReleaseModuleDef(); - } - Runtime.XDecref(root.pyHandle); + ReleaseModuleDef(); } + + Runtime.XDecref(root.pyHandle); root = null; CLRModule.Reset(); } internal static void SaveRuntimeData(RuntimeDataStorage storage) { + Runtime.XIncref(py_clr_module); + Runtime.XIncref(root.pyHandle); storage.AddValue("py_clr_module", py_clr_module); storage.AddValue("root", root.pyHandle); } From c8dee53618bc6f5073cdfde3dceb57101e3639e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 2 Sep 2020 15:30:40 -0400 Subject: [PATCH 114/134] Document refcount increase --- src/runtime/importhook.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index dfa26be0c..8cf57c85d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -124,6 +124,8 @@ internal static void Shutdown() internal static void SaveRuntimeData(RuntimeDataStorage storage) { + // Increment the reference counts here so that the objects don't + // get freed in Shutdown. Runtime.XIncref(py_clr_module); Runtime.XIncref(root.pyHandle); storage.AddValue("py_clr_module", py_clr_module); From 598cb771d3455d0b145ab968ff13450e124a469f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 3 Sep 2020 11:47:04 -0400 Subject: [PATCH 115/134] Disambiguate Initialization between initializing from CPython and initializing after a domain reload Add an optional argument to signal to the initialization that it's coming from CPython --- src/runtime/pythonengine.cs | 10 +++++----- src/runtime/runtime.cs | 7 +++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 7de29d080..f305b3795 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -155,9 +155,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode, fromPython: fromPython); } /// @@ -170,7 +170,7 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false, Shu /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) { if (initialized) { @@ -182,7 +182,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs, mode); + Runtime.Initialize(initSigs, mode, fromPython); initialized = true; Exceptions.Clear(); @@ -265,7 +265,7 @@ public static IntPtr InitExt() { try { - Initialize(setSysArgv: false); + Initialize(setSysArgv: false, fromPython: true); // Trickery - when the import hook is installed into an already // running Python, the standard import machinery is still in diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e82a74adb..1d6096c2e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -129,7 +129,7 @@ internal static Version PyVersion /// /// When calling this method after a soft shutdown or a domain reload, /// this method acquires and releases the GIL. - internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) { if (_isInitialized) { @@ -163,16 +163,15 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd RuntimeState.Save(); } #endif - MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } - else + else if (!fromPython) { // If we're coming back from a domain reload or a soft shutdown, // we have previously released the thread state. Restore the main // thread state here. PyEval_RestoreThread(PyGILState_GetThisThreadState()); - MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; IsFinalizing = false; From 6db318127cbf2229ac911f38e7b95d71ccba525d Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 9 Sep 2020 11:12:29 +0800 Subject: [PATCH 116/134] Revert local changes --- src/runtime/typemanager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3a7a366c5..425b91c09 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -709,7 +709,7 @@ private static void InitMethods(IntPtr pytype, Type type) mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); - //m.DecrRefCount(); + m.DecrRefCount(); addedMethods.Add(method_name); } } From e38a363be385df6fa3922b3f64daebedf289b1a4 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 9 Sep 2020 11:18:38 +0800 Subject: [PATCH 117/134] Remove unused code --- src/runtime/interop.cs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index fa3d81373..1caabab17 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -5,7 +5,6 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; -using System.Linq; namespace Python.Runtime { @@ -356,13 +355,6 @@ public static void FreeModuleDef(IntPtr ptr) public static int name = 0; } - static class TypeOffsetHelper - { - public static string GetSlotNameByOffset(int offset) - { - return typeof(TypeOffset).GetFields().First(fi => (int)fi.GetValue(null) == offset).Name; - } - } /// /// TypeFlags(): The actual bit values for the Type Flags stored From b409a892249f555d77d3f834fc0b1a31397beb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 8 Sep 2020 11:57:03 -0400 Subject: [PATCH 118/134] Check serializability of the whole type hierarchy When serializing wrapper objects on domain shutdown Addresses comment: https://github.com/pythonnet/pythonnet/pull/958#discussion_r390738447 --- src/runtime/runtime_data.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 7151321fd..2f4557bfc 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -137,6 +137,19 @@ public static void ClearStash() PySys_SetObject("clr_data", IntPtr.Zero); } + static bool CheckSerializable (object o) + { + Type type = o.GetType(); + do + { + if (!type.IsSerializable) + { + return false; + } + } while ((type = type.BaseType) != null); + return true; + } + private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) { var objs = ManagedType.GetManagedObjects(); @@ -151,7 +164,7 @@ private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) switch (entry.Value) { case ManagedType.TrackTypes.Extension: - Debug.Assert(obj.GetType().IsSerializable); + Debug.Assert(CheckSerializable(obj)); var context = new InterDomainContext(); contexts[obj.pyHandle] = context; obj.Save(context); @@ -195,7 +208,7 @@ private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) { if (!item.Stored) { - if (!item.Instance.GetType().IsSerializable) + if (!CheckSerializable(item.Instance)) { continue; } From f5c24b03f7483dc9621d6e459e8e9bb57766642a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 8 Sep 2020 12:03:31 -0400 Subject: [PATCH 119/134] Release the GIL on shutdown only if it can be released Fixes an issue where there is no current thread state on shutdown. Also fixes a comment in Runtime.Initialize --- src/runtime/runtime.cs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 1d6096c2e..764050dcd 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -127,8 +127,8 @@ internal static Version PyVersion /// /// Initialize the runtime... /// - /// When calling this method after a soft shutdown or a domain reload, - /// this method acquires and releases the GIL. + /// Always call this method from the Main thread. After the + /// first call to this method, the main thread has acquired the GIL. internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) { if (_isInitialized) @@ -414,8 +414,14 @@ internal static void Shutdown(ShutdownMode mode) // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. } PyGILState_Release(state); - // Then release the GIL for good. - PyEval_SaveThread(); + // Then release the GIL for good, if there is somehting to release + // Use the unchecked version as the checked version calls `abort()` + // if the current state is NULL. + if (_PyThreadState_UncheckedGet() != IntPtr.Zero) + { + PyEval_SaveThread(); + } + } else { @@ -846,6 +852,9 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThreadState_Get(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr _PyThreadState_UncheckedGet(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThread_get_key_value(IntPtr key); From 0d6c6456c629df0a7c0296b4b21d2fbe2c9ae51d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 8 Sep 2020 14:15:37 -0400 Subject: [PATCH 120/134] Call `WaitForPendingFinalizers` instead ... of WaitForFullGCComplete on runtime shutdown Addresses comment: https://github.com/pythonnet/pythonnet/pull/958#discussion_r390695411 --- src/runtime/runtime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 764050dcd..559817776 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -407,7 +407,7 @@ internal static void Shutdown(ShutdownMode mode) GC.Collect(); try { - GC.WaitForFullGCComplete(); + GC.WaitForPendingFinalizers(); } catch (NotImplementedException) { From fa89b487dd377bb0ae36a45cd41cb9da7f15bc8d Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 9 Sep 2020 11:12:29 +0800 Subject: [PATCH 121/134] Revert local changes --- src/runtime/typemanager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 3a7a366c5..425b91c09 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -709,7 +709,7 @@ private static void InitMethods(IntPtr pytype, Type type) mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); - //m.DecrRefCount(); + m.DecrRefCount(); addedMethods.Add(method_name); } } From 80a7644deaac4af8d96d3f66cf98e7081b9fae21 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 9 Sep 2020 11:18:38 +0800 Subject: [PATCH 122/134] Remove unused code --- src/runtime/interop.cs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index fa3d81373..1caabab17 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -5,7 +5,6 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; -using System.Linq; namespace Python.Runtime { @@ -356,13 +355,6 @@ public static void FreeModuleDef(IntPtr ptr) public static int name = 0; } - static class TypeOffsetHelper - { - public static string GetSlotNameByOffset(int offset) - { - return typeof(TypeOffset).GetFields().First(fi => (int)fi.GetValue(null) == offset).Name; - } - } /// /// TypeFlags(): The actual bit values for the Type Flags stored From 6a3cfc8c7f6fee2d3bf674d8b7435e91362ddda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 15 Sep 2020 14:31:41 -0400 Subject: [PATCH 123/134] Adds an unchecked version to get a BorrowedReference pointer And other code review changes --- src/runtime/BorrowedReference.cs | 9 ++++++++- src/runtime/runtime_data.cs | 4 ++-- tools/geninterop/geninterop.py | 5 +---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 06084fd16..ec4962db4 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -11,7 +11,14 @@ readonly ref struct BorrowedReference public bool IsNull => this.pointer == IntPtr.Zero; /// Gets a raw pointer to the Python object - public IntPtr DangerousGetAddress() => this.pointer; + public IntPtr DangerousGetAddress() + => this.IsNull ? throw new NullReferenceException() : this.pointer; + + /// + /// Gets a raw pointer to the Python object. Does not throw an exception + /// if the pointer is null + /// + public IntPtr DangerousGetAddressUnchecked() => this.pointer; /// /// Creates new instance of from raw pointer. Unsafe. diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 2f4557bfc..a19ed176c 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -70,7 +70,7 @@ internal static void Stash() Marshal.WriteIntPtr(mem, (IntPtr)ms.Length); Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); - IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddress(); + IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddressUnchecked(); if (capsule != IntPtr.Zero) { IntPtr oldData = PyCapsule_GetPointer(capsule, null); @@ -96,7 +96,7 @@ internal static void RestoreRuntimeData() private static void RestoreRuntimeDataImpl() { - IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddress(); + IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddressUnchecked(); if (capsule == IntPtr.Zero) { return; diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index aacc4af65..da6c3733a 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -21,10 +21,7 @@ import sysconfig import subprocess -if sys.version_info.major > 2: - from io import StringIO -else: - from StringIO import StringIO +from StringIO import StringIO from pycparser import c_ast, c_parser From 98da1fad91cd7fa3a40d3478f88c5e920d312630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 15 Sep 2020 14:56:19 -0400 Subject: [PATCH 124/134] Remove compile-time check on NETSTANDARD Even if netstandard lacks (for now) the necessary APIs for domain (un)loading, it is still possible to (un)load domains via the Mono C runtime. --- src/runtime/runtime.cs | 14 +------------- src/runtime/typemanager.cs | 4 ---- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 559817776..346114762 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -151,18 +151,12 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd { PyEval_InitThreads(); } - if (mode == ShutdownMode.Soft) - { - RuntimeState.Save(); - } -#if !NETSTANDARD // XXX: Reload mode may reduct to Soft mode, // so even on Reload mode it still needs to save the RuntimeState - else if (mode == ShutdownMode.Reload) + if (mode == ShutdownMode.Soft || mode == ShutdownMode.Reload) { RuntimeState.Save(); } -#endif } else if (!fromPython) { @@ -190,13 +184,11 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); -#if !NETSTANDARD if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) { RuntimeData.RestoreRuntimeData(); } else -#endif { PyCLRMetaType = MetaType.Initialize(); // Steal a reference ImportHook.Initialize(); @@ -342,9 +334,7 @@ static ShutdownMode TryDowngradeShutdown(ShutdownMode mode) if ( mode == Runtime.ShutdownMode || mode == ShutdownMode.Normal -#if !NETSTANDARD || (mode == ShutdownMode.Soft && Runtime.ShutdownMode == ShutdownMode.Reload) -#endif ) { return mode; @@ -374,12 +364,10 @@ internal static void Shutdown(ShutdownMode mode) { RunExitFuncs(); } -#if !NETSTANDARD if (mode == ShutdownMode.Reload) { RuntimeData.Stash(); } -#endif AssemblyManager.Shutdown(); ImportHook.Shutdown(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 425b91c09..fcceea9c2 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -459,7 +459,6 @@ internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); -#if !NETSTANDARD // XXX: Hard code with mode check. if (Runtime.ShutdownMode != ShutdownMode.Reload) { @@ -470,7 +469,6 @@ internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) Marshal.WriteIntPtr(t, offset, IntPtr.Zero); }); } -#endif return slotsHolder; } @@ -480,10 +478,8 @@ private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); slotsHolder.KeeapAlive(thunkInfo); -#if !NETSTANDARD // XXX: Hard code with mode check. if (Runtime.ShutdownMode != ShutdownMode.Reload) -#endif { IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => From d7d44e8a87146bf9024250fe5e1f110bc7e3a989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Tue, 15 Sep 2020 15:35:35 -0400 Subject: [PATCH 125/134] fixup! Remove compile-time check on NETSTANDARD --- src/runtime/runtime.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 346114762..bf0065ea1 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2194,9 +2194,7 @@ public enum ShutdownMode Default, Normal, Soft, -#if !NETSTANDARD Reload, -#endif } From 6aa75c5434a2d7dc644a17569911f2de687d7ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 16 Sep 2020 11:19:59 -0400 Subject: [PATCH 126/134] Factor out the clearing of clr_data In Runtime_data.cs --- src/runtime/runtime_data.cs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index a19ed176c..9b2fdbe24 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -31,6 +31,20 @@ public static Type FormatterType public static ICLRObjectStorer WrappersStorer { get; set; } + /// + /// Clears the old "clr_data" entry if a previous one is present. + /// + static void ClearCLRData () + { + IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddressUnchecked(); + if (capsule != IntPtr.Zero) + { + IntPtr oldData = PyCapsule_GetPointer(capsule, null); + PyMem_Free(oldData); + PyCapsule_SetPointer(capsule, IntPtr.Zero); + } + } + internal static void Stash() { var metaStorage = new RuntimeDataStorage(); @@ -70,15 +84,10 @@ internal static void Stash() Marshal.WriteIntPtr(mem, (IntPtr)ms.Length); Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); - IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddressUnchecked(); - if (capsule != IntPtr.Zero) - { - IntPtr oldData = PyCapsule_GetPointer(capsule, null); - PyMem_Free(oldData); - PyCapsule_SetPointer(capsule, IntPtr.Zero); - } - capsule = PyCapsule_New(mem, null, IntPtr.Zero); + ClearCLRData(); + IntPtr capsule = PyCapsule_New(mem, null, IntPtr.Zero); PySys_SetObject("clr_data", capsule); + // Let the dictionary own the reference XDecref(capsule); } From 2343f897aacc18dd3df15aa5c7a5233d7a79d176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Wed, 16 Sep 2020 11:22:35 -0400 Subject: [PATCH 127/134] Partially Revert "Adds an unchecked version to get a BorrowedReference pointer" We Don't support python 2 anymore, but the CI machines may still be using it to build. --- tools/geninterop/geninterop.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index da6c3733a..aacc4af65 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -21,7 +21,10 @@ import sysconfig import subprocess -from StringIO import StringIO +if sys.version_info.major > 2: + from io import StringIO +else: + from StringIO import StringIO from pycparser import c_ast, c_parser From cc6b8e4f177c6f835b558b9a68022847718e8fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 24 Sep 2020 14:38:24 -0400 Subject: [PATCH 128/134] More changes to use BorrowedReference and NewReference Also adds implicit IntPtr conversion operators to simplify their use. --- src/runtime/BorrowedReference.cs | 2 ++ src/runtime/NewReference.cs | 4 ++++ src/runtime/runtime.cs | 18 +++++++++++++++++- src/runtime/runtime_data.cs | 12 ++++++------ 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index ec4962db4..51fcf6f59 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -10,6 +10,8 @@ readonly ref struct BorrowedReference readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; + public static implicit operator IntPtr(in BorrowedReference self) => self.DangerousGetAddress(); + /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index 89d53bb36..733d45255 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -15,6 +15,10 @@ ref struct NewReference public static implicit operator BorrowedReference(in NewReference reference) => new BorrowedReference(reference.pointer); + [Pure] + public static implicit operator IntPtr(in NewReference reference) + => DangerousGetAddress(reference); + /// /// Returns wrapper around this reference, which now owns /// the pointer. Sets the original reference to null, as it no longer owns it. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index bf0065ea1..87ca3e600 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -748,6 +748,22 @@ internal static IntPtr SelfIncRef(IntPtr op) return op; } + /// + /// We need this method because BorrowedReference can be implicitly casted to IntPtr. + /// + internal static void XDecref(BorrowedReference op) + { + throw new InvalidOperationException("Cannot DecRef a borrowed reference."); + } + + /// + /// We need this method because NewReference can be implicitly casted to IntPtr. + /// + internal static void XDecref(NewReference op) + { + op.Dispose(); + } + internal static unsafe void XDecref(IntPtr op) { #if PYTHON_WITH_PYDEBUG || NETSTANDARD @@ -2129,7 +2145,7 @@ internal static void Py_CLEAR(ref IntPtr ob) //==================================================================== [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyCapsule_GetPointer(IntPtr capsule, string name); diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 9b2fdbe24..29d72dcd8 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -36,8 +36,8 @@ public static Type FormatterType /// static void ClearCLRData () { - IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddressUnchecked(); - if (capsule != IntPtr.Zero) + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (!capsule.IsNull) { IntPtr oldData = PyCapsule_GetPointer(capsule, null); PyMem_Free(oldData); @@ -85,10 +85,10 @@ internal static void Stash() Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); ClearCLRData(); - IntPtr capsule = PyCapsule_New(mem, null, IntPtr.Zero); + NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); PySys_SetObject("clr_data", capsule); // Let the dictionary own the reference - XDecref(capsule); + capsule.Dispose(); } internal static void RestoreRuntimeData() @@ -105,8 +105,8 @@ internal static void RestoreRuntimeData() private static void RestoreRuntimeDataImpl() { - IntPtr capsule = PySys_GetObject("clr_data").DangerousGetAddressUnchecked(); - if (capsule == IntPtr.Zero) + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (capsule.IsNull) { return; } From d5fcfa4fc4ae610e1c961e3136072008b3302d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Thu, 24 Sep 2020 16:10:51 -0400 Subject: [PATCH 129/134] fixup! More changes to use BorrowedReference and NewReference --- src/runtime/BorrowedReference.cs | 1 - src/runtime/NewReference.cs | 4 ---- src/runtime/runtime.cs | 20 ++------------------ 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index 51fcf6f59..eacf62bb5 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -10,7 +10,6 @@ readonly ref struct BorrowedReference readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; - public static implicit operator IntPtr(in BorrowedReference self) => self.DangerousGetAddress(); /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs index 733d45255..89d53bb36 100644 --- a/src/runtime/NewReference.cs +++ b/src/runtime/NewReference.cs @@ -15,10 +15,6 @@ ref struct NewReference public static implicit operator BorrowedReference(in NewReference reference) => new BorrowedReference(reference.pointer); - [Pure] - public static implicit operator IntPtr(in NewReference reference) - => DangerousGetAddress(reference); - /// /// Returns wrapper around this reference, which now owns /// the pointer. Sets the original reference to null, as it no longer owns it. diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 87ca3e600..5a34b30cf 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -748,22 +748,6 @@ internal static IntPtr SelfIncRef(IntPtr op) return op; } - /// - /// We need this method because BorrowedReference can be implicitly casted to IntPtr. - /// - internal static void XDecref(BorrowedReference op) - { - throw new InvalidOperationException("Cannot DecRef a borrowed reference."); - } - - /// - /// We need this method because NewReference can be implicitly casted to IntPtr. - /// - internal static void XDecref(NewReference op) - { - op.Dispose(); - } - internal static unsafe void XDecref(IntPtr op) { #if PYTHON_WITH_PYDEBUG || NETSTANDARD @@ -2148,10 +2132,10 @@ internal static void Py_CLEAR(ref IntPtr ob) internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyCapsule_GetPointer(IntPtr capsule, string name); + internal static extern IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyCapsule_SetPointer(IntPtr capsule, IntPtr pointer); + internal static extern int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer); //==================================================================== // Miscellaneous From fa479579aee65883b8759bdcdb32e87e56219fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 25 Sep 2020 10:59:31 -0400 Subject: [PATCH 130/134] fixup! fixup! More changes to use BorrowedReference and NewReference --- src/runtime/runtime_data.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index 29d72dcd8..060573db4 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -86,7 +86,7 @@ internal static void Stash() ClearCLRData(); NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); - PySys_SetObject("clr_data", capsule); + PySys_SetObject("clr_data", capsule.DangerousGetAddress()); // Let the dictionary own the reference capsule.Dispose(); } From ff956e4e56a73b1012252cd41447b00676ae9ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= Date: Fri, 25 Sep 2020 16:12:51 -0400 Subject: [PATCH 131/134] Remove unused method --- src/runtime/BorrowedReference.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index eacf62bb5..8ae382e77 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -15,12 +15,6 @@ readonly ref struct BorrowedReference public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; - /// - /// Gets a raw pointer to the Python object. Does not throw an exception - /// if the pointer is null - /// - public IntPtr DangerousGetAddressUnchecked() => this.pointer; - /// /// Creates new instance of from raw pointer. Unsafe. /// From 0b9d2c1e2c54635b71e3d32bbb766d836b7d5c0d Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 30 Sep 2020 09:58:17 +0800 Subject: [PATCH 132/134] Wait for full GC Complete --- src/embed_tests/TestFinalizer.cs | 8 ++++++++ src/runtime/runtime.cs | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 5cb210b32..357580daa 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -28,6 +28,14 @@ public void TearDown() private static void FullGCCollect() { GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); + try + { + GC.WaitForFullGCComplete(); + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + } GC.WaitForPendingFinalizers(); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 56a2035cb..68e3891b1 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -391,12 +391,13 @@ internal static void Shutdown(ShutdownMode mode) GC.Collect(); try { - GC.WaitForPendingFinalizers(); + GC.WaitForFullGCComplete(); } catch (NotImplementedException) { // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. } + GC.WaitForPendingFinalizers(); PyGILState_Release(state); // Then release the GIL for good, if there is somehting to release // Use the unchecked version as the checked version calls `abort()` From 178cbc86c954eee1f974fc9b5694ca074d746b32 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 30 Sep 2020 10:29:46 +0800 Subject: [PATCH 133/134] Release atexit manually --- src/runtime/runtime.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 68e3891b1..b54c4d947 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -463,14 +463,17 @@ private static void RunExitFuncs() // The runtime may not provided `atexit` module. return; } - try + using (atexit) { - atexit.InvokeMethod("_run_exitfuncs").Dispose(); - } - catch (PythonException e) - { - Console.Error.WriteLine(e); - e.Dispose(); + try + { + atexit.InvokeMethod("_run_exitfuncs").Dispose(); + } + catch (PythonException e) + { + Console.Error.WriteLine(e); + e.Dispose(); + } } } From 3a17f365346d7252ec9762be7d1c1f37f8a38010 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 30 Sep 2020 10:40:13 +0800 Subject: [PATCH 134/134] * Remove `fromPython` * Fix dead lock occur by calling PyEval_RestoreThread without check * Ignore `CollectOnShutdown` temporarily --- src/embed_tests/TestFinalizer.cs | 58 +++++++++++++++++++++++++++----- src/runtime/pythonengine.cs | 13 +++---- src/runtime/runtime.cs | 7 ++-- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 357580daa..a54bc7a96 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -1,6 +1,9 @@ using NUnit.Framework; using Python.Runtime; using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; using System.Linq; using System.Threading; @@ -25,18 +28,22 @@ public void TearDown() PythonEngine.Shutdown(); } - private static void FullGCCollect() + private static bool FullGCCollect() { GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); try { - GC.WaitForFullGCComplete(); + return GC.WaitForFullGCComplete() == GCNotificationStatus.Succeeded; } catch (NotImplementedException) { // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + return false; + } + finally + { + GC.WaitForPendingFinalizers(); } - GC.WaitForPendingFinalizers(); } [Test] @@ -96,23 +103,33 @@ public void CollectBasicObject() } [Test] + [Ignore("Ignore temporarily")] public void CollectOnShutdown() { - MakeAGarbage(out var shortWeak, out var longWeak); - FullGCCollect(); - var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.IsNotEmpty(garbage); + IntPtr op = MakeAGarbage(out var shortWeak, out var longWeak); + int hash = shortWeak.Target.GetHashCode(); + List garbage; + if (!FullGCCollect()) + { + Assert.IsTrue(WaitForCollected(op, hash, 10000)); + } + Assert.IsFalse(shortWeak.IsAlive); + garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.IsNotEmpty(garbage, "The garbage object should be collected"); + Assert.IsTrue(garbage.Any(r => ReferenceEquals(r.Target, longWeak.Target)), + "Garbage should contains the collected object"); + PythonEngine.Shutdown(); garbage = Finalizer.Instance.GetCollectedObjects(); Assert.IsEmpty(garbage); } - private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + private static IntPtr MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) { PyLong obj = new PyLong(1024); shortWeak = new WeakReference(obj); longWeak = new WeakReference(obj, true); - obj = null; + return obj.Handle; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) @@ -269,5 +286,28 @@ private static IntPtr CreateStringGarbage() return s1.Handle; } + private static bool WaitForCollected(IntPtr op, int hash, int milliseconds) + { + var stopwatch = Stopwatch.StartNew(); + do + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + foreach (var item in garbage) + { + // The validation is not 100% precise, + // but it's rare that two conditions satisfied but they're still not the same object. + if (item.Target.GetHashCode() != hash) + { + continue; + } + var obj = (IPyDisposable)item.Target; + if (obj.GetTrackedHandles().Contains(op)) + { + return true; + } + } + } while (stopwatch.ElapsedMilliseconds < milliseconds); + return false; + } } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index f305b3795..1d688ef9a 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -155,9 +155,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode, fromPython: fromPython); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); } /// @@ -170,7 +170,7 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false, Shu /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { if (initialized) { @@ -182,7 +182,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs, mode, fromPython); + Runtime.Initialize(initSigs, mode); initialized = true; Exceptions.Clear(); @@ -234,7 +234,8 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // add the imported module to the clr module, and copy the API functions // and decorators into the main clr module. Runtime.PyDict_SetItemString(clr_dict, "_extras", module); - foreach (PyObject key in locals.Keys()) + using (var keys = locals.Keys()) + foreach (PyObject key in keys) { if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { @@ -265,7 +266,7 @@ public static IntPtr InitExt() { try { - Initialize(setSysArgv: false, fromPython: true); + Initialize(setSysArgv: false); // Trickery - when the import hook is installed into an already // running Python, the standard import machinery is still in diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b54c4d947..83d404f9d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -125,7 +125,7 @@ internal static Version PyVersion /// /// Always call this method from the Main thread. After the /// first call to this method, the main thread has acquired the GIL. - internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default, bool fromPython = false) + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { if (_isInitialized) { @@ -139,7 +139,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } ShutdownMode = mode; - IntPtr state = IntPtr.Zero; if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); @@ -154,12 +153,12 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd RuntimeState.Save(); } } - else if (!fromPython) + else { // If we're coming back from a domain reload or a soft shutdown, // we have previously released the thread state. Restore the main // thread state here. - PyEval_RestoreThread(PyGILState_GetThisThreadState()); + PyGILState_Ensure(); } MainManagedThreadId = Thread.CurrentThread.ManagedThreadId;