From 8fa031ebfe5ad1902625502a79dd0870b247edab Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Sun, 19 Dec 2021 20:26:13 -0800 Subject: [PATCH 1/4] `ShutdownMode` has been removed. The only shutdown mode supported now is an equivalent of `ShutdownMode.Reload` also in this change: - fixed Python derived types not being decrefed when an instance is deallocated - reduced time and amount of storage needed for runtime reload - removed circular reference loop between Type <-> ConstructorBinding(s) + exposed Runtime.TryCollectingGarbage --- .github/workflows/main.yml | 4 - CHANGELOG.md | 2 + src/embed_tests/Codecs.cs | 20 +-- src/embed_tests/TestCallbacks.cs | 8 +- src/embed_tests/TestDomainReload.cs | 34 +---- src/embed_tests/pyinitialize.cs | 45 ------ src/runtime/ReflectedClrType.cs | 5 +- .../StateSerialization/ClassManagerState.cs | 2 +- .../StateSerialization/ICLRObjectStorer.cs | 4 +- .../StateSerialization/SharedObjectsState.cs | 4 +- src/runtime/Util.cs | 2 +- src/runtime/classbase.cs | 20 +-- src/runtime/classderived.cs | 8 +- src/runtime/classmanager.cs | 11 +- src/runtime/clrobject.cs | 4 +- src/runtime/constructorbinding.cs | 32 +--- src/runtime/extensiontype.cs | 2 +- src/runtime/finalizer.cs | 2 +- src/runtime/importhook.cs | 2 +- src/runtime/interop.cs | 2 +- src/runtime/managedtype.cs | 11 +- src/runtime/methodobject.cs | 8 +- src/runtime/moduleobject.cs | 34 ++++- src/runtime/pythonengine.cs | 50 +++---- src/runtime/runtime.cs | 139 +++++------------- src/runtime/runtime_data.cs | 52 ++----- src/runtime/runtime_state.cs | 2 +- src/runtime/typemanager.cs | 8 +- tests/domain_tests/TestRunner.cs | 13 +- tests/domain_tests/test_domain_reload.py | 1 - 30 files changed, 174 insertions(+), 357 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53a0f3701..8276be16b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,10 +14,6 @@ jobs: os: [windows, ubuntu, macos] python: ["3.6", "3.7", "3.8", "3.9", "3.10"] platform: [x64] - shutdown_mode: [Normal, Soft] - - env: - PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }} steps: - name: Set Environment on macOS diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a01e69fc..86c2a808a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,8 @@ Instead, `PyIterable` does that. ### Removed +- `ShutdownMode` has been removed. The only shutdown mode supported now is an equivalent of `ShutdownMode.Reload`. +There is no need to specify it. - implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) - messages in `PythonException` no longer start with exception type - `PyScopeManager`, `PyScopeException`, `PyScope` (use `PyModule` instead) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 157e60803..a87b287bc 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -34,7 +34,7 @@ static void TupleConversionsGeneric() using (var scope = Py.CreateScope()) { void Accept(T value) => restored = value; - var accept = new Action(Accept).ToPython(); + using var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); scope.Set(nameof(accept), accept); scope.Exec($"{nameof(accept)}({nameof(tuple)})"); @@ -55,7 +55,7 @@ static void TupleConversionsObject() using (var scope = Py.CreateScope()) { void Accept(object value) => restored = (T)value; - var accept = new Action(Accept).ToPython(); + using var accept = new Action(Accept).ToPython(); scope.Set(nameof(tuple), tuple); scope.Set(nameof(accept), accept); scope.Exec($"{nameof(accept)}({nameof(tuple)})"); @@ -71,7 +71,7 @@ public void TupleRoundtripObject() static void TupleRoundtripObject() { var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object()); - var pyTuple = TupleCodec.Instance.TryEncode(tuple); + using var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); Assert.AreEqual(expected: tuple, actual: restored); } @@ -85,7 +85,7 @@ public void TupleRoundtripGeneric() static void TupleRoundtripGeneric() { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - var pyTuple = TupleCodec.Instance.TryEncode(tuple); + using var pyTuple = TupleCodec.Instance.TryEncode(tuple); Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); Assert.AreEqual(expected: tuple, actual: restored); } @@ -98,9 +98,9 @@ public void ListDecoderTest() var codec = ListDecoder.Instance; var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; - var pyList = new PyList(items.ToArray()); + using var pyList = new PyList(items.ToArray()); - var pyListType = pyList.GetPythonType(); + using var pyListType = pyList.GetPythonType(); Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); @@ -128,8 +128,8 @@ public void ListDecoderTest() Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; }); //can't convert python iterable to list (this will require a copy which isn't lossless) - var foo = GetPythonIterable(); - var fooType = foo.GetPythonType(); + using var foo = GetPythonIterable(); + using var fooType = foo.GetPythonType(); Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); } @@ -140,8 +140,8 @@ public void SequenceDecoderTest() var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; //SequenceConverter can only convert to any ICollection - var pyList = new PyList(items.ToArray()); - var listType = pyList.GetPythonType(); + using var pyList = new PyList(items.ToArray()); + using var listType = pyList.GetPythonType(); //it can convert a PyList, since PyList satisfies the python sequence protocol Assert.IsFalse(codec.CanDecode(listType, typeof(bool))); diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs index 6875fde01..88b84d0c3 100644 --- a/src/embed_tests/TestCallbacks.cs +++ b/src/embed_tests/TestCallbacks.cs @@ -4,8 +4,6 @@ using Python.Runtime; namespace Python.EmbeddingTest { - using Runtime = Python.Runtime.Runtime; - public class TestCallbacks { [OneTimeSetUp] public void SetUp() { @@ -22,11 +20,13 @@ public void TestNoOverloadException() { int passed = 0; var aFunctionThatCallsIntoPython = new Action(value => passed = value); using (Py.GIL()) { - dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); - var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); + using dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); + using var pyFunc = aFunctionThatCallsIntoPython.ToPython(); + var error = Assert.Throws(() => callWith42(pyFunc)); Assert.AreEqual("TypeError", error.Type.Name); string expectedArgTypes = "()"; StringAssert.EndsWith(expectedArgTypes, error.Message); + error.Traceback.Dispose(); } } } diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index a0619a93f..498119d1e 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -67,14 +67,6 @@ public static void DomainReloadAndGC() RunAssemblyAndUnload("test2"); Assert.That(PyRuntime.Py_IsInitialized() != 0, "On soft-shutdown mode, Python runtime should still running"); - - if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) - { - // The default mode is a normal mode, - // it should shutdown the Python VM avoiding influence other tests. - PyRuntime.PyGILState_Ensure(); - PyRuntime.Py_Finalize(); - } } #region CrossDomainObject @@ -222,7 +214,7 @@ static void RunAssemblyAndUnload(string domainName) // assembly (and Python .NET) to reside var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL); + theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL); // From now on use the Proxy to call into the new assembly theProxy.RunPython(); @@ -290,7 +282,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro try { var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL); + theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL); var caller = CreateInstanceInstanceAndUnwrap(domain); arg = caller.Execute(arg); @@ -308,7 +300,7 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro try { var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL); + theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL); var caller = CreateInstanceInstanceAndUnwrap(domain); caller.Execute(arg); @@ -319,10 +311,8 @@ static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : Cro AppDomain.Unload(domain); } } - if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) - { - Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0); - } + + Assert.IsTrue(PyRuntime.Py_IsInitialized() != 0); } } @@ -368,10 +358,10 @@ public static void RunPython() private static IntPtr _state; - public static void InitPython(ShutdownMode mode, string dllName) + public static void InitPython(string dllName) { PyRuntime.PythonDLL = dllName; - PythonEngine.Initialize(mode: mode); + PythonEngine.Initialize(); _state = PythonEngine.BeginAllowThreads(); } @@ -384,15 +374,7 @@ public static void ShutdownPython() public static void ShutdownPythonCompletely() { PythonEngine.EndAllowThreads(_state); - // XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`, - // if it used a another mode(the default mode) in other tests, - // when other tests trying to access these reserved objects, it may cause Domain exception, - // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python. - var defaultMode = PythonEngine.DefaultShutdownMode; - if (defaultMode != ShutdownMode.Reload) - { - PythonEngine.ShutdownMode = defaultMode; - } + PythonEngine.Shutdown(); } diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index 8c9d6d251..25dafb686 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -151,51 +151,6 @@ public void ShutdownHandlers() // Wrong: (4 * 2) + 1 + 1 + 1 = 11 Assert.That(shutdown_count, Is.EqualTo(12)); } - - [Test] - public static void TestRunExitFuncs() - { - if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal) - { - // If the runtime using the normal mode, - // callback registered by atexit will be called after we release the clr information, - // thus there's no chance we can check it here. - Assert.Ignore("Skip on normal mode"); - } - Runtime.Runtime.Initialize(); - PyObject atexit; - try - { - atexit = Py.Import("atexit"); - } - catch (PythonException e) - { - string msg = e.ToString(); - bool isImportError = e.Is(Exceptions.ImportError); - Runtime.Runtime.Shutdown(); - - if (isImportError) - { - Assert.Ignore("no atexit module"); - } - else - { - Assert.Fail(msg); - } - PythonEngine.InteropConfiguration = InteropConfiguration.MakeDefault(); - return; - } - bool called = false; - Action callback = () => - { - called = true; - }; - atexit.InvokeMethod("register", callback.ToPython()).Dispose(); - atexit.Dispose(); - Runtime.Runtime.Shutdown(); - Assert.True(called); - PythonEngine.InteropConfiguration = InteropConfiguration.MakeDefault(); - } } public class ImportClassShutdownRefcountClass { } diff --git a/src/runtime/ReflectedClrType.cs b/src/runtime/ReflectedClrType.cs index 93c28fd87..15ea5c2b2 100644 --- a/src/runtime/ReflectedClrType.cs +++ b/src/runtime/ReflectedClrType.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.Runtime.Serialization; @@ -49,9 +50,9 @@ public static ReflectedClrType GetOrCreate(Type type) return pyType; } - internal void Restore(InterDomainContext context) + internal void Restore(Dictionary context) { - var cb = context.Storage.GetValue("impl"); + var cb = (ClassBase)context["impl"]!; Debug.Assert(cb is not null); diff --git a/src/runtime/StateSerialization/ClassManagerState.cs b/src/runtime/StateSerialization/ClassManagerState.cs index 093e5d41c..01c9c472c 100644 --- a/src/runtime/StateSerialization/ClassManagerState.cs +++ b/src/runtime/StateSerialization/ClassManagerState.cs @@ -10,6 +10,6 @@ namespace Python.Runtime.StateSerialization; [Serializable] internal class ClassManagerState { - public Dictionary Contexts { get; set; } + public Dictionary> Contexts { get; set; } public Dictionary Cache { get; set; } } diff --git a/src/runtime/StateSerialization/ICLRObjectStorer.cs b/src/runtime/StateSerialization/ICLRObjectStorer.cs index b87339cd5..547115284 100644 --- a/src/runtime/StateSerialization/ICLRObjectStorer.cs +++ b/src/runtime/StateSerialization/ICLRObjectStorer.cs @@ -4,6 +4,6 @@ namespace Python.Runtime; public interface ICLRObjectStorer { - ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); - CLRWrapperCollection Restore(RuntimeDataStorage storage); + ICollection Store(CLRWrapperCollection wrappers, Dictionary storage); + CLRWrapperCollection Restore(Dictionary storage); } diff --git a/src/runtime/StateSerialization/SharedObjectsState.cs b/src/runtime/StateSerialization/SharedObjectsState.cs index 0375007d6..6c7516623 100644 --- a/src/runtime/StateSerialization/SharedObjectsState.cs +++ b/src/runtime/StateSerialization/SharedObjectsState.cs @@ -12,6 +12,6 @@ internal class SharedObjectsState { public Dictionary InternalStores { get; init; } public Dictionary Extensions { get; init; } - public RuntimeDataStorage Wrappers { get; init; } - public Dictionary Contexts { get; init; } + public Dictionary Wrappers { get; init; } + public Dictionary> Contexts { get; init; } } diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index bbed4ad0a..f5f0d2957 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -13,7 +13,7 @@ internal static class Util internal const string UnstableApiMessage = "This API is unstable, and might be changed or removed in the next minor release"; internal const string MinimalPythonVersionRequired = - "Only Python 3.5 or newer is supported"; + "Only Python 3.6 or newer is supported"; internal const string InternalUseOnly = "This API is for internal use only"; diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 9ef7c626c..349d20b6b 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -341,16 +341,17 @@ public static void tp_dealloc(NewReference lastRef) CallClear(lastRef.Borrow()); - IntPtr addr = lastRef.DangerousGetAddress(); - bool deleted = CLRObject.reflectedObjects.Remove(addr); - Debug.Assert(deleted); - DecrefTypeAndFree(lastRef.Steal()); } public static int tp_clear(BorrowedReference ob) { - TryFreeGCHandle(ob); + if (TryFreeGCHandle(ob)) + { + IntPtr addr = ob.DangerousGetAddress(); + bool deleted = CLRObject.reflectedObjects.Remove(addr); + Debug.Assert(deleted); + } int baseClearResult = BaseUnmanagedClear(ob); if (baseClearResult != 0) @@ -390,13 +391,14 @@ internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) return clear(ob); } - protected override void OnSave(BorrowedReference ob, InterDomainContext context) + protected override Dictionary OnSave(BorrowedReference ob) { - base.OnSave(ob, context); - context.Storage.AddValue("impl", this); + var context = base.OnSave(ob) ?? new(); + context["impl"] = this; + return context; } - protected override void OnLoad(BorrowedReference ob, InterDomainContext? context) + protected override void OnLoad(BorrowedReference ob, Dictionary? context) { base.OnLoad(ob, context); var gcHandle = GCHandle.Alloc(this); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index a39f23bef..5b9e630ca 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -867,17 +867,19 @@ public static void PyFinalize(IPythonDerivedType obj) internal static void Finalize(IntPtr derived) { - bool deleted = CLRObject.reflectedObjects.Remove(derived); - Debug.Assert(deleted); - var @ref = NewReference.DangerousFromPointer(derived); ClassBase.tp_clear(@ref.Borrow()); + var type = Runtime.PyObject_TYPE(@ref.Borrow()); + // rare case when it's needed // matches correspdonging PyObject_GC_UnTrack // in ClassDerivedObject.tp_dealloc Runtime.PyObject_GC_Del(@ref.Steal()); + + // must decref our type + Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress())); } internal static FieldInfo? GetPyObjField(Type type) => type.GetField(PyObjName, PyObjFlags); diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 9e15b2bd1..bfc07874f 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -65,12 +65,15 @@ internal static void RemoveClasses() internal static ClassManagerState SaveRuntimeData() { - var contexts = new Dictionary(); + var contexts = new Dictionary>(); foreach (var cls in cache) { - var context = contexts[cls.Value] = new InterDomainContext(); var cb = (ClassBase)ManagedType.GetManagedObject(cls.Value)!; - cb.Save(cls.Value, context); + var context = cb.Save(cls.Value); + if (context is not null) + { + contexts[cls.Value] = context; + } // Remove all members added in InitBaseClass. // this is done so that if domain reloads and a member of a @@ -201,7 +204,7 @@ internal static ClassBase CreateClass(Type type) return impl; } - internal static void InitClassBase(Type type, ClassBase impl, PyType pyType) + internal static void InitClassBase(Type type, ClassBase impl, ReflectedClrType pyType) { // First, we introspect the managed type and build some class // information, including generating the member descriptors diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index c178ca459..db6e99121 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -53,13 +53,13 @@ internal static NewReference GetReference(object ob) return Create(ob, cc); } - internal static void Restore(object ob, BorrowedReference pyHandle, InterDomainContext context) + internal static void Restore(object ob, BorrowedReference pyHandle, Dictionary context) { var co = new CLRObject(ob); co.OnLoad(pyHandle, context); } - protected override void OnLoad(BorrowedReference ob, InterDomainContext? context) + protected override void OnLoad(BorrowedReference ob, Dictionary? context) { base.OnLoad(ob, context); GCHandle gc = GCHandle.Alloc(this); diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 780db6424..4f82c7728 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Reflection; namespace Python.Runtime @@ -23,16 +24,15 @@ namespace Python.Runtime internal class ConstructorBinding : ExtensionType { private MaybeType type; // The managed Type being wrapped in a ClassObject - private PyType typeToCreate; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; [NonSerialized] private PyObject? repr; - public ConstructorBinding(Type type, PyType typeToCreate, ConstructorBinder ctorBinder) + public ConstructorBinding(Type type, ReflectedClrType typeToCreate, ConstructorBinder ctorBinder) { this.type = type; - this.typeToCreate = typeToCreate; + Debug.Assert(typeToCreate == ReflectedClrType.GetOrCreate(type)); this.ctorBinder = ctorBinder; } @@ -109,7 +109,7 @@ public static NewReference mp_subscript(BorrowedReference op, BorrowedReference { return Exceptions.RaiseTypeError("No match found for constructor signature"); } - var boundCtor = new BoundContructor(tp, self.typeToCreate, self.ctorBinder, ci); + var boundCtor = new BoundContructor(tp, self.ctorBinder, ci); return boundCtor.Alloc(); } @@ -146,15 +146,6 @@ public static NewReference tp_repr(BorrowedReference ob) self.repr = docStr.MoveToPyObject(); return new NewReference(self.repr); } - - public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) - { - var self = (ConstructorBinding?)GetManagedObject(ob); - if (self is null) return 0; - - int res = PyVisit(self.typeToCreate, visit, arg); - return res; - } } /// @@ -169,15 +160,13 @@ public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject - private PyType typeToCreate; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; private ConstructorInfo ctorInfo; private PyObject? repr; - public BoundContructor(Type type, PyType typeToCreate, ConstructorBinder ctorBinder, ConstructorInfo ci) + public BoundContructor(Type type, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - this.typeToCreate = typeToCreate; this.ctorBinder = ctorBinder; ctorInfo = ci; } @@ -207,7 +196,7 @@ public static NewReference tp_call(BorrowedReference op, BorrowedReference args, } // Instantiate the python object that wraps the result of the method call // and return the PyObject* to it. - return CLRObject.GetReference(obj, self.typeToCreate); + return CLRObject.GetReference(obj, ReflectedClrType.GetOrCreate(self.type)); } /// @@ -229,14 +218,5 @@ public static NewReference tp_repr(BorrowedReference ob) self.repr = docStr.MoveToPyObject(); return new NewReference(self.repr); } - - public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) - { - var self = (BoundContructor?)GetManagedObject(ob); - if (self is null) return 0; - - int res = PyVisit(self.typeToCreate, visit, arg); - return res; - } } } diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index e3f049e1a..4ed5d8417 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -95,7 +95,7 @@ public static int tp_clear(BorrowedReference ob) return res; } - protected override void OnLoad(BorrowedReference ob, InterDomainContext? context) + protected override void OnLoad(BorrowedReference ob, Dictionary? context) { base.OnLoad(ob, context); SetupGc(ob, Runtime.PyObject_TYPE(ob)); diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 09ffe5c06..bfb1e228d 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -27,7 +27,7 @@ public ErrorArgs(Exception error) public Exception Error { get; } } - public static readonly Finalizer Instance = new Finalizer(); + public static Finalizer Instance { get; } = new (); public event EventHandler? BeforeCollect; public event EventHandler? ErrorHandler; diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index b40fa2cd6..b82c503b5 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -87,11 +87,11 @@ internal static void Shutdown() } TeardownNameSpaceTracking(); + clrModule.ResetModuleMembers(); Runtime.Py_CLEAR(ref py_clr_module!); root.Dispose(); root = null!; - CLRModule.Reset(); } private static Dictionary GetDotNetModules() diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index adaac1b6e..bcf99bede 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -61,7 +61,7 @@ public ModulePropertyAttribute() /// // Py_TPFLAGS_* [Flags] - public enum TypeFlags: int + public enum TypeFlags: long { HeapType = (1 << 9), BaseType = (1 << 10), diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 91ed43473..c71529628 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -109,19 +109,18 @@ internal static unsafe int CallTypeClear(BorrowedReference ob, BorrowedReference return clearFunc(ob); } - internal void Save(BorrowedReference ob, InterDomainContext context) + internal Dictionary? Save(BorrowedReference ob) { - OnSave(ob, context); + return OnSave(ob); } -#warning context appears to be unused - internal void Load(BorrowedReference ob, InterDomainContext? context) + internal void Load(BorrowedReference ob, Dictionary? context) { OnLoad(ob, context); } - protected virtual void OnSave(BorrowedReference ob, InterDomainContext context) { } - protected virtual void OnLoad(BorrowedReference ob, InterDomainContext? context) { } + protected virtual Dictionary? OnSave(BorrowedReference ob) => null; + protected virtual void OnLoad(BorrowedReference ob, Dictionary? context) { } protected static void ClearObjectDict(BorrowedReference ob) { diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 14e26c86d..ce64a3f7c 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -21,7 +21,6 @@ internal class MethodObject : ExtensionType private MethodInfo[]? _info = null; private readonly List infoList; internal string name; - internal PyObject? unbound; internal readonly MethodBinder binder; internal bool is_static = false; @@ -164,11 +163,8 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference if (ob == null) { - if (self.unbound is null) - { - self.unbound = new PyObject(new MethodBinding(self, target: null, targetType: new PyType(tp)).Alloc().Steal()); - } - return new NewReference(self.unbound); + var binding = new MethodBinding(self, target: null, targetType: new PyType(tp)); + return binding.Alloc(); } if (Runtime.PyObject_IsInstance(ob, tp) < 1) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 68acaf022..1e86d4472 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -204,6 +204,7 @@ public void LoadNames() } } + const BindingFlags ModuleMethodFlags = BindingFlags.Public | BindingFlags.Static; /// /// Initialize module level functions and attributes /// @@ -214,11 +215,9 @@ internal void InitializeModuleMembers() Type ftmarker = typeof(ForbidPythonThreadsAttribute); Type type = GetType(); - BindingFlags flags = BindingFlags.Public | BindingFlags.Static; - while (type != null) { - MethodInfo[] methods = type.GetMethods(flags); + MethodInfo[] methods = type.GetMethods(ModuleMethodFlags); foreach (MethodInfo method in methods) { object[] attrs = method.GetCustomAttributes(funcmarker, false); @@ -249,6 +248,28 @@ internal void InitializeModuleMembers() } } + internal void ResetModuleMembers() + { + Type type = GetType(); + var methods = type.GetMethods(ModuleMethodFlags) + .Where(m => m.GetCustomAttribute() is not null) + .OfType(); + var properties = type.GetProperties().Where(p => p.GetCustomAttribute() is not null); + + foreach (string memberName in methods.Concat(properties).Select(m => m.Name)) + { + if (Runtime.PyDict_DelItemString(dict, memberName) != 0) + { + if (!PythonException.CurrentMatches(Exceptions.KeyError)) + { + throw PythonException.ThrowLastAsClrException(); + } + Runtime.PyErr_Clear(); + } + cache.Remove(memberName); + } + } + /// /// ModuleObject __getattribute__ implementation. Module attributes @@ -353,9 +374,9 @@ public static int tp_traverse(BorrowedReference ob, IntPtr visit, IntPtr arg) return ExtensionType.tp_setattro(ob, key, val); } - protected override void OnSave(BorrowedReference ob, InterDomainContext context) + protected override Dictionary? OnSave(BorrowedReference ob) { - base.OnSave(ob, context); + var context = base.OnSave(ob); System.Diagnostics.Debug.Assert(dict == GetObjectDict(ob)); // destroy the cache(s) foreach (var pair in cache) @@ -374,9 +395,10 @@ protected override void OnSave(BorrowedReference ob, InterDomainContext context) } cache.Clear(); + return context; } - protected override void OnLoad(BorrowedReference ob, InterDomainContext? context) + protected override void OnLoad(BorrowedReference ob, Dictionary? context) { base.OnLoad(ob, context); SetObjectDict(ob, new NewReference(dict).Steal()); diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index f3b7fa770..c93443025 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -17,14 +17,6 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { - public static ShutdownMode ShutdownMode - { - get => Runtime.ShutdownMode; - set => Runtime.ShutdownMode = value; - } - - public static ShutdownMode DefaultShutdownMode => Runtime.GetDefaultShutdownMode(); - private static DelegateManager? delegateManager; private static bool initialized; private static IntPtr _pythonHome = IntPtr.Zero; @@ -182,9 +174,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + public static void Initialize(bool setSysArgv = true, bool initSigs = false) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); } /// @@ -197,7 +189,7 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false, Shu /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) { if (initialized) { @@ -209,7 +201,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs, mode); + Runtime.Initialize(initSigs); initialized = true; Exceptions.Clear(); @@ -318,7 +310,16 @@ public static IntPtr InitExt() { try { - Initialize(setSysArgv: false, mode: ShutdownMode.Extension); + if (Runtime.IsInitialized) + { + var builtins = Runtime.PyEval_GetBuiltins(); + var runtimeError = Runtime.PyDict_GetItemString(builtins, "RuntimeError"); + Exceptions.SetError(runtimeError, "Python.NET runtime is already initialized"); + return IntPtr.Zero; + } + Runtime.HostedInPython = true; + + Initialize(setSysArgv: false); Finalizer.Instance.ErrorHandler += AllowLeaksDuringShutdown; @@ -372,15 +373,11 @@ private static void AllowLeaksDuringShutdown(object sender, Finalizer.ErrorArgs } /// - /// Shutdown Method - /// - /// /// Shutdown and release resources held by the Python runtime. The /// Python runtime can no longer be used in the current process /// after calling the Shutdown method. - /// - /// The ShutdownMode to use when shutting down the Runtime - public static void Shutdown(ShutdownMode mode) + /// + public static void Shutdown() { if (!initialized) { @@ -393,26 +390,13 @@ public static void Shutdown(ShutdownMode mode) ExecuteShutdownHandlers(); // Remember to shut down the runtime. - Runtime.Shutdown(mode); + Runtime.Shutdown(); initialized = false; InteropConfiguration = InteropConfiguration.MakeDefault(); } - /// - /// Shutdown Method - /// - /// - /// Shutdown and release resources held by the Python runtime. The - /// Python runtime can no longer be used in the current process - /// after calling the Shutdown method. - /// - public static void Shutdown() - { - Shutdown(Runtime.ShutdownMode); - } - /// /// Called when the engine is shut down. /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2f1d36ac6..9a99c5d80 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -56,6 +56,7 @@ private static string GetDefaultDllName(Version version) private static bool _isInitialized = false; + internal static bool IsInitialized => _isInitialized; internal static readonly bool Is32Bit = IntPtr.Size == 4; // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) @@ -66,7 +67,6 @@ private static string GetDefaultDllName(Version version) public static int MainManagedThreadId { get; private set; } - public static ShutdownMode ShutdownMode { get; internal set; } private static readonly List _pyRefs = new (); internal static Version PyVersion @@ -91,11 +91,13 @@ internal static int GetRun() return runNumber; } + internal static bool HostedInPython; + /// Initialize the runtime... /// /// Always call this method from the Main thread. After the /// first call to this method, the main thread has acquired the GIL. - internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) + internal static void Initialize(bool initSigs = false) { if (_isInitialized) { @@ -103,12 +105,6 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } _isInitialized = true; - if (mode == ShutdownMode.Default) - { - mode = GetDefaultShutdownMode(); - } - ShutdownMode = mode; - bool interpreterAlreadyInitialized = TryUsingDll( () => Py_IsInitialized() != 0 ); @@ -122,19 +118,11 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd { PyEval_InitThreads(); } - // XXX: Reload mode may reduct to Soft mode, - // so even on Reload mode it still needs to save the RuntimeState - if (mode == ShutdownMode.Soft || mode == ShutdownMode.Reload) - { - RuntimeState.Save(); - } + RuntimeState.Save(); } else { - // If we're coming back from a domain reload or a soft shutdown, - // we have previously released the thread state. Restore the main - // thread state here. - if (mode != ShutdownMode.Extension) + if (!HostedInPython) { PyGILState_Ensure(); } @@ -167,7 +155,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); OperatorMethod.Initialize(); - if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) + if (RuntimeData.HasStashData()) { RuntimeData.RestoreRuntimeData(); } @@ -256,33 +244,7 @@ private static void InitPyMembers() return Util.ReadPtr(pyType.Borrow(), TypeOffset.tp_iternext); } - /// - /// Tries to downgrade the shutdown mode, if possible. - /// The only possibles downgrades are: - /// Soft -> Normal - /// Reload -> Soft - /// Reload -> Normal - /// - /// The desired shutdown mode - /// The `mode` parameter if the downgrade is supported, the ShutdownMode - /// set at initialization otherwise. - static ShutdownMode TryDowngradeShutdown(ShutdownMode mode) - { - if ( - mode == Runtime.ShutdownMode - || mode == ShutdownMode.Normal - || (mode == ShutdownMode.Soft && Runtime.ShutdownMode == ShutdownMode.Reload) - ) - { - return mode; - } - else // we can't downgrade - { - return Runtime.ShutdownMode; - } - } - - internal static void Shutdown(ShutdownMode mode) + internal static void Shutdown() { if (Py_IsInitialized() == 0 || !_isInitialized) { @@ -290,21 +252,16 @@ internal static void Shutdown(ShutdownMode mode) } _isInitialized = false; - // If the shutdown mode specified is not the the same as the one specified - // during Initialization, we need to validate it; we can only downgrade, - // not upgrade the shutdown mode. - mode = TryDowngradeShutdown(mode); - var state = PyGILState_Ensure(); - if (mode == ShutdownMode.Soft) - { - RunExitFuncs(); - } - if (mode == ShutdownMode.Reload) + if (!HostedInPython) { + // avoid saving dead objects + TryCollectingGarbage(runs: 3); + RuntimeData.Stash(); } + AssemblyManager.Shutdown(); OperatorMethod.Shutdown(); ImportHook.Shutdown(); @@ -314,7 +271,7 @@ internal static void Shutdown(ShutdownMode mode) NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); - TypeManager.RemoveTypes(mode); + TypeManager.RemoveTypes(); MetaType.Release(); PyCLRMetaType.Dispose(); @@ -327,18 +284,15 @@ internal static void Shutdown(ShutdownMode mode) PyObjectConversions.Reset(); PyGC_Collect(); - bool everythingSeemsCollected = TryCollectingGarbage(); + bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown, + forceBreakLoops: true); Debug.Assert(everythingSeemsCollected); Finalizer.Shutdown(); InternString.Shutdown(); - if (mode != ShutdownMode.Normal && mode != ShutdownMode.Extension) + if (!HostedInPython) { - if (mode == ShutdownMode.Soft) - { - RuntimeState.Restore(); - } ResetPyMembers(); GC.Collect(); GC.WaitForPendingFinalizers(); @@ -351,26 +305,23 @@ internal static void Shutdown(ShutdownMode mode) PyEval_SaveThread(); } + ExtensionType.loadedExtensions.Clear(); + CLRObject.reflectedObjects.Clear(); } else { ResetPyMembers(); - if (mode != ShutdownMode.Extension) - { - Py_Finalize(); - } - else - { - PyGILState_Release(state); - } + PyGILState_Release(state); } } const int MaxCollectRetriesOnShutdown = 20; internal static int _collected; - static bool TryCollectingGarbage() + static bool TryCollectingGarbage(int runs, bool forceBreakLoops) { - for (int attempt = 0; attempt < MaxCollectRetriesOnShutdown; attempt++) + if (runs <= 0) throw new ArgumentOutOfRangeException(nameof(runs)); + + for (int attempt = 0; attempt < runs; attempt++) { Interlocked.Exchange(ref _collected, 0); nint pyCollected = 0; @@ -383,21 +334,22 @@ static bool TryCollectingGarbage() } if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) { - return true; + if (attempt + 1 == runs) return true; } - else + else if (forceBreakLoops) { NullGCHandles(CLRObject.reflectedObjects); } } return false; } - - internal static void Shutdown() - { - var mode = ShutdownMode; - Shutdown(mode); - } + /// + /// Alternates .NET and Python GC runs in an attempt to collect all garbage + /// + /// Total number of GC loops to run + /// true if a steady state was reached upon the requested number of tries (e.g. on the last try no objects were collected). + public static bool TryCollectingGarbage(int runs) + => TryCollectingGarbage(runs, forceBreakLoops: false); static void DisposeLazyModule(Lazy module) { @@ -412,21 +364,6 @@ private static Lazy GetModuleLazy(string moduleName) ? throw new ArgumentNullException(nameof(moduleName)) : new Lazy(() => PyModule.Import(moduleName), isThreadSafe: false); - internal static ShutdownMode GetDefaultShutdownMode() - { - string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); - if (modeEvn == null) - { - return ShutdownMode.Normal; - } - ShutdownMode mode; - if (Enum.TryParse(modeEvn, true, out mode)) - { - return mode; - } - return ShutdownMode.Normal; - } - private static void RunExitFuncs() { PyObject atexit; @@ -2495,14 +2432,4 @@ internal class BadPythonDllException : MissingMethodException public BadPythonDllException(string message, Exception innerException) : base(message, innerException) { } } - - - public enum ShutdownMode - { - Default, - Normal, - Soft, - Reload, - Extension, - } } diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs index a4726b479..204e15b5b 100644 --- a/src/runtime/runtime_data.cs +++ b/src/runtime/runtime_data.cs @@ -138,7 +138,7 @@ static bool CheckSerializable (object o) private static SharedObjectsState SaveRuntimeDataObjects() { - var contexts = new Dictionary(PythonReferenceComparer.Instance); + var contexts = new Dictionary>(PythonReferenceComparer.Instance); var extensionObjs = new Dictionary(PythonReferenceComparer.Instance); // make a copy with strongly typed references to avoid concurrent modification var extensions = ExtensionType.loadedExtensions @@ -151,9 +151,11 @@ private static SharedObjectsState SaveRuntimeDataObjects() { var extension = (ExtensionType)ManagedType.GetManagedObject(pyObj)!; Debug.Assert(CheckSerializable(extension)); - var context = new InterDomainContext(); - contexts[pyObj] = context; - extension.Save(pyObj, context); + var context = extension.Save(pyObj); + if (context is not null) + { + contexts[pyObj] = context; + } extensionObjs.Add(pyObj, extension); } @@ -189,7 +191,7 @@ private static SharedObjectsState SaveRuntimeDataObjects() mappedObjs.Add(clrObj); } - var wrapperStorage = new RuntimeDataStorage(); + var wrapperStorage = new Dictionary(); WrappersStorer?.Store(userObjects, wrapperStorage); var internalStores = new Dictionary(PythonReferenceComparer.Instance); @@ -225,7 +227,8 @@ private static void RestoreRuntimeDataObjects(SharedObjectsState storage) var contexts = storage.Contexts; foreach (var extension in extensions) { - extension.Value.Load(extension.Key, contexts[extension.Key]); + contexts.TryGetValue(extension.Key, out var context); + extension.Value.Load(extension.Key, context); } foreach (var clrObj in internalStores) { @@ -254,41 +257,4 @@ internal static IFormatter CreateFormatter() : new BinaryFormatter(); } } - - - [Serializable] - public class RuntimeDataStorage - { - private Dictionary? _namedValues; - - public T AddValue(string name, T value) - { - if (_namedValues == null) - { - _namedValues = new Dictionary(); - } - _namedValues.Add(name, value); - return value; - } - - public object? GetValue(string name) - { - return _namedValues is null - ? throw new KeyNotFoundException() - : _namedValues[name]; - } - - public T? GetValue(string name) - { - return (T?)GetValue(name); - } - } - - - [Serializable] - class InterDomainContext - { - private RuntimeDataStorage? _storage; - public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); - } } diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs index 3cd842d39..2bb78094a 100644 --- a/src/runtime/runtime_state.cs +++ b/src/runtime/runtime_state.cs @@ -10,7 +10,7 @@ class RuntimeState { public static void Save() { - if (!PySys_GetObject("dummy_gc").IsNull) + if (!PySys_GetObject("initial_modules").IsNull) { throw new Exception("Runtime State set already"); } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index c0a835943..c3325c20d 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -49,11 +49,11 @@ internal static void Initialize() pythonBaseTypeProvider = PythonEngine.InteropConfiguration.pythonBaseTypeProviders; } - internal static void RemoveTypes(ShutdownMode shutdownMode) + internal static void RemoveTypes() { foreach (var type in cache.Values) { - if (shutdownMode == ShutdownMode.Extension + if (Runtime.HostedInPython && _slotsHolders.TryGetValue(type, out var holder)) { // If refcount > 1, it needs to reset the managed slot, @@ -540,7 +540,7 @@ internal static SlotsHolder SetupMetaSlots(Type impl, PyType type) Util.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); // XXX: Hard code with mode check. - if (Runtime.ShutdownMode != ShutdownMode.Reload) + if (Runtime.HostedInPython) { slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => { @@ -559,7 +559,7 @@ private static IntPtr AddCustomMetaMethod(string name, PyType type, IntPtr mdef, slotsHolder.KeeapAlive(thunkInfo); // XXX: Hard code with mode check. - if (Runtime.ShutdownMode != ShutdownMode.Reload) + if (Runtime.HostedInPython) { IntPtr mdefAddr = mdef; slotsHolder.AddDealloctor(() => diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index cec380467..4f6a3ea28 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -48,6 +48,8 @@ class TestCase /// public string Name; + public override string ToString() => Name; + /// /// The C# code to run in the first domain. /// @@ -1135,8 +1137,7 @@ import System /// /// The runner's code. Runs the python code /// This is a template for string.Format - /// Arg 0 is the reload mode: ShutdownMode.Reload or other. - /// Arg 1 is the no-arg python function to run, before or after. + /// Arg 0 is the no-arg python function to run, before or after. /// const string CaseRunnerTemplate = @" using System; @@ -1150,14 +1151,14 @@ public static int Main() {{ try {{ - PythonEngine.Initialize(mode:{0}); + PythonEngine.Initialize(); using (Py.GIL()) {{ var temp = AppDomain.CurrentDomain.BaseDirectory; dynamic sys = Py.Import(""sys""); sys.path.append(new PyString(temp)); dynamic test_mod = Py.Import(""domain_test_module.mod""); - test_mod.{1}_reload(); + test_mod.{0}_reload(); }} PythonEngine.Shutdown(); }} @@ -1280,9 +1281,9 @@ static string CreateTestClassAssembly(string code) return CreateAssembly(TestAssemblyName + ".dll", code, exe: false); } - static string CreateCaseRunnerAssembly(string verb, string shutdownMode = "ShutdownMode.Reload") + static string CreateCaseRunnerAssembly(string verb) { - var code = string.Format(CaseRunnerTemplate, shutdownMode, verb); + var code = string.Format(CaseRunnerTemplate, verb); var name = "TestCaseRunner.exe"; return CreateAssembly(name, code, exe: true); diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index f0890c7c3..d04d5a1f6 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -65,7 +65,6 @@ def test_method_return_type_change(): def test_field_type_change(): _run_test("field_type_change") -@pytest.mark.xfail(reason="Events not yet serializable") def test_rename_event(): _run_test('event_rename') From 84a4ae0526474f687fd57d79125baa12c8a00b2d Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 20 Dec 2021 10:05:08 -0800 Subject: [PATCH 2/4] fixed InterfaceObject and MethodObject not being serializable --- src/runtime/interfaceobject.cs | 22 +++++++++++++++++----- src/runtime/methodobject.cs | 13 +++++++++---- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index f71f78236..b7e865b62 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -13,15 +13,18 @@ namespace Python.Runtime [Serializable] internal class InterfaceObject : ClassBase { + [NonSerialized] internal ConstructorInfo? ctor; internal InterfaceObject(Type tp) : base(tp) { - var coclass = (CoClassAttribute)Attribute.GetCustomAttribute(tp, cc_attr); - if (coclass != null) - { - ctor = coclass.CoClass.GetConstructor(Type.EmptyTypes); - } + this.ctor = TryGetCOMConstructor(tp); + } + + static ConstructorInfo? TryGetCOMConstructor(Type tp) + { + var comClass = (CoClassAttribute?)Attribute.GetCustomAttribute(tp, cc_attr); + return comClass?.CoClass.GetConstructor(Type.EmptyTypes); } private static Type cc_attr; @@ -113,5 +116,14 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k return Runtime.PyObject_GenericGetAttr(ob, key); } + + protected override void OnDeserialization(object sender) + { + base.OnDeserialization(sender); + if (this.type.Valid) + { + this.ctor = TryGetCOMConstructor(this.type.Value); + } + } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index ce64a3f7c..afbcaf631 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -25,7 +25,7 @@ internal class MethodObject : ExtensionType internal bool is_static = false; internal PyString? doc; - internal Type type; + internal MaybeType type; public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads = MethodBinder.DefaultAllowThreads) { @@ -157,6 +157,11 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference { var self = (MethodObject)GetManagedObject(ds)!; + if (!self.type.Valid) + { + return Exceptions.RaiseTypeError(self.type.DeletedMessage); + } + // If the method is accessed through its type (rather than via // an instance) we return an 'unbound' MethodBinding that will // cached for future accesses through the type. @@ -178,11 +183,11 @@ public static NewReference tp_descr_get(BorrowedReference ds, BorrowedReference // In which case create a MethodBinding bound to the base class. var obj = GetManagedObject(ob) as CLRObject; if (obj != null - && obj.inst.GetType() != self.type + && obj.inst.GetType() != self.type.Value && obj.inst is IPythonDerivedType - && self.type.IsInstanceOfType(obj.inst)) + && self.type.Value.IsInstanceOfType(obj.inst)) { - var basecls = ClassManager.GetClass(self.type); + var basecls = ClassManager.GetClass(self.type.Value); return new MethodBinding(self, new PyObject(ob), basecls).Alloc(); } From 1b42732d84a4043429bfeaf23923dc553c693833 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 20 Dec 2021 12:27:56 -0800 Subject: [PATCH 3/4] fixed Python derived types crashing on shutdown of Python process clearing GCHandle from an instance of Python derived type would drop the last reference to it, so it was destroyed without being removed from reflectedObjects collection --- src/runtime/extensiontype.cs | 9 +++++---- src/runtime/runtime.cs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 4ed5d8417..d680067c2 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -80,16 +80,17 @@ public unsafe static void tp_dealloc(NewReference lastRef) tp_clear(lastRef.Borrow()); - bool deleted = loadedExtensions.Remove(lastRef.DangerousGetAddress()); - Debug.Assert(deleted); - // we must decref our type: https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc DecrefTypeAndFree(lastRef.Steal()); } public static int tp_clear(BorrowedReference ob) { - TryFreeGCHandle(ob); + if (TryFreeGCHandle(ob)) + { + bool deleted = loadedExtensions.Remove(ob.DangerousGetAddress()); + Debug.Assert(deleted); + } int res = ClassBase.BaseUnmanagedClear(ob); return res; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9a99c5d80..1db86bc49 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -339,6 +339,7 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops) else if (forceBreakLoops) { NullGCHandles(CLRObject.reflectedObjects); + CLRObject.reflectedObjects.Clear(); } } return false; From dfb87dc26946260733f7e1e117a25b7b28e59485 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 20 Dec 2021 15:31:56 -0800 Subject: [PATCH 4/4] attempt to fix crash on shutdown of Python executable --- src/runtime/metatype.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index f4ad5a4b1..c51ce1a22 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -37,6 +37,10 @@ public static PyType Initialize() public static void Release() { + if (Runtime.HostedInPython) + { + _metaSlotsHodler.ResetSlots(); + } PyCLRMetaType.Dispose(); }