pythonnet · lostmsu · Dec 25, 2021 · Dec 20, 2021 · Dec 20, 2021 · Dec 20, 2021
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,10 +14,6 @@ jobs:
         os: [windows, ubuntu, macos]
         python: ["3.6", "3.7", "3.8", "3.9", "3.10"]
         platform: [x64]
-        shutdown_mode: [Normal, Soft]
-
-    env:
-      PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }}
 
     steps:
       - name: Set Environment on macOS

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -107,6 +107,8 @@ Instead, `PyIterable` does that.
 
 ### Removed
 
+-   `ShutdownMode` has been removed. The only shutdown mode supported now is an equivalent of `ShutdownMode.Reload`.
+There is no need to specify it.
 -   implicit assembly loading (you have to explicitly `clr.AddReference` before doing import)
 -   messages in `PythonException` no longer start with exception type
 -   `PyScopeManager`, `PyScopeException`, `PyScope` (use `PyModule` instead)

diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs
@@ -34,7 +34,7 @@ static void TupleConversionsGeneric<T, TTuple>()
             using (var scope = Py.CreateScope())
             {
                 void Accept(T value) => restored = value;
-                var accept = new Action<T>(Accept).ToPython();
+                using var accept = new Action<T>(Accept).ToPython();
                 scope.Set(nameof(tuple), tuple);
                 scope.Set(nameof(accept), accept);
                 scope.Exec($"{nameof(accept)}({nameof(tuple)})");
@@ -55,7 +55,7 @@ static void TupleConversionsObject<T, TTuple>()
             using (var scope = Py.CreateScope())
             {
                 void Accept(object value) => restored = (T)value;
-                var accept = new Action<object>(Accept).ToPython();
+                using var accept = new Action<object>(Accept).ToPython();
                 scope.Set(nameof(tuple), tuple);
                 scope.Set(nameof(accept), accept);
                 scope.Exec($"{nameof(accept)}({nameof(tuple)})");
@@ -71,7 +71,7 @@ public void TupleRoundtripObject()
         static void TupleRoundtripObject<T, TTuple>()
         {
             var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object());
-            var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
+            using var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
             Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out object restored));
             Assert.AreEqual(expected: tuple, actual: restored);
         }
@@ -85,7 +85,7 @@ public void TupleRoundtripGeneric()
         static void TupleRoundtripGeneric<T, TTuple>()
         {
             var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
-            var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
+            using var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
             Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out T restored));
             Assert.AreEqual(expected: tuple, actual: restored);
         }
@@ -98,9 +98,9 @@ public void ListDecoderTest()
             var codec = ListDecoder.Instance;
             var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
 
-            var pyList = new PyList(items.ToArray());
+            using var pyList = new PyList(items.ToArray());
 
-            var pyListType = pyList.GetPythonType();
+            using var pyListType = pyList.GetPythonType();
             Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<bool>)));
             Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<int>)));
             Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable)));
@@ -128,8 +128,8 @@ public void ListDecoderTest()
             Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; });
 
             //can't convert python iterable to list (this will require a copy which isn't lossless)
-            var foo = GetPythonIterable();
-            var fooType = foo.GetPythonType();
+            using var foo = GetPythonIterable();
+            using var fooType = foo.GetPythonType();
             Assert.IsFalse(codec.CanDecode(fooType, typeof(IList<int>)));
         }
 
@@ -140,8 +140,8 @@ public void SequenceDecoderTest()
             var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
 
             //SequenceConverter can only convert to any ICollection
-            var pyList = new PyList(items.ToArray());
-            var listType = pyList.GetPythonType();
+            using var pyList = new PyList(items.ToArray());
+            using var listType = pyList.GetPythonType();
             //it can convert a PyList, since PyList satisfies the python sequence protocol
 
             Assert.IsFalse(codec.CanDecode(listType, typeof(bool)));

diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs
@@ -4,8 +4,6 @@
 using Python.Runtime;
 
 namespace Python.EmbeddingTest {
-    using Runtime = Python.Runtime.Runtime;
-
     public class TestCallbacks {
         [OneTimeSetUp]
         public void SetUp() {
@@ -22,11 +20,13 @@ public void TestNoOverloadException() {
             int passed = 0;
             var aFunctionThatCallsIntoPython = new Action<int>(value => passed = value);
             using (Py.GIL()) {
-                dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])");
-                var error =  Assert.Throws<PythonException>(() => callWith42(aFunctionThatCallsIntoPython.ToPython()));
+                using dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])");
+                using var pyFunc = aFunctionThatCallsIntoPython.ToPython();
+                var error =  Assert.Throws<PythonException>(() => callWith42(pyFunc));
                 Assert.AreEqual("TypeError", error.Type.Name);
                 string expectedArgTypes = "(<class 'list'>)";
                 StringAssert.EndsWith(expectedArgTypes, error.Message);
+                error.Traceback.Dispose();
             }
         }
     }

diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs
@@ -67,14 +67,6 @@ public static void DomainReloadAndGC()
             RunAssemblyAndUnload("test2");
             Assert.That(PyRuntime.Py_IsInitialized() != 0,
                 "On soft-shutdown mode, Python runtime should still running");
-
-            if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal)
-            {
-                // The default mode is a normal mode,
-                // it should shutdown the Python VM avoiding influence other tests.
-                PyRuntime.PyGILState_Ensure();
-                PyRuntime.Py_Finalize();
-            }
         }
 
         #region CrossDomainObject
@@ -222,7 +214,7 @@ static void RunAssemblyAndUnload(string domainName)
             // assembly (and Python .NET) to reside
             var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
 
-            theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL);
+            theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL);
             // From now on use the Proxy to call into the new assembly
             theProxy.RunPython();
 
@@ -290,7 +282,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
+                    theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T1>(domain);
                     arg = caller.Execute(arg);
@@ -308,7 +300,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
+                    theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T2>(domain);
                     caller.Execute(arg);
@@ -319,10 +311,8 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                     AppDomain.Unload(domain);
                 }
             }
-            if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal)
-            {
-                Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0);
-            }
+
+            Assert.IsTrue(PyRuntime.Py_IsInitialized() != 0);
         }
     }
 
@@ -368,10 +358,10 @@ public static void RunPython()
 
         private static IntPtr _state;
 
-        public static void InitPython(ShutdownMode mode, string dllName)
+        public static void InitPython(string dllName)
         {
             PyRuntime.PythonDLL = dllName;
-            PythonEngine.Initialize(mode: mode);
+            PythonEngine.Initialize();
             _state = PythonEngine.BeginAllowThreads();
         }
 
@@ -384,15 +374,7 @@ public static void ShutdownPython()
         public static void ShutdownPythonCompletely()
         {
             PythonEngine.EndAllowThreads(_state);
-            // XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`,
-            // if it used a another mode(the default mode) in other tests,
-            // when other tests trying to access these reserved objects, it may cause Domain exception,
-            // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python.
-            var defaultMode = PythonEngine.DefaultShutdownMode;
-            if (defaultMode != ShutdownMode.Reload)
-            {
-                PythonEngine.ShutdownMode = defaultMode;
-            }
+
             PythonEngine.Shutdown();
         }
 

diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs
@@ -151,51 +151,6 @@ public void ShutdownHandlers()
             // Wrong:   (4 * 2) + 1 + 1 + 1 = 11
             Assert.That(shutdown_count, Is.EqualTo(12));
         }
-
-        [Test]
-        public static void TestRunExitFuncs()
-        {
-            if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal)
-            {
-                // If the runtime using the normal mode,
-                // callback registered by atexit will be called after we release the clr information,
-                // thus there's no chance we can check it here.
-                Assert.Ignore("Skip on normal mode");
-            }
-            Runtime.Runtime.Initialize();
-            PyObject atexit;
-            try
-            {
-                atexit = Py.Import("atexit");
-            }
-            catch (PythonException e)
-            {
-                string msg = e.ToString();
-                bool isImportError = e.Is(Exceptions.ImportError);
-                Runtime.Runtime.Shutdown();
-
-                if (isImportError)
-                {
-                    Assert.Ignore("no atexit module");
-                }
-                else
-                {
-                    Assert.Fail(msg);
-                }
-                PythonEngine.InteropConfiguration = InteropConfiguration.MakeDefault();
-                return;
-            }
-            bool called = false;
-            Action callback = () =>
-            {
-                called = true;
-            };
-            atexit.InvokeMethod("register", callback.ToPython()).Dispose();
-            atexit.Dispose();
-            Runtime.Runtime.Shutdown();
-            Assert.True(called);
-            PythonEngine.InteropConfiguration = InteropConfiguration.MakeDefault();
-        }
     }
 
     public class ImportClassShutdownRefcountClass { }

diff --git a/src/runtime/ReflectedClrType.cs b/src/runtime/ReflectedClrType.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Runtime.Serialization;
 
@@ -49,9 +50,9 @@ public static ReflectedClrType GetOrCreate(Type type)
         return pyType;
     }
 
-    internal void Restore(InterDomainContext context)
+    internal void Restore(Dictionary<string, object?> context)
     {
-        var cb = context.Storage.GetValue<ClassBase>("impl");
+        var cb = (ClassBase)context["impl"]!;
 
         Debug.Assert(cb is not null);
 

diff --git a/src/runtime/StateSerialization/ClassManagerState.cs b/src/runtime/StateSerialization/ClassManagerState.cs
@@ -10,6 +10,6 @@ namespace Python.Runtime.StateSerialization;
 [Serializable]
 internal class ClassManagerState
 {
-    public Dictionary<ReflectedClrType, InterDomainContext> Contexts { get; set; }
+    public Dictionary<ReflectedClrType, Dictionary<string, object?>> Contexts { get; set; }
     public Dictionary<MaybeType, ReflectedClrType> Cache { get; set; }
 }
diff --git a/src/runtime/StateSerialization/ICLRObjectStorer.cs b/src/runtime/StateSerialization/ICLRObjectStorer.cs
@@ -4,6 +4,6 @@ namespace Python.Runtime;
 
 public interface ICLRObjectStorer
 {
-    ICollection<CLRMappedItem> Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage);
-    CLRWrapperCollection Restore(RuntimeDataStorage storage);
+    ICollection<CLRMappedItem> Store(CLRWrapperCollection wrappers, Dictionary<string, object?> storage);
+    CLRWrapperCollection Restore(Dictionary<string, object?> storage);
 }
diff --git a/src/runtime/StateSerialization/SharedObjectsState.cs b/src/runtime/StateSerialization/SharedObjectsState.cs
@@ -12,6 +12,6 @@ internal class SharedObjectsState
 {
     public Dictionary<PyObject, CLRObject> InternalStores { get; init; }
     public Dictionary<PyObject, ExtensionType> Extensions { get; init; }
-    public RuntimeDataStorage Wrappers { get; init; }
-    public Dictionary<PyObject, InterDomainContext> Contexts { get; init; }
+    public Dictionary<string, object?> Wrappers { get; init; }
+    public Dictionary<PyObject, Dictionary<string, object?>> Contexts { get; init; }
 }
diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs
@@ -13,7 +13,7 @@ internal static class Util
         internal const string UnstableApiMessage =
             "This API is unstable, and might be changed or removed in the next minor release";
         internal const string MinimalPythonVersionRequired =
-            "Only Python 3.5 or newer is supported";
+            "Only Python 3.6 or newer is supported";
         internal const string InternalUseOnly =
             "This API is for internal use only";
 

diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs
@@ -341,16 +341,17 @@ public static void tp_dealloc(NewReference lastRef)
 
             CallClear(lastRef.Borrow());
 
-            IntPtr addr = lastRef.DangerousGetAddress();
-            bool deleted = CLRObject.reflectedObjects.Remove(addr);
-            Debug.Assert(deleted);
-
             DecrefTypeAndFree(lastRef.Steal());
         }
 
         public static int tp_clear(BorrowedReference ob)
         {
-            TryFreeGCHandle(ob);
+            if (TryFreeGCHandle(ob))
+            {
+                IntPtr addr = ob.DangerousGetAddress();
+                bool deleted = CLRObject.reflectedObjects.Remove(addr);
+                Debug.Assert(deleted);
+            }
 
             int baseClearResult = BaseUnmanagedClear(ob);
             if (baseClearResult != 0)
@@ -390,13 +391,14 @@ internal static unsafe int BaseUnmanagedClear(BorrowedReference ob)
             return clear(ob);
         }
 
-        protected override void OnSave(BorrowedReference ob, InterDomainContext context)
+        protected override Dictionary<string, object?> OnSave(BorrowedReference ob)
         {
-            base.OnSave(ob, context);
-            context.Storage.AddValue("impl", this);
+            var context = base.OnSave(ob) ?? new();
+            context["impl"] = this;
+            return context;
         }
 
-        protected override void OnLoad(BorrowedReference ob, InterDomainContext? context)
+        protected override void OnLoad(BorrowedReference ob, Dictionary<string, object?>? context)
         {
             base.OnLoad(ob, context);
             var gcHandle = GCHandle.Alloc(this);

diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs
@@ -867,17 +867,19 @@ public static void PyFinalize(IPythonDerivedType obj)
 
         internal static void Finalize(IntPtr derived)
         {
-            bool deleted = CLRObject.reflectedObjects.Remove(derived);
-            Debug.Assert(deleted);
-
             var @ref = NewReference.DangerousFromPointer(derived);
 
             ClassBase.tp_clear(@ref.Borrow());
 
+            var type = Runtime.PyObject_TYPE(@ref.Borrow());
+
             // rare case when it's needed
             // matches correspdonging PyObject_GC_UnTrack
             // in ClassDerivedObject.tp_dealloc
             Runtime.PyObject_GC_Del(@ref.Steal());
+
+            // must decref our type
+            Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress()));
         }
 
         internal static FieldInfo? GetPyObjField(Type type) => type.GetField(PyObjName, PyObjFlags);

diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs
@@ -65,12 +65,15 @@ internal static void RemoveClasses()
 
         internal static ClassManagerState SaveRuntimeData()
         {
-            var contexts = new Dictionary<ReflectedClrType, InterDomainContext>();
+            var contexts = new Dictionary<ReflectedClrType, Dictionary<string, object?>>();
             foreach (var cls in cache)
             {
-                var context = contexts[cls.Value] = new InterDomainContext();
                 var cb = (ClassBase)ManagedType.GetManagedObject(cls.Value)!;
-                cb.Save(cls.Value, context);
+                var context = cb.Save(cls.Value);
+                if (context is not null)
+                {
+                    contexts[cls.Value] = context;
+                }
 
                 // Remove all members added in InitBaseClass.
                 // this is done so that if domain reloads and a member of a
@@ -201,7 +204,7 @@ internal static ClassBase CreateClass(Type type)
             return impl;
         }
 
-        internal static void InitClassBase(Type type, ClassBase impl, PyType pyType)
+        internal static void InitClassBase(Type type, ClassBase impl, ReflectedClrType pyType)
         {
             // First, we introspect the managed type and build some class
             // information, including generating the member descriptors