Skip to content

Commit 53836de

Browse files
lostmsulostmsu
committed
ShutdownMode has been removed. The only shutdown mode supported now is an equivalent of ShutdownMode.Reload
also in this change: - fixed Python derived types not being decrefed when an instance is deallocated - reduced time and amount of storage needed for runtime reload - removed circular reference loop between Type <-> ConstructorBinding(s) + exposed Runtime.TryCollectingGarbage
1 parent dfeb354 commit 53836de

30 files changed

+174
-357
lines changed

‎.github/workflows/main.yml

-4
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,6 @@ jobs:
1414
os: [windows, ubuntu, macos]
1515
python: ["3.6", "3.7", "3.8", "3.9", "3.10"]
1616
platform: [x64]
17-
shutdown_mode: [Normal, Soft]
18-
19-
env:
20-
PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }}
2117

2218
steps:
2319
- name: Set Environment on macOS

‎CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ Instead, `PyIterable` does that.
107107

108108
### Removed
109109

110+
- `ShutdownMode` has been removed. The only shutdown mode supported now is an equivalent of `ShutdownMode.Reload`.
111+
There is no need to specify it.
110112
- implicit assembly loading (you have to explicitly `clr.AddReference` before doing import)
111113
- messages in `PythonException` no longer start with exception type
112114
- `PyScopeManager`, `PyScopeException`, `PyScope` (use `PyModule` instead)

‎src/embed_tests/Codecs.cs

+10-10
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ static void TupleConversionsGeneric<T, TTuple>()
3434
using (var scope = Py.CreateScope())
3535
{
3636
void Accept(T value) => restored = value;
37-
var accept = new Action<T>(Accept).ToPython();
37+
using var accept = new Action<T>(Accept).ToPython();
3838
scope.Set(nameof(tuple), tuple);
3939
scope.Set(nameof(accept), accept);
4040
scope.Exec($"{nameof(accept)}({nameof(tuple)})");
@@ -55,7 +55,7 @@ static void TupleConversionsObject<T, TTuple>()
5555
using (var scope = Py.CreateScope())
5656
{
5757
void Accept(object value) => restored = (T)value;
58-
var accept = new Action<object>(Accept).ToPython();
58+
using var accept = new Action<object>(Accept).ToPython();
5959
scope.Set(nameof(tuple), tuple);
6060
scope.Set(nameof(accept), accept);
6161
scope.Exec($"{nameof(accept)}({nameof(tuple)})");
@@ -71,7 +71,7 @@ public void TupleRoundtripObject()
7171
static void TupleRoundtripObject<T, TTuple>()
7272
{
7373
var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object());
74-
var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
74+
using var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
7575
Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out object restored));
7676
Assert.AreEqual(expected: tuple, actual: restored);
7777
}
@@ -85,7 +85,7 @@ public void TupleRoundtripGeneric()
8585
static void TupleRoundtripGeneric<T, TTuple>()
8686
{
8787
var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object());
88-
var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
88+
using var pyTuple = TupleCodec<TTuple>.Instance.TryEncode(tuple);
8989
Assert.IsTrue(TupleCodec<TTuple>.Instance.TryDecode(pyTuple, out T restored));
9090
Assert.AreEqual(expected: tuple, actual: restored);
9191
}
@@ -98,9 +98,9 @@ public void ListDecoderTest()
9898
var codec = ListDecoder.Instance;
9999
var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
100100

101-
var pyList = new PyList(items.ToArray());
101+
using var pyList = new PyList(items.ToArray());
102102

103-
var pyListType = pyList.GetPythonType();
103+
using var pyListType = pyList.GetPythonType();
104104
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<bool>)));
105105
Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList<int>)));
106106
Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable)));
@@ -128,8 +128,8 @@ public void ListDecoderTest()
128128
Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; });
129129

130130
//can't convert python iterable to list (this will require a copy which isn't lossless)
131-
var foo = GetPythonIterable();
132-
var fooType = foo.GetPythonType();
131+
using var foo = GetPythonIterable();
132+
using var fooType = foo.GetPythonType();
133133
Assert.IsFalse(codec.CanDecode(fooType, typeof(IList<int>)));
134134
}
135135

@@ -140,8 +140,8 @@ public void SequenceDecoderTest()
140140
var items = new List<PyObject>() { new PyInt(1), new PyInt(2), new PyInt(3) };
141141

142142
//SequenceConverter can only convert to any ICollection
143-
var pyList = new PyList(items.ToArray());
144-
var listType = pyList.GetPythonType();
143+
using var pyList = new PyList(items.ToArray());
144+
using var listType = pyList.GetPythonType();
145145
//it can convert a PyList, since PyList satisfies the python sequence protocol
146146

147147
Assert.IsFalse(codec.CanDecode(listType, typeof(bool)));

‎src/embed_tests/TestCallbacks.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44
using Python.Runtime;
55

66
namespace Python.EmbeddingTest {
7-
using Runtime = Python.Runtime.Runtime;
8-
97
public class TestCallbacks {
108
[OneTimeSetUp]
119
public void SetUp() {
@@ -22,11 +20,13 @@ public void TestNoOverloadException() {
2220
int passed = 0;
2321
var aFunctionThatCallsIntoPython = new Action<int>(value => passed = value);
2422
using (Py.GIL()) {
25-
dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])");
26-
var error = Assert.Throws<PythonException>(() => callWith42(aFunctionThatCallsIntoPython.ToPython()));
23+
using dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])");
24+
using var pyFunc = aFunctionThatCallsIntoPython.ToPython();
25+
var error = Assert.Throws<PythonException>(() => callWith42(pyFunc));
2726
Assert.AreEqual("TypeError", error.Type.Name);
2827
string expectedArgTypes = "(<class 'list'>)";
2928
StringAssert.EndsWith(expectedArgTypes, error.Message);
29+
error.Traceback.Dispose();
3030
}
3131
}
3232
}

‎src/embed_tests/TestDomainReload.cs

+8-26
Original file line numberDiff line numberDiff line change
@@ -67,14 +67,6 @@ public static void DomainReloadAndGC()
6767
RunAssemblyAndUnload("test2");
6868
Assert.That(PyRuntime.Py_IsInitialized() != 0,
6969
"On soft-shutdown mode, Python runtime should still running");
70-
71-
if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal)
72-
{
73-
// The default mode is a normal mode,
74-
// it should shutdown the Python VM avoiding influence other tests.
75-
PyRuntime.PyGILState_Ensure();
76-
PyRuntime.Py_Finalize();
77-
}
7870
}
7971

8072
#region CrossDomainObject
@@ -222,7 +214,7 @@ static void RunAssemblyAndUnload(string domainName)
222214
// assembly (and Python .NET) to reside
223215
var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
224216

225-
theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL);
217+
theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL);
226218
// From now on use the Proxy to call into the new assembly
227219
theProxy.RunPython();
228220

@@ -290,7 +282,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
290282
try
291283
{
292284
var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
293-
theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
285+
theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL);
294286

295287
var caller = CreateInstanceInstanceAndUnwrap<T1>(domain);
296288
arg = caller.Execute(arg);
@@ -308,7 +300,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
308300
try
309301
{
310302
var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
311-
theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
303+
theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL);
312304

313305
var caller = CreateInstanceInstanceAndUnwrap<T2>(domain);
314306
caller.Execute(arg);
@@ -319,10 +311,8 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
319311
AppDomain.Unload(domain);
320312
}
321313
}
322-
if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal)
323-
{
324-
Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0);
325-
}
314+
315+
Assert.IsTrue(PyRuntime.Py_IsInitialized() != 0);
326316
}
327317
}
328318

@@ -368,10 +358,10 @@ public static void RunPython()
368358

369359
private static IntPtr _state;
370360

371-
public static void InitPython(ShutdownMode mode, string dllName)
361+
public static void InitPython(string dllName)
372362
{
373363
PyRuntime.PythonDLL = dllName;
374-
PythonEngine.Initialize(mode: mode);
364+
PythonEngine.Initialize();
375365
_state = PythonEngine.BeginAllowThreads();
376366
}
377367

@@ -384,15 +374,7 @@ public static void ShutdownPython()
384374
public static void ShutdownPythonCompletely()
385375
{
386376
PythonEngine.EndAllowThreads(_state);
387-
// XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`,
388-
// if it used a another mode(the default mode) in other tests,
389-
// when other tests trying to access these reserved objects, it may cause Domain exception,
390-
// thus it needs to reduct to Soft mode to make sure all clr objects remove from Python.
391-
var defaultMode = PythonEngine.DefaultShutdownMode;
392-
if (defaultMode != ShutdownMode.Reload)
393-
{
394-
PythonEngine.ShutdownMode = defaultMode;
395-
}
377+
396378
PythonEngine.Shutdown();
397379
}
398380

‎src/embed_tests/pyinitialize.cs

-45
Original file line numberDiff line numberDiff line change
@@ -151,51 +151,6 @@ public void ShutdownHandlers()
151151
// Wrong: (4 * 2) + 1 + 1 + 1 = 11
152152
Assert.That(shutdown_count, Is.EqualTo(12));
153153
}
154-
155-
[Test]
156-
public static void TestRunExitFuncs()
157-
{
158-
if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal)
159-
{
160-
// If the runtime using the normal mode,
161-
// callback registered by atexit will be called after we release the clr information,
162-
// thus there's no chance we can check it here.
163-
Assert.Ignore("Skip on normal mode");
164-
}
165-
Runtime.Runtime.Initialize();
166-
PyObject atexit;
167-
try
168-
{
169-
atexit = Py.Import("atexit");
170-
}
171-
catch (PythonException e)
172-
{
173-
string msg = e.ToString();
174-
bool isImportError = e.Is(Exceptions.ImportError);
175-
Runtime.Runtime.Shutdown();
176-
177-
if (isImportError)
178-
{
179-
Assert.Ignore("no atexit module");
180-
}
181-
else
182-
{
183-
Assert.Fail(msg);
184-
}
185-
PythonEngine.InteropConfiguration = InteropConfiguration.MakeDefault();
186-
return;
187-
}
188-
bool called = false;
189-
Action callback = () =>
190-
{
191-
called = true;
192-
};
193-
atexit.InvokeMethod("register", callback.ToPython()).Dispose();
194-
atexit.Dispose();
195-
Runtime.Runtime.Shutdown();
196-
Assert.True(called);
197-
PythonEngine.InteropConfiguration = InteropConfiguration.MakeDefault();
198-
}
199154
}
200155

201156
public class ImportClassShutdownRefcountClass { }

‎src/runtime/ReflectedClrType.cs

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections.Generic;
23
using System.Diagnostics;
34
using System.Runtime.Serialization;
45

@@ -49,9 +50,9 @@ public static ReflectedClrType GetOrCreate(Type type)
4950
return pyType;
5051
}
5152

52-
internal void Restore(InterDomainContext context)
53+
internal void Restore(Dictionary<string, object?> context)
5354
{
54-
var cb = context.Storage.GetValue<ClassBase>("impl");
55+
var cb = (ClassBase)context["impl"]!;
5556

5657
Debug.Assert(cb is not null);
5758

‎src/runtime/StateSerialization/ClassManagerState.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,6 @@ namespace Python.Runtime.StateSerialization;
1010
[Serializable]
1111
internal class ClassManagerState
1212
{
13-
public Dictionary<ReflectedClrType, InterDomainContext> Contexts { get; set; }
13+
public Dictionary<ReflectedClrType, Dictionary<string, object?>> Contexts { get; set; }
1414
public Dictionary<MaybeType, ReflectedClrType> Cache { get; set; }
1515
}

‎src/runtime/StateSerialization/ICLRObjectStorer.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ namespace Python.Runtime;
44

55
public interface ICLRObjectStorer
66
{
7-
ICollection<CLRMappedItem> Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage);
8-
CLRWrapperCollection Restore(RuntimeDataStorage storage);
7+
ICollection<CLRMappedItem> Store(CLRWrapperCollection wrappers, Dictionary<string, object?> storage);
8+
CLRWrapperCollection Restore(Dictionary<string, object?> storage);
99
}

‎src/runtime/StateSerialization/SharedObjectsState.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,6 @@ internal class SharedObjectsState
1212
{
1313
public Dictionary<PyObject, CLRObject> InternalStores { get; init; }
1414
public Dictionary<PyObject, ExtensionType> Extensions { get; init; }
15-
public RuntimeDataStorage Wrappers { get; init; }
16-
public Dictionary<PyObject, InterDomainContext> Contexts { get; init; }
15+
public Dictionary<string, object?> Wrappers { get; init; }
16+
public Dictionary<PyObject, Dictionary<string, object?>> Contexts { get; init; }
1717
}

‎src/runtime/Util.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ internal static class Util
1313
internal const string UnstableApiMessage =
1414
"This API is unstable, and might be changed or removed in the next minor release";
1515
internal const string MinimalPythonVersionRequired =
16-
"Only Python 3.5 or newer is supported";
16+
"Only Python 3.6 or newer is supported";
1717
internal const string InternalUseOnly =
1818
"This API is for internal use only";
1919

‎src/runtime/classbase.cs

+11-9
Original file line numberDiff line numberDiff line change
@@ -341,16 +341,17 @@ public static void tp_dealloc(NewReference lastRef)
341341

342342
CallClear(lastRef.Borrow());
343343

344-
IntPtr addr = lastRef.DangerousGetAddress();
345-
bool deleted = CLRObject.reflectedObjects.Remove(addr);
346-
Debug.Assert(deleted);
347-
348344
DecrefTypeAndFree(lastRef.Steal());
349345
}
350346

351347
public static int tp_clear(BorrowedReference ob)
352348
{
353-
TryFreeGCHandle(ob);
349+
if (TryFreeGCHandle(ob))
350+
{
351+
IntPtr addr = ob.DangerousGetAddress();
352+
bool deleted = CLRObject.reflectedObjects.Remove(addr);
353+
Debug.Assert(deleted);
354+
}
354355

355356
int baseClearResult = BaseUnmanagedClear(ob);
356357
if (baseClearResult != 0)
@@ -390,13 +391,14 @@ internal static unsafe int BaseUnmanagedClear(BorrowedReference ob)
390391
return clear(ob);
391392
}
392393

393-
protected override void OnSave(BorrowedReference ob, InterDomainContext context)
394+
protected override Dictionary<string, object?> OnSave(BorrowedReference ob)
394395
{
395-
base.OnSave(ob, context);
396-
context.Storage.AddValue("impl", this);
396+
var context = base.OnSave(ob) ?? new();
397+
context["impl"] = this;
398+
return context;
397399
}
398400

399-
protected override void OnLoad(BorrowedReference ob, InterDomainContext? context)
401+
protected override void OnLoad(BorrowedReference ob, Dictionary<string, object?>? context)
400402
{
401403
base.OnLoad(ob, context);
402404
var gcHandle = GCHandle.Alloc(this);

‎src/runtime/classderived.cs

+5-3
Original file line numberDiff line numberDiff line change
@@ -867,17 +867,19 @@ public static void PyFinalize(IPythonDerivedType obj)
867867

868868
internal static void Finalize(IntPtr derived)
869869
{
870-
bool deleted = CLRObject.reflectedObjects.Remove(derived);
871-
Debug.Assert(deleted);
872-
873870
var @ref = NewReference.DangerousFromPointer(derived);
874871

875872
ClassBase.tp_clear(@ref.Borrow());
876873

874+
var type = Runtime.PyObject_TYPE(@ref.Borrow());
875+
877876
// rare case when it's needed
878877
// matches correspdonging PyObject_GC_UnTrack
879878
// in ClassDerivedObject.tp_dealloc
880879
Runtime.PyObject_GC_Del(@ref.Steal());
880+
881+
// must decref our type
882+
Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress()));
881883
}
882884

883885
internal static FieldInfo? GetPyObjField(Type type) => type.GetField(PyObjName, PyObjFlags);

‎src/runtime/classmanager.cs

+7-4
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,15 @@ internal static void RemoveClasses()
6565

6666
internal static ClassManagerState SaveRuntimeData()
6767
{
68-
var contexts = new Dictionary<ReflectedClrType, InterDomainContext>();
68+
var contexts = new Dictionary<ReflectedClrType, Dictionary<string, object?>>();
6969
foreach (var cls in cache)
7070
{
71-
var context = contexts[cls.Value] = new InterDomainContext();
7271
var cb = (ClassBase)ManagedType.GetManagedObject(cls.Value)!;
73-
cb.Save(cls.Value, context);
72+
var context = cb.Save(cls.Value);
73+
if (context is not null)
74+
{
75+
contexts[cls.Value] = context;
76+
}
7477

7578
// Remove all members added in InitBaseClass.
7679
// this is done so that if domain reloads and a member of a
@@ -201,7 +204,7 @@ internal static ClassBase CreateClass(Type type)
201204
return impl;
202205
}
203206

204-
internal static void InitClassBase(Type type, ClassBase impl, PyType pyType)
207+
internal static void InitClassBase(Type type, ClassBase impl, ReflectedClrType pyType)
205208
{
206209
// First, we introspect the managed type and build some class
207210
// information, including generating the member descriptors

0 commit comments

Comments
 (0)