Skip to content

PyObject finalizer #692

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 29 commits into from
Apr 1, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
8083f3b
Finalizer for PyObject
amos402 Jun 23, 2018
af33e74
Avoid test interdependency
amos402 Jun 24, 2018
6d9f897
Add source to .csproj
amos402 Jun 24, 2018
7140fd0
Make sure recover the environment
amos402 Jun 24, 2018
f66697d
Add StackTrace of C# exception
amos402 Jun 24, 2018
799d37e
Clean up the test and interface
amos402 Jun 24, 2018
cb55163
Update CHANGELOG.md
amos402 Jun 24, 2018
bfc0392
Mono doesn't have GC.WaitForFullGCComplete
amos402 Jun 24, 2018
59b614d
Fixed PythonException leak
amos402 Jun 25, 2018
569cd94
Merge branch 'master' into pyobject-finalizer
den-run-ai Jul 5, 2018
f4f5032
Fixed nPython.exe crash on Shutdown
amos402 Jul 10, 2018
0967a12
Merge branch 'master' into pyobject-finalizer
filmor Jul 17, 2018
f071c55
Add error handler
amos402 Aug 4, 2018
cfda491
Merge branch 'master' into pyobject-finalizer
amos402 Aug 4, 2018
f6c6e42
Merge branch 'master' into pyobject-finalizer
filmor Aug 30, 2018
0d96641
Make collect callback without JIT
amos402 Sep 6, 2018
34713f7
Merge branch 'master' into pyobject-finalizer
filmor Oct 16, 2018
b4e30ac
Merge branch 'master' into pyobject-finalizer
filmor Oct 17, 2018
f836ffa
Merge remote-tracking branch 'remotes/upstream/master' into pyobject-…
amos402 Oct 20, 2018
a4bb82d
Add pending marker
amos402 Oct 23, 2018
21da86d
Merge branch 'master' into pyobject-finalizer
amos402 Oct 24, 2018
5254c65
Remove PYTHONMALLOC setting
amos402 Oct 25, 2018
916e85e
Merge remote-tracking branch 'remotes/upstream/master' into pyobject-…
amos402 Nov 22, 2018
eee3683
Fix ref count error
amos402 Nov 22, 2018
cee8e17
Add ref count check for helping discover the bugs of decref too much
amos402 Nov 22, 2018
247e2d9
Fix ref count error
amos402 Nov 25, 2018
90c67ca
typo error
amos402 Nov 26, 2018
6d68d70
Merge branch 'master' into pyobject-finalizer
filmor Mar 6, 2019
4eff81e
Merge branch 'master' into pyobject-finalizer
filmor Mar 29, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Clean up the test and interface
  • Loading branch information
amos402 committed Jun 24, 2018
commit 799d37e51232e814294b292c4414ea6edacba924
55 changes: 30 additions & 25 deletions src/embed_tests/TestFinalizer.cs
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
using NUnit.Framework;
using Python.Runtime;
using System;
using System.Linq;
using System.Threading;

namespace Python.EmbeddingTest
{
public class TestFinalizer
{
private string _PYTHONMALLOC = string.Empty;
private int _oldThreshold;

[SetUp]
public void SetUp()
Expand All @@ -21,13 +23,15 @@ public void SetUp()
_PYTHONMALLOC = string.Empty;
}
Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc");
_oldThreshold = Finalizer.Instance.Threshold;
PythonEngine.Initialize();
Exceptions.Clear();
}

[TearDown]
public void TearDown()
{
Finalizer.Instance.Threshold = _oldThreshold;
PythonEngine.Shutdown();
if (string.IsNullOrEmpty(_PYTHONMALLOC))
{
Expand Down Expand Up @@ -56,41 +60,42 @@ public void CollectBasicObject()
Assert.GreaterOrEqual(e.ObjectCount, 1);
called = true;
};
Finalizer.Instance.CollectOnce += handler;
try
{
FullGCCollect();
PyLong obj = new PyLong(1024);

WeakReference shortWeak = new WeakReference(obj);
WeakReference longWeak = new WeakReference(obj, true);
obj = null;
FullGCCollect();
// The object has been resurrected
// FIXME: Sometimes the shortWeak would get alive
//Assert.IsFalse(shortWeak.IsAlive);
Assert.IsTrue(longWeak.IsAlive);
WeakReference shortWeak;
WeakReference longWeak;
{
MakeAGarbage(out shortWeak, out longWeak);
}
FullGCCollect();
// The object has been resurrected
Assert.IsFalse(shortWeak.IsAlive);
Assert.IsTrue(longWeak.IsAlive);

Assert.IsFalse(called);
{
var garbage = Finalizer.Instance.GetCollectedObjects();
Assert.NotZero(garbage.Count);
// FIXME: If make some query for garbage,
// the above case will failed Assert.IsFalse(shortWeak.IsAlive)
//Assert.IsTrue(garbage.All(T => T.IsAlive));
Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)));
}

Assert.IsFalse(called);
Finalizer.Instance.CollectOnce += handler;
try
{
Finalizer.Instance.CallPendingFinalizers();
Assert.IsTrue(called);

FullGCCollect();
//Assert.IsFalse(garbage.All(T => T.IsAlive));

Assert.IsNull(longWeak.Target);
}
finally
{
Finalizer.Instance.CollectOnce -= handler;
}
Assert.IsTrue(called);
}

private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak)
{
PyLong obj = new PyLong(1024);
shortWeak = new WeakReference(obj);
longWeak = new WeakReference(obj, true);
obj = null;
}

private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale)
Expand All @@ -101,7 +106,7 @@ private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale)
FullGCCollect();
FullGCCollect();
pyCollect.Invoke();
Finalizer.Instance.CallPendingFinalizers();
Finalizer.Instance.Collect();
Finalizer.Instance.Enable = enbale;

// Estimate unmanaged memory size
Expand All @@ -116,7 +121,7 @@ private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale)
pyCollect.Invoke();
if (enbale)
{
Finalizer.Instance.CallPendingFinalizers();
Finalizer.Instance.Collect();
}

FullGCCollect();
Expand Down
29 changes: 19 additions & 10 deletions src/runtime/finalizer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ public class CollectArgs : EventArgs

[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
private delegate int PedingCall(IntPtr arg);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PendingCall?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The calling type for Py_AddPendingCall

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know, there's just a typo in the name ;)

private PedingCall _collectAction;
private readonly PedingCall _collectAction;

private bool _pending = false;
private readonly object _collectingLock = new object();
public int Threshold { get; set; }
Expand All @@ -32,7 +33,7 @@ private Finalizer()
{
Enable = true;
Threshold = 200;
_collectAction = OnCollect;
_collectAction = OnPendingCollect;
}

public void CallPendingFinalizers()
Expand All @@ -44,6 +45,14 @@ public void CallPendingFinalizers()
Runtime.Py_MakePendingCalls();
}

public void Collect()
{
using (var gilState = new Py.GILState())
{
DisposeAll();
}
}

public List<WeakReference> GetCollectedObjects()
{
return _objQueue.Select(T => new WeakReference(T)).ToList();
Expand All @@ -65,7 +74,7 @@ internal void AddFinalizedObject(IDisposable obj)
GC.ReRegisterForFinalize(obj);
if (_objQueue.Count >= Threshold)
{
Collect();
AddPendingCollect();
}
}

Expand All @@ -76,7 +85,7 @@ internal static void Shutdown()
Runtime.PyErr_Clear();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are you calling this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make sure no garbage left and clear the pending calls.
Pending calls wouldn't clear after calling Py_Finalize, if reset the .net environment, the function pointers would point to unknown memory and cause crash on calling them.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are you calling PyErr_Clear?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

emmm...It seems I should expose the exceptions in DisposeAll after each obj.Dispose()

}

private void Collect()
private void AddPendingCollect()
{
lock (_collectingLock)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This lock should probably encompass the whole function.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why it needs to be lock whole function, the flag already set.
I can't image any situations that Py_AddPendingCall will be called in multiple threads on same time.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't follow, either you expect multiple concurrent executions, then the lock needs to surround the whole function, or you don't, then you don't need any locking.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, yes, thank you for you reminder, I neglect that there is another store at line 114.

{
Expand All @@ -94,19 +103,19 @@ private void Collect()
}
}

private int OnCollect(IntPtr arg)
private int OnPendingCollect(IntPtr arg)
{
CollectOnce?.Invoke(this, new CollectArgs()
{
ObjectCount = _objQueue.Count
});
DisposeAll();
Collect();
_pending = false;
return 0;
}

private void DisposeAll()
{
CollectOnce?.Invoke(this, new CollectArgs()
{
ObjectCount = _objQueue.Count
});
IDisposable obj;
while (_objQueue.TryDequeue(out obj))
{
Expand Down