Skip to content

Commit 410ac15

Browse files
author
Benoit Hudson
committed
UNI-63112: unit test for the domain reload crash
When this test is not ignored, we get a crash. The bug fixes are coming soon (there's three bugs).
1 parent b6417ca commit 410ac15

File tree

2 files changed

+226
-1
lines changed

2 files changed

+226
-1
lines changed

‎src/embed_tests/Python.EmbeddingTest.csproj

+2-1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
<Compile Include="pyrunstring.cs" />
8787
<Compile Include="TestConverter.cs" />
8888
<Compile Include="TestCustomMarshal.cs" />
89+
<Compile Include="TestDomainReload.cs" />
8990
<Compile Include="TestExample.cs" />
9091
<Compile Include="TestPyAnsiString.cs" />
9192
<Compile Include="TestPyFloat.cs" />
@@ -122,4 +123,4 @@
122123
<Copy SourceFiles="$(TargetAssembly)" DestinationFolder="$(PythonBuildDir)" />
123124
<!--Copy SourceFiles="$(TargetAssemblyPdb)" Condition="Exists('$(TargetAssemblyPdb)')" DestinationFolder="$(PythonBuildDir)" /-->
124125
</Target>
125-
</Project>
126+
</Project>

‎src/embed_tests/TestDomainReload.cs

+224
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
using System;
2+
using System.CodeDom.Compiler;
3+
using System.Reflection;
4+
using NUnit.Framework;
5+
6+
namespace Python.EmbeddingTest
7+
{
8+
class TestDomainReload
9+
{
10+
/// <summary>
11+
/// Test that the python runtime can survive a C# domain reload without crashing.
12+
///
13+
/// At the time this test was written, there was a very annoying
14+
/// seemingly random crash bug when integrating pythonnet into Unity.
15+
///
16+
/// The repro steps we that David Lassonde, Viktoria Kovecses and
17+
/// Benoit Hudson eventually worked out:
18+
/// 1. Write a HelloWorld.cs script that uses Python.Runtime to access
19+
/// some C# data from python: C# calls python, which calls C#.
20+
/// 2. Execute the script (e.g. make it a MenuItem and click it).
21+
/// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts.
22+
/// 4. Wait several seconds for Unity to be done recompiling and
23+
/// reloading the C# domain.
24+
/// 5. Make python run the gc (e.g. by calling gc.collect()).
25+
///
26+
/// The reason:
27+
/// A. In step 2, Python.Runtime registers a bunch of new types with
28+
/// their tp_traverse slot pointing to managed code, and allocates
29+
/// some objects of those types.
30+
/// B. In step 4, Unity unloads the C# domain. That frees the managed
31+
/// code. But at the time of the crash investigation, pythonnet
32+
/// leaked the python side of the objects allocated in step 1.
33+
/// C. In step 5, python sees some pythonnet objects in its gc list of
34+
/// potentially-leaked objects. It calls tp_traverse on those objects.
35+
/// But tp_traverse was freed in step 3 => CRASH.
36+
///
37+
/// This test distills what's going on without needing Unity around (we'd see
38+
/// similar behaviour if we were using pythonnet on a .NET web server that did
39+
/// a hot reload).
40+
/// </summary>
41+
[Test]
42+
[Ignore("Test crashes")]
43+
public static void DomainReloadAndGC()
44+
{
45+
// We're set up to run in the directory that includes the bin directory.
46+
System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory);
47+
48+
Assembly pythonRunner1 = BuildAssembly("test1");
49+
RunAssemblyAndUnload(pythonRunner1, "test1");
50+
51+
// This caused a crash because objects allocated in pythonRunner1
52+
// still existed in memory, but the code to do python GC on those
53+
// objects is gone.
54+
Assembly pythonRunner2 = BuildAssembly("test2");
55+
RunAssemblyAndUnload(pythonRunner2, "test2");
56+
}
57+
58+
//
59+
// The code we'll test. All that really matters is
60+
// using GIL { Python.Exec(pyScript); }
61+
// but the rest is useful for debugging.
62+
//
63+
// What matters in the python code is gc.collect and clr.AddReference.
64+
//
65+
const string TestCode = @"
66+
using Python.Runtime;
67+
using System;
68+
class PythonRunner {
69+
public static void RunPython() {
70+
AppDomain.CurrentDomain.DomainUnload += OnDomainUnload;
71+
string name = AppDomain.CurrentDomain.FriendlyName;
72+
Console.WriteLine($""[{name} in .NET] In PythonRunner.RunPython"");
73+
using (Py.GIL()) {
74+
try {
75+
var pyScript = ""import clr\n""
76+
+ $""print('[{name} in python] imported clr')\n""
77+
+ ""clr.AddReference('System')\n""
78+
+ $""print('[{name} in python] allocated a clr object')\n""
79+
+ ""import gc\n""
80+
+ ""gc.collect()\n""
81+
+ $""print('[{name} in python] collected garbage')\n"";
82+
PythonEngine.Exec(pyScript);
83+
} catch(Exception e) {
84+
Console.WriteLine($""[{name} in .NET] Caught exception: {e}"");
85+
}
86+
}
87+
}
88+
static void OnDomainUnload(object sender, EventArgs e) {
89+
System.Console.WriteLine(string.Format($""[{AppDomain.CurrentDomain.FriendlyName} in .NET] unloading""));
90+
}
91+
}";
92+
93+
94+
/// <summary>
95+
/// Build an assembly out of the source code above.
96+
///
97+
/// This creates a file <paramref name="assemblyName"/>.dll in order
98+
/// to support the statement "proxy.theAssembly = assembly" below.
99+
/// That statement needs a file, can't run via memory.
100+
/// </summary>
101+
static Assembly BuildAssembly(string assemblyName)
102+
{
103+
var provider = CodeDomProvider.CreateProvider("CSharp");
104+
105+
var compilerparams = new CompilerParameters();
106+
compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll");
107+
compilerparams.GenerateExecutable = false;
108+
compilerparams.GenerateInMemory = false;
109+
compilerparams.IncludeDebugInformation = false;
110+
compilerparams.OutputAssembly = assemblyName;
111+
112+
var results = provider.CompileAssemblyFromSource(compilerparams, TestCode);
113+
if (results.Errors.HasErrors)
114+
{
115+
var errors = new System.Text.StringBuilder("Compiler Errors:\n");
116+
foreach (CompilerError error in results.Errors)
117+
{
118+
errors.AppendFormat("Line {0},{1}\t: {2}\n",
119+
error.Line, error.Column, error.ErrorText);
120+
}
121+
throw new Exception(errors.ToString());
122+
}
123+
else
124+
{
125+
return results.CompiledAssembly;
126+
}
127+
}
128+
129+
/// <summary>
130+
/// This is a magic incantation required to run code in an application
131+
/// domain other than the current one.
132+
/// </summary>
133+
class Proxy : MarshalByRefObject
134+
{
135+
Assembly theAssembly = null;
136+
137+
public void InitAssembly(string assemblyPath)
138+
{
139+
theAssembly = Assembly.LoadFile(assemblyPath);
140+
}
141+
142+
public void RunPython()
143+
{
144+
Console.WriteLine("[Proxy] Entering RunPython");
145+
146+
// Call into the new assembly. Will execute Python code
147+
var pythonrunner = theAssembly.GetType("PythonRunner");
148+
var runPythonMethod = pythonrunner.GetMethod("RunPython");
149+
runPythonMethod.Invoke(null, new object[] { });
150+
151+
Console.WriteLine("[Proxy] Leaving RunPython");
152+
}
153+
}
154+
155+
/// <summary>
156+
/// Create a domain, run the assembly in it (the RunPython function),
157+
/// and unload the domain.
158+
/// </summary>
159+
static void RunAssemblyAndUnload(Assembly assembly, string assemblyName)
160+
{
161+
Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}");
162+
163+
// Create the domain. Make sure to set PrivateBinPath to a relative
164+
// path from the CWD (namely, 'bin').
165+
// See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain
166+
var currentDomain = AppDomain.CurrentDomain;
167+
var domainsetup = new AppDomainSetup()
168+
{
169+
ApplicationBase = currentDomain.SetupInformation.ApplicationBase,
170+
ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile,
171+
LoaderOptimization = LoaderOptimization.SingleDomain,
172+
PrivateBinPath = "."
173+
};
174+
var domain = AppDomain.CreateDomain(
175+
$"My Domain {assemblyName}",
176+
currentDomain.Evidence,
177+
domainsetup);
178+
179+
// Create a Proxy object in the new domain, where we want the
180+
// assembly (and Python .NET) to reside
181+
Type type = typeof(Proxy);
182+
System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory);
183+
var theProxy = (Proxy)domain.CreateInstanceAndUnwrap(
184+
type.Assembly.FullName,
185+
type.FullName);
186+
187+
// From now on use the Proxy to call into the new assembly
188+
theProxy.InitAssembly(assemblyName);
189+
theProxy.RunPython();
190+
191+
Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}");
192+
AppDomain.Unload(domain);
193+
Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}");
194+
195+
// Validate that the assembly does not exist anymore
196+
try
197+
{
198+
Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior");
199+
}
200+
catch (Exception)
201+
{
202+
Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete.");
203+
}
204+
}
205+
206+
/// <summary>
207+
/// Resolves the assembly. Why doesn't this just work normally?
208+
/// </summary>
209+
static Assembly ResolveAssembly(object sender, ResolveEventArgs args)
210+
{
211+
var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies();
212+
213+
foreach (var assembly in loadedAssemblies)
214+
{
215+
if (assembly.FullName == args.Name)
216+
{
217+
return assembly;
218+
}
219+
}
220+
221+
return null;
222+
}
223+
}
224+
}

0 commit comments

Comments
 (0)