diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 6fc5bb59b..b8b1b8c78 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -31,7 +31,6 @@ static void TupleConversionsGeneric() TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; - using (Py.GIL()) using (var scope = Py.CreateScope()) { void Accept(T value) => restored = value; @@ -53,7 +52,6 @@ static void TupleConversionsObject() TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; - using (Py.GIL()) using (var scope = Py.CreateScope()) { void Accept(object value) => restored = (T)value; @@ -73,12 +71,9 @@ public void TupleRoundtripObject() static void TupleRoundtripObject() { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) - { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); } [Test] @@ -90,21 +85,12 @@ public void TupleRoundtripGeneric() static void TupleRoundtripGeneric() { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) - { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); } - static PyObject GetPythonIterable() - { - using (Py.GIL()) - { - return PythonEngine.Eval("map(lambda x: x, [1,2,3])"); - } - } + static PyObject GetPythonIterable() => PythonEngine.Eval("map(lambda x: x, [1,2,3])"); [Test] public void ListDecoderTest() @@ -330,7 +316,6 @@ public void ExceptionEncoded() PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); var callMeAction = new Action(CallMe); - using var _ = Py.GIL(); using var scope = Py.CreateScope(); scope.Exec(@" def call(func): @@ -348,7 +333,6 @@ def call(func): public void ExceptionDecoded() { PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); - using var _ = Py.GIL(); using var scope = Py.CreateScope(); var error = Assert.Throws(() => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); @@ -371,6 +355,16 @@ from datetime import datetime scope.Exec("Codecs.AcceptsDateTime(datetime(2021, 1, 22))"); } + [Test] + public void ExceptionDecodedNoInstance() + { + PyObjectConversions.RegisterDecoder(new InstancelessExceptionDecoder()); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() => PythonEngine.Exec( + $"[].__iter__().__next__()")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + public static void AcceptsDateTime(DateTime v) {} [Test] @@ -406,7 +400,8 @@ public ValueErrorWrapper(string message) : base(message) { } class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder { public bool CanDecode(PyObject objectType, Type targetType) - => this.CanEncode(targetType) && objectType.Equals(PythonEngine.Eval("ValueError")); + => this.CanEncode(targetType) + && PythonReferenceComparer.Instance.Equals(objectType, PythonEngine.Eval("ValueError")); public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper) || typeof(ValueErrorWrapper).IsSubclassOf(type); @@ -424,6 +419,26 @@ public PyObject TryEncode(object value) return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython()); } } + + class InstancelessExceptionDecoder : IPyObjectDecoder + { + readonly PyObject PyErr = Py.Import("clr.interop").GetAttr("PyErr"); + + public bool CanDecode(PyObject objectType, Type targetType) + => PythonReferenceComparer.Instance.Equals(PyErr, objectType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj.HasAttr("value")) + { + value = default; + return false; + } + + value = (T)(object)new ValueErrorWrapper(TestExceptionMessage); + return true; + } + } } /// diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 0311dbf9a..79778aabf 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -43,6 +43,9 @@ clr.py + + interop.py + diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index c70f5cf51..193e57520 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -1,4 +1,6 @@ +#nullable enable using System; +using System.IO; using System.Runtime.InteropServices; namespace Python.Runtime @@ -41,5 +43,27 @@ internal static void WriteCLong(IntPtr type, int offset, Int64 flags) /// internal static IntPtr Coalesce(this IntPtr primary, IntPtr fallback) => primary == IntPtr.Zero ? fallback : primary; + + /// + /// Gets substring after last occurrence of + /// + internal static string? AfterLast(this string str, char symbol) + { + if (str is null) + throw new ArgumentNullException(nameof(str)); + + int last = str.LastIndexOf(symbol); + return last >= 0 ? str.Substring(last + 1) : null; + } + + internal static string ReadStringResource(this System.Reflection.Assembly assembly, string resourceName) + { + if (assembly is null) throw new ArgumentNullException(nameof(assembly)); + if (string.IsNullOrEmpty(resourceName)) throw new ArgumentNullException(nameof(resourceName)); + + using var stream = assembly.GetManifestResourceStream(resourceName); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index cd608fe93..8da8ea5f7 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; @@ -229,13 +230,11 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); Assembly assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("clr.py")) - using (var reader = new StreamReader(stream)) - { - // add the contents of clr.py to the module - string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Reference); - } + // add the contents of clr.py to the module + string clr_py = assembly.ReadStringResource("clr.py"); + Exec(clr_py, module_globals, locals.Reference); + + LoadSubmodule(module_globals, "clr.interop", "interop.py"); // add the imported module to the clr module, and copy the API functions // and decorators into the main clr module. @@ -258,6 +257,30 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } } + static BorrowedReference DefineModule(string name) + { + var module = Runtime.PyImport_AddModule(name); + var module_globals = Runtime.PyModule_GetDict(module); + var builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + return module; + } + + static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, string resourceName) + { + string memberName = fullName.AfterLast('.'); + Debug.Assert(memberName != null); + + var module = DefineModule(fullName); + var module_globals = Runtime.PyModule_GetDict(module); + + Assembly assembly = Assembly.GetExecutingAssembly(); + string pyCode = assembly.ReadStringResource(resourceName); + Exec(pyCode, module_globals.DangerousGetAddress(), module_globals.DangerousGetAddress()); + + Runtime.PyDict_SetItemString(targetModuleDict, memberName, module); + } + static void OnDomainUnload(object _, EventArgs __) { Shutdown(); diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index cca7c439f..8ca596cb9 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -37,6 +37,10 @@ public PythonException(PyType type, PyObject? value, PyObject? traceback) /// internal static Exception ThrowLastAsClrException() { + // prevent potential interop errors in this method + // from crashing process with undebuggable StackOverflowException + RuntimeHelpers.EnsureSufficientExecutionStack(); + var exception = FetchCurrentOrNull(out ExceptionDispatchInfo? dispatchInfo) ?? throw new InvalidOperationException("No exception is set"); dispatchInfo?.Throw(); @@ -83,6 +87,24 @@ internal static PythonException FetchCurrentRaw() return null; } + try + { + if (TryDecodePyErr(type, value, traceback) is { } pyErr) + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + return pyErr; + } + } + catch + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + throw; + } + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref traceback); try @@ -153,6 +175,11 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference return e; } + if (TryDecodePyErr(typeRef, valRef, tbRef) is { } pyErr) + { + return pyErr; + } + if (PyObjectConversions.TryDecode(valRef, typeRef, typeof(Exception), out object decoded) && decoded is Exception decodedException) { @@ -164,6 +191,28 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference return new PythonException(type, value, traceback, inner); } + private static Exception? TryDecodePyErr(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef) + { + using var type = PyType.FromReference(typeRef); + using var value = PyObject.FromNullableReference(valRef); + using var traceback = PyObject.FromNullableReference(tbRef); + + using var errorDict = new PyDict(); + if (typeRef != null) errorDict["type"] = type; + if (valRef != null) errorDict["value"] = value; + if (tbRef != null) errorDict["traceback"] = traceback; + + using var pyErrType = Runtime.InteropModule.GetAttr("PyErr"); + using var pyErrInfo = pyErrType.Invoke(new PyTuple(), errorDict); + if (PyObjectConversions.TryDecode(pyErrInfo.Reference, pyErrType.Reference, + typeof(Exception), out object decoded) && decoded is Exception decodedPyErrInfo) + { + return decodedPyErrInfo; + } + + return null; + } + private static Exception? FromCause(BorrowedReference cause) { if (cause == null || cause.IsNone()) return null; diff --git a/src/runtime/resources/interop.py b/src/runtime/resources/interop.py new file mode 100644 index 000000000..a47d16c68 --- /dev/null +++ b/src/runtime/resources/interop.py @@ -0,0 +1,10 @@ +_UNSET = object() + +class PyErr: + def __init__(self, type=_UNSET, value=_UNSET, traceback=_UNSET): + if not(type is _UNSET): + self.type = type + if not(value is _UNSET): + self.value = value + if not(traceback is _UNSET): + self.traceback = traceback diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 4114fc4d0..015b6002e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -178,6 +178,8 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } XDecref(item); AssemblyManager.UpdatePath(); + + clrInterop = GetModuleLazy("clr.interop"); } private static void InitPyMembers() @@ -406,6 +408,11 @@ internal static void Shutdown() Shutdown(mode); } + private static Lazy GetModuleLazy(string moduleName) + => moduleName is null + ? throw new ArgumentNullException(nameof(moduleName)) + : new Lazy(() => PyModule.Import(moduleName), isThreadSafe: false); + internal static ShutdownMode GetDefaultShutdownMode() { string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); @@ -574,6 +581,9 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyNone; internal static IntPtr Error; + private static Lazy clrInterop; + internal static PyObject InteropModule => clrInterop.Value; + internal static BorrowedReference CLRMetaType => new BorrowedReference(PyCLRMetaType); public static PyObject None