From a5bb2941329d3ae7537c9844e0b89e13d14c5ba0 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 28 May 2020 18:42:47 -0700 Subject: [PATCH] enabled decoding instanceless exceptions Added new class clr.interop.PyErr with optional type, value, and traceback attributes. User can register decoders for it, that would let them decode instanceless (and even typeless) Python exceptions. These decoders will be invoked before the regular exception instance decoders. --- src/embed_tests/Codecs.cs | 63 +++++++++++++++++++------------ src/runtime/Python.Runtime.csproj | 3 ++ src/runtime/Util.cs | 24 ++++++++++++ src/runtime/pythonengine.cs | 37 ++++++++++++++---- src/runtime/pythonexception.cs | 49 ++++++++++++++++++++++++ src/runtime/resources/interop.py | 10 +++++ src/runtime/runtime.cs | 10 +++++ 7 files changed, 165 insertions(+), 31 deletions(-) create mode 100644 src/runtime/resources/interop.py diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 6fc5bb59b..b8b1b8c78 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -31,7 +31,6 @@ static void TupleConversionsGeneric() TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; - using (Py.GIL()) using (var scope = Py.CreateScope()) { void Accept(T value) => restored = value; @@ -53,7 +52,6 @@ static void TupleConversionsObject() TupleCodec.Register(); var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); T restored = default; - using (Py.GIL()) using (var scope = Py.CreateScope()) { void Accept(object value) => restored = (T)value; @@ -73,12 +71,9 @@ public void TupleRoundtripObject() static void TupleRoundtripObject() { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) - { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); } [Test] @@ -90,21 +85,12 @@ public void TupleRoundtripGeneric() static void TupleRoundtripGeneric() { var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); - using (Py.GIL()) - { - var pyTuple = TupleCodec.Instance.TryEncode(tuple); - Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); - Assert.AreEqual(expected: tuple, actual: restored); - } + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); } - static PyObject GetPythonIterable() - { - using (Py.GIL()) - { - return PythonEngine.Eval("map(lambda x: x, [1,2,3])"); - } - } + static PyObject GetPythonIterable() => PythonEngine.Eval("map(lambda x: x, [1,2,3])"); [Test] public void ListDecoderTest() @@ -330,7 +316,6 @@ public void ExceptionEncoded() PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); var callMeAction = new Action(CallMe); - using var _ = Py.GIL(); using var scope = Py.CreateScope(); scope.Exec(@" def call(func): @@ -348,7 +333,6 @@ def call(func): public void ExceptionDecoded() { PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); - using var _ = Py.GIL(); using var scope = Py.CreateScope(); var error = Assert.Throws(() => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); @@ -371,6 +355,16 @@ from datetime import datetime scope.Exec("Codecs.AcceptsDateTime(datetime(2021, 1, 22))"); } + [Test] + public void ExceptionDecodedNoInstance() + { + PyObjectConversions.RegisterDecoder(new InstancelessExceptionDecoder()); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() => PythonEngine.Exec( + $"[].__iter__().__next__()")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + public static void AcceptsDateTime(DateTime v) {} [Test] @@ -406,7 +400,8 @@ public ValueErrorWrapper(string message) : base(message) { } class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder { public bool CanDecode(PyObject objectType, Type targetType) - => this.CanEncode(targetType) && objectType.Equals(PythonEngine.Eval("ValueError")); + => this.CanEncode(targetType) + && PythonReferenceComparer.Instance.Equals(objectType, PythonEngine.Eval("ValueError")); public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper) || typeof(ValueErrorWrapper).IsSubclassOf(type); @@ -424,6 +419,26 @@ public PyObject TryEncode(object value) return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython()); } } + + class InstancelessExceptionDecoder : IPyObjectDecoder + { + readonly PyObject PyErr = Py.Import("clr.interop").GetAttr("PyErr"); + + public bool CanDecode(PyObject objectType, Type targetType) + => PythonReferenceComparer.Instance.Equals(PyErr, objectType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj.HasAttr("value")) + { + value = default; + return false; + } + + value = (T)(object)new ValueErrorWrapper(TestExceptionMessage); + return true; + } + } } /// diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 0311dbf9a..79778aabf 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -43,6 +43,9 @@ clr.py + + interop.py + diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index c70f5cf51..193e57520 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -1,4 +1,6 @@ +#nullable enable using System; +using System.IO; using System.Runtime.InteropServices; namespace Python.Runtime @@ -41,5 +43,27 @@ internal static void WriteCLong(IntPtr type, int offset, Int64 flags) /// internal static IntPtr Coalesce(this IntPtr primary, IntPtr fallback) => primary == IntPtr.Zero ? fallback : primary; + + /// + /// Gets substring after last occurrence of + /// + internal static string? AfterLast(this string str, char symbol) + { + if (str is null) + throw new ArgumentNullException(nameof(str)); + + int last = str.LastIndexOf(symbol); + return last >= 0 ? str.Substring(last + 1) : null; + } + + internal static string ReadStringResource(this System.Reflection.Assembly assembly, string resourceName) + { + if (assembly is null) throw new ArgumentNullException(nameof(assembly)); + if (string.IsNullOrEmpty(resourceName)) throw new ArgumentNullException(nameof(resourceName)); + + using var stream = assembly.GetManifestResourceStream(resourceName); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index cd608fe93..8da8ea5f7 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; @@ -229,13 +230,11 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); Assembly assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("clr.py")) - using (var reader = new StreamReader(stream)) - { - // add the contents of clr.py to the module - string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Reference); - } + // add the contents of clr.py to the module + string clr_py = assembly.ReadStringResource("clr.py"); + Exec(clr_py, module_globals, locals.Reference); + + LoadSubmodule(module_globals, "clr.interop", "interop.py"); // add the imported module to the clr module, and copy the API functions // and decorators into the main clr module. @@ -258,6 +257,30 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } } + static BorrowedReference DefineModule(string name) + { + var module = Runtime.PyImport_AddModule(name); + var module_globals = Runtime.PyModule_GetDict(module); + var builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + return module; + } + + static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, string resourceName) + { + string memberName = fullName.AfterLast('.'); + Debug.Assert(memberName != null); + + var module = DefineModule(fullName); + var module_globals = Runtime.PyModule_GetDict(module); + + Assembly assembly = Assembly.GetExecutingAssembly(); + string pyCode = assembly.ReadStringResource(resourceName); + Exec(pyCode, module_globals.DangerousGetAddress(), module_globals.DangerousGetAddress()); + + Runtime.PyDict_SetItemString(targetModuleDict, memberName, module); + } + static void OnDomainUnload(object _, EventArgs __) { Shutdown(); diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index cca7c439f..8ca596cb9 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -37,6 +37,10 @@ public PythonException(PyType type, PyObject? value, PyObject? traceback) /// internal static Exception ThrowLastAsClrException() { + // prevent potential interop errors in this method + // from crashing process with undebuggable StackOverflowException + RuntimeHelpers.EnsureSufficientExecutionStack(); + var exception = FetchCurrentOrNull(out ExceptionDispatchInfo? dispatchInfo) ?? throw new InvalidOperationException("No exception is set"); dispatchInfo?.Throw(); @@ -83,6 +87,24 @@ internal static PythonException FetchCurrentRaw() return null; } + try + { + if (TryDecodePyErr(type, value, traceback) is { } pyErr) + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + return pyErr; + } + } + catch + { + type.Dispose(); + value.Dispose(); + traceback.Dispose(); + throw; + } + Runtime.PyErr_NormalizeException(type: ref type, val: ref value, tb: ref traceback); try @@ -153,6 +175,11 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference return e; } + if (TryDecodePyErr(typeRef, valRef, tbRef) is { } pyErr) + { + return pyErr; + } + if (PyObjectConversions.TryDecode(valRef, typeRef, typeof(Exception), out object decoded) && decoded is Exception decodedException) { @@ -164,6 +191,28 @@ private static Exception FromPyErr(BorrowedReference typeRef, BorrowedReference return new PythonException(type, value, traceback, inner); } + private static Exception? TryDecodePyErr(BorrowedReference typeRef, BorrowedReference valRef, BorrowedReference tbRef) + { + using var type = PyType.FromReference(typeRef); + using var value = PyObject.FromNullableReference(valRef); + using var traceback = PyObject.FromNullableReference(tbRef); + + using var errorDict = new PyDict(); + if (typeRef != null) errorDict["type"] = type; + if (valRef != null) errorDict["value"] = value; + if (tbRef != null) errorDict["traceback"] = traceback; + + using var pyErrType = Runtime.InteropModule.GetAttr("PyErr"); + using var pyErrInfo = pyErrType.Invoke(new PyTuple(), errorDict); + if (PyObjectConversions.TryDecode(pyErrInfo.Reference, pyErrType.Reference, + typeof(Exception), out object decoded) && decoded is Exception decodedPyErrInfo) + { + return decodedPyErrInfo; + } + + return null; + } + private static Exception? FromCause(BorrowedReference cause) { if (cause == null || cause.IsNone()) return null; diff --git a/src/runtime/resources/interop.py b/src/runtime/resources/interop.py new file mode 100644 index 000000000..a47d16c68 --- /dev/null +++ b/src/runtime/resources/interop.py @@ -0,0 +1,10 @@ +_UNSET = object() + +class PyErr: + def __init__(self, type=_UNSET, value=_UNSET, traceback=_UNSET): + if not(type is _UNSET): + self.type = type + if not(value is _UNSET): + self.value = value + if not(traceback is _UNSET): + self.traceback = traceback diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 4114fc4d0..015b6002e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -178,6 +178,8 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd } XDecref(item); AssemblyManager.UpdatePath(); + + clrInterop = GetModuleLazy("clr.interop"); } private static void InitPyMembers() @@ -406,6 +408,11 @@ internal static void Shutdown() Shutdown(mode); } + private static Lazy GetModuleLazy(string moduleName) + => moduleName is null + ? throw new ArgumentNullException(nameof(moduleName)) + : new Lazy(() => PyModule.Import(moduleName), isThreadSafe: false); + internal static ShutdownMode GetDefaultShutdownMode() { string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); @@ -574,6 +581,9 @@ private static void MoveClrInstancesOnwershipToPython() internal static IntPtr PyNone; internal static IntPtr Error; + private static Lazy clrInterop; + internal static PyObject InteropModule => clrInterop.Value; + internal static BorrowedReference CLRMetaType => new BorrowedReference(PyCLRMetaType); public static PyObject None