From 6f1064185250ae4080b8d7f3fa491b78425380db Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:55:30 -0800 Subject: [PATCH 1/5] PyImport test validate no initialization failures --- src/embed_tests/pyimport.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index 24f31acff..2823c3d19 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -34,7 +34,9 @@ public void SetUp() TestContext.Out.WriteLine(testPath); IntPtr str = Runtime.Runtime.PyString_FromString(testPath); + Assert.IsFalse(str == IntPtr.Zero); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); + Assert.IsFalse(path.IsNull); Runtime.Runtime.PyList_Append(path, str); Runtime.Runtime.XDecref(str); } From 676fbb42cd904f042ed1387c95c54baf090fbc98 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:56:50 -0800 Subject: [PATCH 2/5] PyObject.Length raises an exception when object does not have a concept of length --- CHANGELOG.md | 1 + src/runtime/pyobject.cs | 12 ++++-------- src/runtime/runtime.cs | 12 +++--------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bee653e8..7fbd6f7b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ when .NET expects an integer [#1342][i1342] - BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. - BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. +- BREAKING: `PyObject.Length()` now raises a `PythonException` when object does not support a concept of length. - Sign Runtime DLL with a strong name - Implement loading through `clr_loader` instead of the included `ClrModule`, enables support for .NET Core diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3c7f13ec6..382ed8ccd 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -615,19 +615,15 @@ public virtual void DelItem(int index) /// - /// Length Method - /// - /// /// Returns the length for objects that support the Python sequence - /// protocol, or 0 if the object does not support the protocol. - /// + /// protocol. + /// public virtual long Length() { - var s = Runtime.PyObject_Size(obj); + var s = Runtime.PyObject_Size(Reference); if (s < 0) { - Runtime.PyErr_Clear(); - return 0; + throw new PythonException(); } return s; } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec7f5e446..332222e18 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1115,13 +1115,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static int PyObject_Not(IntPtr pointer) => Delegates.PyObject_Not(pointer); - internal static long PyObject_Size(IntPtr pointer) - { - return (long)_PyObject_Size(pointer); - } - - - private static IntPtr _PyObject_Size(IntPtr pointer) => Delegates._PyObject_Size(pointer); + internal static nint PyObject_Size(BorrowedReference pointer) => Delegates.PyObject_Size(pointer); internal static nint PyObject_Hash(IntPtr op) => Delegates.PyObject_Hash(op); @@ -2317,7 +2311,7 @@ static Delegates() PyCallable_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyCallable_Check), GetUnmanagedDll(_PythonDll)); PyObject_IsTrue = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_IsTrue), GetUnmanagedDll(_PythonDll)); PyObject_Not = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Not), GetUnmanagedDll(_PythonDll)); - _PyObject_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Size", GetUnmanagedDll(_PythonDll)); + PyObject_Size = (delegate* unmanaged[Cdecl])GetFunctionByName("PyObject_Size", GetUnmanagedDll(_PythonDll)); PyObject_Hash = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll)); PyObject_Repr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll)); PyObject_Str = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll)); @@ -2589,7 +2583,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyCallable_Check { get; } internal static delegate* unmanaged[Cdecl] PyObject_IsTrue { get; } internal static delegate* unmanaged[Cdecl] PyObject_Not { get; } - internal static delegate* unmanaged[Cdecl] _PyObject_Size { get; } + internal static delegate* unmanaged[Cdecl] PyObject_Size { get; } internal static delegate* unmanaged[Cdecl] PyObject_Hash { get; } internal static delegate* unmanaged[Cdecl] PyObject_Repr { get; } internal static delegate* unmanaged[Cdecl] PyObject_Str { get; } From b2c15a4dafd6bdc02e79cf80cc48a91c2f0cb39b Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:57:45 -0800 Subject: [PATCH 3/5] ImportHook: verify __import__ was not updated before overwriting it with original value --- src/runtime/importhook.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 066c765fe..0fba8f6a6 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -61,6 +61,13 @@ static void RestoreImport() { IntPtr builtins = Runtime.GetBuiltins(); + IntPtr existing = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); + Runtime.XDecref(existing); + if (existing != hook.ptr) + { + throw new NotSupportedException("Unable to restore original __import__."); + } + int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); PythonException.ThrowIfIsNotZero(res); Runtime.XDecref(py_import); @@ -374,6 +381,8 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) private static bool IsLoadAll(BorrowedReference fromList) { + if (fromList == null) throw new ArgumentNullException(nameof(fromList)); + if (CLRModule.preload) { return false; From fefa322cc49772bc5d85803a9d39fb78ce04a065 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 15:58:06 -0800 Subject: [PATCH 4/5] drop _PyObject_GetDictPtr and use PyObject_GenericGetDict instead --- src/runtime/importhook.cs | 8 +++++--- src/runtime/runtime.cs | 9 ++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 0fba8f6a6..14f48712a 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -95,7 +95,7 @@ internal static unsafe void Initialize() // both dicts are borrowed references BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); Runtime.PyDict_Update(mod_dict, clr_dict); BorrowedReference dict = Runtime.PyImport_GetModuleDict(); @@ -157,8 +157,10 @@ public static unsafe NewReference GetCLRModule(BorrowedReference fromList = defa // update the module dictionary with the contents of the root dictionary root.LoadNames(); BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference); - Runtime.PyDict_Update(py_mod_dict, clr_dict); + using (var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference)) + { + Runtime.PyDict_Update(py_mod_dict, clr_dict); + } // find any items from the from list and get them from the root if they're not // already in the module dictionary diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 332222e18..f6a376145 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2016,9 +2016,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - - internal static BorrowedReference* _PyObject_GetDictPtr(BorrowedReference obj) => Delegates._PyObject_GetDictPtr(obj); - + internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); + internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); @@ -2466,8 +2465,8 @@ static Delegates() PyType_Ready = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_Ready), GetUnmanagedDll(_PythonDll)); _PyType_Lookup = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyType_Lookup), GetUnmanagedDll(_PythonDll)); PyObject_GenericGetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetAttr), GetUnmanagedDll(_PythonDll)); + PyObject_GenericGetDict = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericGetDict), GetUnmanagedDll(PythonDLL)); PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); - _PyObject_GetDictPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GetDictPtr), GetUnmanagedDll(_PythonDll)); PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); @@ -2732,7 +2731,6 @@ static Delegates() internal static delegate* unmanaged[Cdecl] _PyType_Lookup { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetAttr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } - internal static delegate* unmanaged[Cdecl] _PyObject_GetDictPtr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } @@ -2769,6 +2767,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyException_SetCause { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } + internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } } } From 908316fa766b1db33bda7a28a59c81684775ac89 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 25 Feb 2021 16:48:18 -0800 Subject: [PATCH 5/5] ImportHook: drop dead code around clr_prefix --- src/runtime/importhook.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 14f48712a..9caa04e60 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -259,7 +259,6 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) } string realname = mod_name; - string clr_prefix = null; // 2010-08-15: Always seemed smart to let python try first... // This shaves off a few tenths of a second on test_module.py @@ -317,10 +316,7 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) } return new NewReference(module).DangerousMoveToPointer(); } - if (clr_prefix != null) - { - return GetCLRModule(fromList).DangerousMoveToPointerOrNull(); - } + module = Runtime.PyDict_GetItemString(modules, names[0]); return new NewReference(module, canBeNull: true).DangerousMoveToPointer(); } @@ -360,12 +356,6 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) // Add the module to sys.modules Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference); - - // If imported from CLR add clr. to sys.modules as well - if (clr_prefix != null) - { - Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.ObjectReference); - } } {