diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aba9e9b4..da8f94774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - .NET and Python exceptions are preserved when crossing Python/.NET boundary - BREAKING: custom encoders are no longer called for instances of `System.Type` - `PythonException.Restore` no longer clears `PythonException` instance. +- Replaced the old `__import__` hook hack with a PEP302-style Meta Path Loader ### Fixed diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 0763bfb34..a4b28906c 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -178,6 +178,7 @@ public void TestPythonException_Normalize_ThrowsWhenErrorSet() var pythonException = PythonException.FetchCurrentRaw(); Exceptions.SetError(Exceptions.TypeError, "Another error"); Assert.Throws(() => pythonException.Normalize()); + Exceptions.Clear(); } } } diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index e98461cbb..de8a06bf8 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -37,7 +37,7 @@ public void SetUp() Assert.IsFalse(str == IntPtr.Zero); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); Assert.IsFalse(path.IsNull); - Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.PyList_Append(path, new BorrowedReference(str)); Runtime.Runtime.XDecref(str); } diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 0387d2dfc..d44f5f666 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -37,7 +37,6 @@ internal class AssemblyManager // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; internal static List pypath; - private AssemblyManager() { } @@ -312,6 +311,15 @@ public static bool IsValidNamespace(string name) return !string.IsNullOrEmpty(name) && namespaces.ContainsKey(name); } + /// + /// Returns an IEnumerable containing the namepsaces exported + /// by loaded assemblies in the current app domain. + /// + public static IEnumerable GetNamespaces () + { + return namespaces.Keys; + } + /// /// Returns list of assemblies that declare types in a given namespace /// diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 80f31f058..7cee0890c 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -202,6 +202,16 @@ internal static IntPtr ToPython(object value, Type type) return ClassDerivedObject.ToPython(pyderived); } + // ModuleObjects are created in a way that their wrapping them as + // a CLRObject fails, the ClassObject has no tpHandle. Return the + // pyHandle as is, do not convert. + if (value is ModuleObject modobj) + { + var handle = modobj.pyHandle; + Runtime.XIncref(handle); + return handle; + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 78df805ee..0ebd7ec4c 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -83,7 +83,7 @@ public static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) { message = "readonly attribute"; } - Exceptions.SetError(Exceptions.TypeError, message); + Exceptions.SetError(Exceptions.AttributeError, message); return -1; } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 9ac492d21..1111adc28 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -9,63 +9,45 @@ namespace Python.Runtime /// internal static class ImportHook { - private static IntPtr py_import; private static CLRModule root; - private static MethodWrapper hook; private static IntPtr py_clr_module; static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); - /// - /// Initialize just the __import__ hook itself. - /// - static void InitImport() - { - // We replace the built-in Python __import__ with our own: first - // look in CLR modules, then if we don't find any call the default - // Python __import__. - IntPtr builtins = Runtime.GetBuiltins(); - py_import = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - PythonException.ThrowIfIsNull(py_import); - - hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, hook.ptr); - PythonException.ThrowIfIsNotZero(res); - - Runtime.XDecref(builtins); - } + private const string LoaderCode = @" +import importlib.abc +import sys - /// - /// Restore the __import__ hook. - /// - static void RestoreImport() - { - IntPtr builtins = Runtime.GetBuiltins(); +class DotNetLoader(importlib.abc.Loader): - IntPtr existing = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - Runtime.XDecref(existing); - if (existing != hook.ptr) - { - throw new NotSupportedException("Unable to restore original __import__."); - } + @classmethod + def exec_module(klass, mod): + # This method needs to exist. + pass - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); - PythonException.ThrowIfIsNotZero(res); - Runtime.XDecref(py_import); - py_import = IntPtr.Zero; + @classmethod + def create_module(klass, spec): + import clr + return clr._load_clr_module(spec) - hook.Release(); - hook = null; +class DotNetFinder(importlib.abc.MetaPathFinder): - Runtime.XDecref(builtins); - } + @classmethod + def find_spec(klass, fullname, paths=None, target=None): + # Don't import, we might call ourselves recursively! + if 'clr' not in sys.modules: + return None + clr = sys.modules['clr'] + if clr._available_namespaces and fullname in clr._available_namespaces: + return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) + return None + "; + const string availableNsKey = "_available_namespaces"; /// /// Initialization performed on startup of the Python runtime. /// internal static unsafe void Initialize() { - InitImport(); - // Initialize the clr module and tell Python about it. root = new CLRModule(); @@ -80,9 +62,10 @@ internal static unsafe void Initialize() BorrowedReference dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference); Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + SetupNamespaceTracking(); + SetupImportHook(); } - /// /// Cleanup resources upon shutdown of the Python runtime. /// @@ -93,8 +76,7 @@ internal static void Shutdown() return; } - RestoreImport(); - + TeardownNameSpaceTracking(); Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; @@ -115,187 +97,133 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) internal static void RestoreRuntimeData(RuntimeDataStorage storage) { - InitImport(); storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + BorrowedReference dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + SetupNamespaceTracking(); } - /// - /// Return the clr python module (new reference) - /// - public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default) + static void SetupImportHook() { - root.InitializePreload(); - - // update the module dictionary with the contents of the root dictionary - root.LoadNames(); - BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - using (var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference)) + // Create the import hook module + var import_hook_module = Runtime.PyModule_New("clr.loader"); + + // Run the python code to create the module's classes. + var builtins = Runtime.PyEval_GetBuiltins(); + var exec = Runtime.PyDict_GetItemString(builtins, "exec"); + using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2)); + + var codeStr = NewReference.DangerousFromPointer(Runtime.PyString_FromString(LoaderCode)); + Runtime.PyTuple_SetItem(args, 0, codeStr); + var mod_dict = Runtime.PyModule_GetDict(import_hook_module); + // reference not stolen due to overload incref'ing for us. + Runtime.PyTuple_SetItem(args, 1, mod_dict); + Runtime.PyObject_Call(exec, args, default); + // Set as a sub-module of clr. + if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module.DangerousGetAddress()) != 0) { - Runtime.PyDict_Update(py_mod_dict, clr_dict); + Runtime.XDecref(import_hook_module.DangerousGetAddress()); + throw PythonException.ThrowLastAsClrException(); } - // find any items from the from list and get them from the root if they're not - // already in the module dictionary - if (fromList != null) - { - if (Runtime.PyTuple_Check(fromList)) - { - using var mod_dict = new PyDict(py_mod_dict); - using var from = new PyTuple(fromList); - foreach (PyObject item in from) - { - if (mod_dict.HasKey(item)) - { - continue; - } - - var s = item.AsManagedObject(typeof(string)) as string; - if (s == null) - { - continue; - } - - ManagedType attr = root.GetAttribute(s, true); - if (attr == null) - { - continue; - } - - Runtime.XIncref(attr.pyHandle); - using (var obj = new PyObject(attr.pyHandle)) - { - mod_dict.SetItem(s, obj); - } - } - } - } - Runtime.XIncref(py_clr_module); - return NewReference.DangerousFromPointer(py_clr_module); + // Finally, add the hook to the meta path + var findercls = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder"); + var finderCtorArgs = NewReference.DangerousFromPointer(Runtime.PyTuple_New(0)); + var finder_inst = Runtime.PyObject_CallObject(findercls, finderCtorArgs); + var metapath = Runtime.PySys_GetObject("meta_path"); + Runtime.PyList_Append(metapath, finder_inst); } /// - /// The actual import hook that ties Python to the managed world. + /// Sets up the tracking of loaded namespaces. This makes available to + /// Python, as a Python object, the loaded namespaces. The set of loaded + /// namespaces is used during the import to verify if we can import a + /// CLR assembly as a module or not. The set is stored on the clr module. /// - public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) + static void SetupNamespaceTracking() { - var args = new BorrowedReference(argsRaw); - - // Replacement for the builtin __import__. The original import - // hook is saved as this.py_import. This version handles CLR - // import and defers to the normal builtin for everything else. - - var num_args = Runtime.PyTuple_Size(args); - if (num_args < 1) + using var newset = Runtime.PySet_New(default); + foreach (var ns in AssemblyManager.GetNamespaces()) { - return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); - } - - BorrowedReference py_mod_name = Runtime.PyTuple_GetItem(args, 0); - if (py_mod_name.IsNull || - !Runtime.IsStringType(py_mod_name)) - { - return Exceptions.RaiseTypeError("string expected"); - } - - // Check whether the import is of the form 'from x import y'. - // This determines whether we return the head or tail module. - - BorrowedReference fromList = default; - var fromlist = false; - if (num_args >= 4) - { - fromList = Runtime.PyTuple_GetItem(args, 3); - if (fromList != null && - Runtime.PyObject_IsTrue(fromList) == 1) + using var pyNs = NewReference.DangerousFromPointer(Runtime.PyString_FromString(ns)); + if (Runtime.PySet_Add(newset, pyNs) != 0) { - fromlist = true; + throw PythonException.ThrowLastAsClrException(); } - } - - string mod_name = Runtime.GetManagedString(py_mod_name); - // Check these BEFORE the built-in import runs; may as well - // do the Incref()ed return here, since we've already found - // the module. - if (mod_name == "clr") - { - NewReference clr_module = GetCLRModule(fromList); - if (!clr_module.IsNull()) + if (Runtime.PyDict_SetItemString(root.DictRef, availableNsKey, newset) != 0) { - BorrowedReference sys_modules = Runtime.PyImport_GetModuleDict(); - if (!sys_modules.IsNull) - { - Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); - } + throw PythonException.ThrowLastAsClrException(); } - return clr_module.DangerousMoveToPointerOrNull(); } - string realname = mod_name; - - // 2010-08-15: Always seemed smart to let python try first... - // This shaves off a few tenths of a second on test_module.py - // and works around a quirk where 'sys' is found by the - // LoadImplicit() deprecation logic. - // Turns out that the AssemblyManager.ResolveHandler() checks to see if any - // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very - // little sense to me. - IntPtr res = Runtime.PyObject_Call(py_import, args.DangerousGetAddress(), kw); - if (res != IntPtr.Zero) + } + + /// + /// Removes the set of available namespaces from the clr module. + /// + static void TeardownNameSpaceTracking() + { + // If the C# runtime isn't loaded, then there are no namespaces available + Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone); + } + + public static void AddNamespace(string name) + { + var pyNs = Runtime.PyString_FromString(name); + try { - // There was no error. - if (fromlist && IsLoadAll(fromList)) + var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); + if (!(nsSet.IsNull || nsSet.DangerousGetAddress() == Runtime.PyNone)) { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); + if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } } - return res; } - // There was an error - if (!Exceptions.ExceptionMatches(Exceptions.ImportError)) + finally { - // and it was NOT an ImportError; bail out here. - return IntPtr.Zero; + Runtime.XDecref(pyNs); } + } - if (mod_name == string.Empty) - { - // Most likely a missing relative import. - // For example site-packages\bs4\builder\__init__.py uses it to check if a package exists: - // from . import _html5lib - // We don't support them anyway - return IntPtr.Zero; - } - // Save the exception - var originalException = PythonException.FetchCurrentRaw(); - string[] names = realname.Split('.'); + /// + /// Because we use a proxy module for the clr module, we somtimes need + /// to force the py_clr_module to sync with the actual clr module's dict. + /// + internal static void UpdateCLRModuleDict() + { + root.InitializePreload(); - // See if sys.modules for this interpreter already has the - // requested module. If so, just return the existing module. - BorrowedReference modules = Runtime.PyImport_GetModuleDict(); - BorrowedReference module = Runtime.PyDict_GetItem(modules, py_mod_name); + // update the module dictionary with the contents of the root dictionary + root.LoadNames(); + BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); - if (module != null) - { - if (fromlist) - { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } - return new NewReference(module).DangerousMoveToPointer(); - } + Runtime.PyDict_Update(py_mod_dict, clr_dict); + } - module = Runtime.PyDict_GetItemString(modules, names[0]); - return new NewReference(module, canBeNull: true).DangerousMoveToPointer(); - } - Exceptions.Clear(); + /// + /// Return the clr python module (new reference) + /// + public static unsafe NewReference GetCLRModule() + { + UpdateCLRModuleDict(); + Runtime.XIncref(py_clr_module); + return NewReference.DangerousFromPointer(py_clr_module); + } - // Traverse the qualified module name to get the named module - // and place references in sys.modules as we go. Note that if + /// + /// The hook to import a CLR module into Python. Returns a new reference + /// to the module. + /// + public static ModuleObject Import(string modname) + { + // Traverse the qualified module name to get the named module. + // Note that if // we are running in interactive mode we pre-load the names in // each module, which is often useful for introspection. If we // are not interactive, we stick to just-in-time creation of @@ -304,17 +232,18 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) // enable preloading in a non-interactive python processing by // setting clr.preload = True - ModuleObject head = mod_name == realname ? null : root; + ModuleObject head = null; ModuleObject tail = root; root.InitializePreload(); + string[] names = modname.Split('.'); foreach (string name in names) { ManagedType mt = tail.GetAttribute(name, true); if (!(mt is ModuleObject)) { - originalException.Restore(); - return IntPtr.Zero; + Exceptions.SetError(Exceptions.ImportError, $"'{name}' Is not a ModuleObject."); + throw PythonException.ThrowLastAsClrException(); } if (head == null) { @@ -325,22 +254,9 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) { tail.LoadNames(); } - - // Add the module to sys.modules - Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference); - } - - { - var mod = fromlist ? tail : head; - - if (fromlist && IsLoadAll(fromList)) - { - mod.LoadNames(); - } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; } + tail.IncrRefCount(); + return tail; } private static bool IsLoadAll(BorrowedReference fromList) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index dfb6fdf55..c2614b1d8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -20,6 +20,12 @@ internal class ModuleObject : ExtensionType internal IntPtr dict; internal BorrowedReference DictRef => new BorrowedReference(dict); protected string _namespace; + private IntPtr __all__ = IntPtr.Zero; + + // Attributes to be set on the module according to PEP302 and 451 + // by the import machinery. + static readonly HashSet settableAttributes = + new HashSet {"__spec__", "__file__", "__name__", "__path__", "__loader__", "__package__"}; public ModuleObject(string name) { @@ -47,7 +53,7 @@ public ModuleObject(string name) var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); PythonException.ThrowIfIsNull(dictRef); dict = dictRef.DangerousMoveToPointer(); - + __all__ = Runtime.PyList_New(0); using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); @@ -181,7 +187,23 @@ public void LoadNames() { continue; } - GetAttribute(name, true); + + if(GetAttribute(name, true) != null) + { + // if it's a valid attribute, add it to __all__ + var pyname = Runtime.PyString_FromString(name); + try + { + if (Runtime.PyList_Append(new BorrowedReference(__all__), new BorrowedReference(pyname)) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + finally + { + Runtime.XDecref(pyname); + } + } } } @@ -263,6 +285,13 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return self.dict; } + if (name == "__all__") + { + self.LoadNames(); + Runtime.XIncref(self.__all__); + return self.__all__; + } + ManagedType attr = null; try @@ -320,6 +349,25 @@ protected override void Clear() base.Clear(); } + /// + /// Override the setattr implementation. + /// This is needed because the import mechanics need + /// to set a few attributes + /// + [ForbidPythonThreads] + public new static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) + { + var managedKey = Runtime.GetManagedString(key); + if ((settableAttributes.Contains(managedKey)) || + (ManagedType.GetManagedObject(val)?.GetType() == typeof(ModuleObject)) ) + { + var self = (ModuleObject)ManagedType.GetManagedObject(ob); + return Runtime.PyDict_SetItem(self.dict, key, val); + } + + return ExtensionType.tp_setattro(ob, key, val); + } + protected override void OnSave(InterDomainContext context) { base.OnSave(context); @@ -461,6 +509,7 @@ public static bool SuppressOverloads public static Assembly AddReference(string name) { AssemblyManager.UpdatePath(); + var origNs = AssemblyManager.GetNamespaces(); Assembly assembly = null; assembly = AssemblyManager.FindLoadedAssembly(name); if (assembly == null) @@ -479,7 +528,16 @@ public static Assembly AddReference(string name) { throw new FileNotFoundException($"Unable to find assembly '{name}'."); } - + // Classes that are not in a namespace needs an extra nudge to be found. + ImportHook.UpdateCLRModuleDict(); + + // A bit heavyhanded, but we can't use the AssemblyManager's AssemblyLoadHandler + // method because it may be called from other threads, leading to deadlocks + // if it is called while Python code is executing. + var currNs = AssemblyManager.GetNamespaces().Except(origNs); + foreach(var ns in currNs){ + ImportHook.AddNamespace(ns); + } return assembly; } @@ -526,5 +584,23 @@ public static string[] ListAssemblies(bool verbose) } return names; } + + /// + /// Note: This should *not* be called directly. + /// The function that get/import a CLR assembly as a python module. + /// This function should only be called by the import machinery as seen + /// in importhook.cs + /// + /// A ModuleSpec Python object + /// A new reference to the imported module, as a PyObject. + [ModuleFunction] + [ForbidPythonThreads] + public static ModuleObject _load_clr_module(PyObject spec) + { + ModuleObject mod = null; + using var modname = spec.GetAttr("name"); + mod = ImportHook.Import(modname.ToString()); + return mod; + } } } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index a73a9ae43..6e6da2d93 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -160,6 +160,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "getPreload", "Initialize", "ListAssemblies", + "_load_clr_module", "Release", "Reset", "set_SuppressDocs", diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 039f5e313..8f346524f 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -132,7 +132,7 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(this.Reference, item.obj); + int r = Runtime.PyList_Append(this.Reference, new BorrowedReference(item.obj)); if (r < 0) { throw PythonException.ThrowLastAsClrException(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 537e5348f..009412ea5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -174,7 +174,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd IntPtr item = PyString_FromString(rtdir); if (PySequence_Contains(path, item) == 0) { - PyList_Append(new BorrowedReference(path), item); + PyList_Append(new BorrowedReference(path), new BorrowedReference(item)); } XDecref(item); AssemblyManager.UpdatePath(); @@ -1087,6 +1087,8 @@ internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); + internal static IntPtr PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) + => Delegates.PyObject_Call(pointer.DangerousGetAddress(), args.DangerousGetAddress(), kw.DangerousGetAddressOrNull()); internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); @@ -1796,7 +1798,7 @@ internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr private static int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value) => Delegates.PyList_Insert(pointer, index, value); - internal static int PyList_Append(BorrowedReference pointer, IntPtr value) => Delegates.PyList_Append(pointer, value); + internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); @@ -1859,7 +1861,15 @@ internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) { return PyTuple_SetItem(pointer, new IntPtr(index), value); } + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, StolenReference value) + => PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), value.DangerousGetAddressOrNull()); + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, BorrowedReference value) + { + var increfValue = value.DangerousGetAddress(); + Runtime.XIncref(increfValue); + return PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), increfValue); + } private static int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyTuple_SetItem(pointer, index, value); @@ -1919,6 +1929,22 @@ internal static string PyModule_GetFilename(IntPtr module) internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); + /// + /// We can't use a StolenReference here because the reference is stolen only on success. + /// + /// The module to add the object to. + /// The key that will refer to the object. + /// + /// The object to add to the module. The reference will be stolen only if the + /// method returns 0. + /// + /// Return -1 on error, 0 on success. + internal static int PyModule_AddObject(BorrowedReference module, string name, IntPtr stolenObject) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_AddObject(module, namePtr, stolenObject); + } + /// /// Return value: New reference. /// @@ -2451,7 +2477,7 @@ static Delegates() PyList_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetItem), GetUnmanagedDll(_PythonDll)); PyList_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetItem), GetUnmanagedDll(_PythonDll)); PyList_Insert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Insert), GetUnmanagedDll(_PythonDll)); - PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); + PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); PyList_Reverse = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Reverse), GetUnmanagedDll(_PythonDll)); PyList_Sort = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Sort), GetUnmanagedDll(_PythonDll)); PyList_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetSlice), GetUnmanagedDll(_PythonDll)); @@ -2475,6 +2501,7 @@ static Delegates() { PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); } + PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); @@ -2746,7 +2773,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyList_GetItem { get; } internal static delegate* unmanaged[Cdecl] PyList_SetItem { get; } internal static delegate* unmanaged[Cdecl] PyList_Insert { get; } - internal static delegate* unmanaged[Cdecl] PyList_Append { get; } + internal static delegate* unmanaged[Cdecl] PyList_Append { get; } internal static delegate* unmanaged[Cdecl] PyList_Reverse { get; } internal static delegate* unmanaged[Cdecl] PyList_Sort { get; } internal static delegate* unmanaged[Cdecl] PyList_GetSlice { get; } @@ -2763,6 +2790,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } + internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index a21297829..66fb4f894 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -30,6 +30,11 @@ namespace Python.DomainReloadTests /// which test case to run. That's because pytest assumes we'll run /// everything in one process, but we really want a clean process on each /// test case to test the init/reload/teardown parts of the domain reload. + /// + /// ### Debugging tips: ### + /// * Running pytest with the `-s` argument prevents stdout capture by pytest + /// * Add a sleep into the python test case before the crash/failure, then while + /// sleeping, attach the debugger to the Python.TestDomainReload.exe process. /// /// class TestRunner @@ -1092,6 +1097,29 @@ assert sys.my_obj is not None foo = sys.my_obj.Inner() print(foo) + ", + }, + new TestCase + { + // The C# code for this test doesn't matter; we're testing + // that the import hook behaves properly after a domain reload + Name = "import_after_reload", + DotNetBefore = "", + DotNetAfter = "", + PythonCode = @" +import sys + +def before_reload(): + import clr + import System + + +def after_reload(): + assert 'System' in sys.modules + assert 'clr' in sys.modules + import clr + import System + ", }, }; @@ -1264,7 +1292,13 @@ static string CreateAssembly(string name, string code, bool exe = false) } parameters.ReferencedAssemblies.Add(netstandard); parameters.ReferencedAssemblies.Add(PythonDllLocation); - CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + // Write code to file so it can debugged. + var sourcePath = Path.Combine(TestPath, name+"_source.cs"); + using(var file = new StreamWriter(sourcePath)) + { + file.Write(code); + } + CompilerResults results = provider.CompileAssemblyFromFile(parameters, sourcePath); if (results.NativeCompilerReturnValue != 0) { var stderr = System.Console.Error; diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index a7cd2fa4d..e7a82ded2 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -88,3 +88,6 @@ def test_in_to_ref_param(): def test_nested_type(): _run_test("nested_type") + +def test_import_after_reload(): + _run_test("import_after_reload") \ No newline at end of file