From ea87103c093fb2eac4a2a4bff46226961e88d6b8 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Wed, 21 Apr 2021 16:28:27 -0700 Subject: [PATCH 1/5] Add ability to create module from C# --- src/runtime/pythonengine.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 5925880c0..df522e03a 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -472,6 +472,17 @@ public static void EndAllowThreads(IntPtr ts) Runtime.PyEval_RestoreThread(ts); } + public static PyObject AddModule(string name, string file="") + { + IntPtr module = Runtime.PyImport_AddModule(name); + IntPtr module_globals = Runtime.PyModule_GetDict(module); + IntPtr builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + Runtime.PyDict_SetItemString(module_globals, "__file__", file.ToPython().Handle); + + return new PyObject(module); + } + [Obsolete("Use PyModule.Import")] public static PyObject ImportModule(string name) => PyModule.Import(name); From bf679dc135f105d2c2f526c3f6d3c456dd2b3c53 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Mon, 26 Apr 2021 15:10:03 -0700 Subject: [PATCH 2/5] Move to PyModule and add test --- src/embed_tests/TestPyModule.cs | 52 +++++++++++++++++++++++++++++++++ src/runtime/pymodule.cs | 38 ++++++++++++++++++++++++ src/runtime/pythonengine.cs | 11 ------- 3 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 src/embed_tests/TestPyModule.cs diff --git a/src/embed_tests/TestPyModule.cs b/src/embed_tests/TestPyModule.cs new file mode 100644 index 000000000..ec447fb0c --- /dev/null +++ b/src/embed_tests/TestPyModule.cs @@ -0,0 +1,52 @@ + +using System; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + class TestPyModule + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestCreate() + { + using PyScope scope = Py.CreateScope(); + PyModule testmod = PyModule.Create("testmod"); + testmod.SetAttr("testattr1", "True".ToPython()); + + using PyObject code = PythonEngine.Compile( + "import testmod\n" + + "x = testmod.testattr1" + ); + scope.Execute(code); + + Assert.IsTrue(scope.TryGet("x", out dynamic x)); + Assert.AreEqual("True", x.ToString()); + } + + [Test] + public void TestCreateExisting() + { + using PyScope scope = Py.CreateScope(); + PyModule sysmod = PyModule.Create("sys"); + sysmod.SetAttr("testattr1", "Hello, Python".ToPython()); + + dynamic sys = Py.Import("sys"); + Assert.AreEqual("Hello, Python", sys.testattr1.ToString()); + } + } +} diff --git a/src/runtime/pymodule.cs b/src/runtime/pymodule.cs index 800edb686..0da6bb7ad 100644 --- a/src/runtime/pymodule.cs +++ b/src/runtime/pymodule.cs @@ -1,4 +1,7 @@ using System; +using System.Text; + +using Python.Runtime.Native; namespace Python.Runtime { @@ -37,5 +40,40 @@ public static PyModule FromString(string name, string code) PythonException.ThrowIfIsNull(m); return new PyModule(ref m); } + + public static PyModule Create(string name, string filename = "none") + { + NewReference op; + + // first check if there is an existing module + BorrowedReference modules = Runtime.PyImport_GetModuleDict(); + BorrowedReference m = Runtime.PyDict_GetItemString(modules, name); + if(!m.IsNull && Runtime.PyObject_TypeCheck(m, new BorrowedReference(Runtime.PyModuleType))) + { + // there is already a module by this name + op = new NewReference(m); + return new PyModule(ref op); + } + + // create a new module + op = Runtime.PyModule_New(name); + PythonException.ThrowIfIsNull(op); + + // setup the module basics (__builtins__ and __file__) + BorrowedReference globals = Runtime.PyModule_GetDict(new BorrowedReference(op.DangerousGetAddress())); + PythonException.ThrowIfIsNull(globals); + BorrowedReference __builtins__ = Runtime.PyEval_GetBuiltins(); + PythonException.ThrowIfIsNull(__builtins__); + int rc = Runtime.PyDict_SetItemString(globals, "__builtins__", __builtins__); + PythonException.ThrowIfIsNotZero(rc); + rc = Runtime.PyDict_SetItemString(globals, "__file__", new BorrowedReference(filename.ToPython().Handle)); + PythonException.ThrowIfIsNotZero(rc); + + // add to sys.modules + rc = Runtime.PyDict_SetItemString(modules, name, op); + PythonException.ThrowIfIsNotZero(rc); + + return new PyModule(ref op); + } } } diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index df522e03a..5925880c0 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -472,17 +472,6 @@ public static void EndAllowThreads(IntPtr ts) Runtime.PyEval_RestoreThread(ts); } - public static PyObject AddModule(string name, string file="") - { - IntPtr module = Runtime.PyImport_AddModule(name); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); - Runtime.PyDict_SetItemString(module_globals, "__file__", file.ToPython().Handle); - - return new PyObject(module); - } - [Obsolete("Use PyModule.Import")] public static PyObject ImportModule(string name) => PyModule.Import(name); From f872b3d61296e4042666ef0442a44e2c9df5dfea Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Tue, 27 Jul 2021 16:56:06 -0700 Subject: [PATCH 3/5] Changes based on feedback --- src/embed_tests/TestPyModule.cs | 18 +++++-------- src/runtime/pymodule.cs | 46 +++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/embed_tests/TestPyModule.cs b/src/embed_tests/TestPyModule.cs index ec447fb0c..0ef24544b 100644 --- a/src/embed_tests/TestPyModule.cs +++ b/src/embed_tests/TestPyModule.cs @@ -7,7 +7,7 @@ namespace Python.EmbeddingTest { - class TestPyModule + public class TestPyModule { [OneTimeSetUp] public void SetUp() @@ -25,9 +25,14 @@ public void Dispose() public void TestCreate() { using PyScope scope = Py.CreateScope(); + + Assert.IsFalse(PyModule.IsInSysModules("testmod")); + PyModule testmod = PyModule.Create("testmod"); testmod.SetAttr("testattr1", "True".ToPython()); + testmod.AddToSysModules(); + using PyObject code = PythonEngine.Compile( "import testmod\n" + "x = testmod.testattr1" @@ -37,16 +42,5 @@ public void TestCreate() Assert.IsTrue(scope.TryGet("x", out dynamic x)); Assert.AreEqual("True", x.ToString()); } - - [Test] - public void TestCreateExisting() - { - using PyScope scope = Py.CreateScope(); - PyModule sysmod = PyModule.Create("sys"); - sysmod.SetAttr("testattr1", "Hello, Python".ToPython()); - - dynamic sys = Py.Import("sys"); - Assert.AreEqual("Hello, Python", sys.testattr1.ToString()); - } } } diff --git a/src/runtime/pymodule.cs b/src/runtime/pymodule.cs index 0da6bb7ad..686831d93 100644 --- a/src/runtime/pymodule.cs +++ b/src/runtime/pymodule.cs @@ -32,6 +32,13 @@ public PyModule Reload() return new PyModule(ref op); } + public void AddToSysModules() + { + BorrowedReference modules = Runtime.PyImport_GetModuleDict(); + int rc = Runtime.PyDict_SetItemString(modules, this.Name, this.Reference); + PythonException.ThrowIfIsNotZero(rc); + } + public static PyModule FromString(string name, string code) { using NewReference c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); @@ -41,39 +48,38 @@ public static PyModule FromString(string name, string code) return new PyModule(ref m); } - public static PyModule Create(string name, string filename = "none") + public static bool IsInSysModules(string name) { - NewReference op; - // first check if there is an existing module BorrowedReference modules = Runtime.PyImport_GetModuleDict(); BorrowedReference m = Runtime.PyDict_GetItemString(modules, name); - if(!m.IsNull && Runtime.PyObject_TypeCheck(m, new BorrowedReference(Runtime.PyModuleType))) + return !m.IsNull && Runtime.PyObject_TypeCheck(m, new BorrowedReference(Runtime.PyModuleType)); + } + + public static PyModule Create(string name, string filename=null) + { + NewReference op = Runtime.PyModule_New(name); + PythonException.ThrowIfIsNull(op); + + if (filename != null) { - // there is already a module by this name - op = new NewReference(m); - return new PyModule(ref op); + BorrowedReference globals = Runtime.PyModule_GetDict(new BorrowedReference(op.DangerousGetAddress())); + PythonException.ThrowIfIsNull(globals); + int rc = Runtime.PyDict_SetItemString(globals, "__file__", new BorrowedReference(filename.ToPython().Handle)); + PythonException.ThrowIfIsNotZero(rc); } - // create a new module - op = Runtime.PyModule_New(name); - PythonException.ThrowIfIsNull(op); + return new PyModule(ref op); + } - // setup the module basics (__builtins__ and __file__) - BorrowedReference globals = Runtime.PyModule_GetDict(new BorrowedReference(op.DangerousGetAddress())); + public void AddBuiltins() + { + BorrowedReference globals = Runtime.PyModule_GetDict(this.Reference); PythonException.ThrowIfIsNull(globals); BorrowedReference __builtins__ = Runtime.PyEval_GetBuiltins(); PythonException.ThrowIfIsNull(__builtins__); int rc = Runtime.PyDict_SetItemString(globals, "__builtins__", __builtins__); PythonException.ThrowIfIsNotZero(rc); - rc = Runtime.PyDict_SetItemString(globals, "__file__", new BorrowedReference(filename.ToPython().Handle)); - PythonException.ThrowIfIsNotZero(rc); - - // add to sys.modules - rc = Runtime.PyDict_SetItemString(modules, name, op); - PythonException.ThrowIfIsNotZero(rc); - - return new PyModule(ref op); } } } From d4a32ed37dcc36d9ce9a9b9b5a8f4e502be9af26 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Wed, 28 Jul 2021 14:26:01 -0700 Subject: [PATCH 4/5] Fix some of the feedback --- src/runtime/pymodule.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/runtime/pymodule.cs b/src/runtime/pymodule.cs index 686831d93..26120a4cf 100644 --- a/src/runtime/pymodule.cs +++ b/src/runtime/pymodule.cs @@ -58,14 +58,19 @@ public static bool IsInSysModules(string name) public static PyModule Create(string name, string filename=null) { + if(string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentNullException(nameof(name)); + } + NewReference op = Runtime.PyModule_New(name); PythonException.ThrowIfIsNull(op); if (filename != null) { - BorrowedReference globals = Runtime.PyModule_GetDict(new BorrowedReference(op.DangerousGetAddress())); + BorrowedReference globals = Runtime.PyModule_GetDict(op); PythonException.ThrowIfIsNull(globals); - int rc = Runtime.PyDict_SetItemString(globals, "__file__", new BorrowedReference(filename.ToPython().Handle)); + int rc = Runtime.PyDict_SetItemString(globals, "__file__", filename.ToPython().Reference); PythonException.ThrowIfIsNotZero(rc); } From ab0cedb848c1feade97d8c1f63c6430143ee7fb2 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Wed, 28 Jul 2021 19:06:30 -0700 Subject: [PATCH 5/5] More feedback fixup --- src/embed_tests/TestPyModule.cs | 7 +++--- src/runtime/pymodule.cs | 39 ++++++++++++++++----------------- src/runtime/runtime.cs | 10 +++++++++ 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/embed_tests/TestPyModule.cs b/src/embed_tests/TestPyModule.cs index 0ef24544b..e575a73a6 100644 --- a/src/embed_tests/TestPyModule.cs +++ b/src/embed_tests/TestPyModule.cs @@ -26,12 +26,13 @@ public void TestCreate() { using PyScope scope = Py.CreateScope(); - Assert.IsFalse(PyModule.IsInSysModules("testmod")); + Assert.IsFalse(PyModule.SysModules.HasKey("testmod")); + + PyModule testmod = new PyModule("testmod"); - PyModule testmod = PyModule.Create("testmod"); testmod.SetAttr("testattr1", "True".ToPython()); - testmod.AddToSysModules(); + PyModule.SysModules.SetItem("testmod", testmod); using PyObject code = PythonEngine.Compile( "import testmod\n" + diff --git a/src/runtime/pymodule.cs b/src/runtime/pymodule.cs index 26120a4cf..e6c50bc66 100644 --- a/src/runtime/pymodule.cs +++ b/src/runtime/pymodule.cs @@ -9,6 +9,7 @@ public class PyModule : PyScope { internal PyModule(ref NewReference reference) : base(ref reference, PyScopeManager.Global) { } public PyModule(PyObject o) : base(o.Reference, PyScopeManager.Global) { } + public PyModule(string name, string filename = null) : this(Create(name, filename)) { } /// /// Given a module or package name, import the @@ -32,13 +33,6 @@ public PyModule Reload() return new PyModule(ref op); } - public void AddToSysModules() - { - BorrowedReference modules = Runtime.PyImport_GetModuleDict(); - int rc = Runtime.PyDict_SetItemString(modules, this.Name, this.Reference); - PythonException.ThrowIfIsNotZero(rc); - } - public static PyModule FromString(string name, string code) { using NewReference c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); @@ -48,15 +42,7 @@ public static PyModule FromString(string name, string code) return new PyModule(ref m); } - public static bool IsInSysModules(string name) - { - // first check if there is an existing module - BorrowedReference modules = Runtime.PyImport_GetModuleDict(); - BorrowedReference m = Runtime.PyDict_GetItemString(modules, name); - return !m.IsNull && Runtime.PyObject_TypeCheck(m, new BorrowedReference(Runtime.PyModuleType)); - } - - public static PyModule Create(string name, string filename=null) + private static PyModule Create(string name, string filename=null) { if(string.IsNullOrWhiteSpace(name)) { @@ -77,14 +63,27 @@ public static PyModule Create(string name, string filename=null) return new PyModule(ref op); } - public void AddBuiltins() + public void SetBuiltins(PyDict builtins) { + if(builtins == null || builtins.IsNone()) + { + throw new ArgumentNullException(nameof(builtins)); + } + BorrowedReference globals = Runtime.PyModule_GetDict(this.Reference); PythonException.ThrowIfIsNull(globals); - BorrowedReference __builtins__ = Runtime.PyEval_GetBuiltins(); - PythonException.ThrowIfIsNull(__builtins__); - int rc = Runtime.PyDict_SetItemString(globals, "__builtins__", __builtins__); + int rc = Runtime.PyDict_SetItemString(globals, "__builtins__", builtins.Reference); PythonException.ThrowIfIsNotZero(rc); } + + public static PyDict SysModules + { + get + { + BorrowedReference sysModulesRef = Runtime.PyImport_GetModuleDict(); + PythonException.ThrowIfIsNull(sysModulesRef); + return new PyDict(sysModulesRef); + } + } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 43ee08716..4114fc4d0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2280,6 +2280,16 @@ internal static IntPtr GetBuiltins() return PyImport_Import(PyIdentifier.builtins); } + public static PyDict Builtins + { + get + { + BorrowedReference builtins = PyEval_GetBuiltins(); + PythonException.ThrowIfIsNull(builtins); + return new PyDict(builtins); + } + } + internal static class Delegates { static readonly ILibraryLoader libraryLoader = LibraryLoader.Instance;