From 8134b3390acec8efe7d7e37b852aa1161e07030e Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 23 May 2013 16:47:53 +0100 Subject: [PATCH 01/77] add Python 3 support --- pythonnet/setupegg.py | 8 + pythonnet/setupwin.py | 65 +++++ pythonnet/src/clrmodule/ClrModule.cs | 21 +- pythonnet/src/clrmodule/clrmodule.csproj | 18 +- pythonnet/src/runtime/Python.Runtime.csproj | 34 +-- pythonnet/src/runtime/buildclrmodule.bat | 1 + pythonnet/src/runtime/classbase.cs | 39 ++- pythonnet/src/runtime/clrobject.cs | 6 +- pythonnet/src/runtime/converter.cs | 37 ++- pythonnet/src/runtime/delegateobject.cs | 31 ++- pythonnet/src/runtime/exceptions.cs | 16 +- pythonnet/src/runtime/extensiontype.cs | 2 +- pythonnet/src/runtime/importhook.cs | 57 ++++- pythonnet/src/runtime/interop.cs | 195 +++++++++++++- pythonnet/src/runtime/managedtype.cs | 2 +- pythonnet/src/runtime/methodwrapper.cs | 8 +- pythonnet/src/runtime/moduleobject.cs | 2 +- pythonnet/src/runtime/pythonengine.cs | 12 +- pythonnet/src/runtime/runtime.cs | 269 +++++++++++++++++++- pythonnet/src/runtime/typemanager.cs | 51 +++- 20 files changed, 778 insertions(+), 96 deletions(-) create mode 100644 pythonnet/setupegg.py create mode 100644 pythonnet/setupwin.py diff --git a/pythonnet/setupegg.py b/pythonnet/setupegg.py new file mode 100644 index 000000000..19bd58c75 --- /dev/null +++ b/pythonnet/setupegg.py @@ -0,0 +1,8 @@ +import sys +from setuptools import setup + +if sys.version_info[0] >= 3: + import imp + setupfile = imp.load_source('setupfile', 'setupwin.py') +else: + execfile('setupwin.py') diff --git a/pythonnet/setupwin.py b/pythonnet/setupwin.py new file mode 100644 index 000000000..b40d96519 --- /dev/null +++ b/pythonnet/setupwin.py @@ -0,0 +1,65 @@ +""" +Setup for packaging clr into an egg. +""" +from distutils.core import setup, Extension +from distutils.command.build_ext import build_ext +from platform import architecture +import subprocess +import shutil +import sys +import os + +from distutils import msvc9compiler +msvc9compiler.VERSION = 11 + +class PythonNET_BuildExt(build_ext): + + def build_extension(self, ext): + """ + Builds the .pyd file using msbuild. + """ + if ext.name != "clr": + return super(PythonNET_BuildExt, self).build_extension(ext) + + cc = msvc9compiler.MSVCCompiler() + cc.initialize() + msbuild = cc.find_exe("msbuild.exe") + platform = "x64" if architecture()[0] == "64bit" else "x86" + defines = [ + "PYTHON%d%s" % (sys.version_info[:2]), + "UCS2" + ] + + cmd = [ + msbuild, + "pythonnet.sln", + "/p:Configuration=ReleaseWin", + "/p:Platform=%s" % platform, + "/p:DefineConstants=\"%s\"" % ";".join(defines), + "/t:clrmodule", + ] + self.announce("Building: %s" % " ".join(cmd)) + subprocess.check_call(" ".join(cmd)) + + dest_file = self.get_ext_fullpath(ext.name) + dest_dir = os.path.dirname(dest_file) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + + src_file = os.path.join("src", "clrmodule", "bin", platform, "Release", "clr.pyd") + self.announce("Copying %s to %s" % (src_file, dest_file)) + shutil.copyfile(src_file, dest_file) + + dest_file = os.path.join(dest_dir, "Python.Runtime.dll") + src_file = os.path.join("src", "runtime", "bin", platform, "Release", "Python.Runtime.dll") + self.announce("Copying %s to %s" % (src_file, dest_file)) + shutil.copyfile(src_file, dest_file) + +setup(name="pythonnet", + ext_modules=[ + Extension("clr", sources=[]) + ], + cmdclass = { + "build_ext" : PythonNET_BuildExt + } +) diff --git a/pythonnet/src/clrmodule/ClrModule.cs b/pythonnet/src/clrmodule/ClrModule.cs index 7258017ca..8abb01749 100644 --- a/pythonnet/src/clrmodule/ClrModule.cs +++ b/pythonnet/src/clrmodule/ClrModule.cs @@ -31,7 +31,7 @@ // to indicate what's going on during the load... #define DEBUG_PRINT //============================================================================ - +using System; // ReSharper disable CheckNamespace // ReSharper disable InconsistentNaming @@ -39,10 +39,14 @@ public class clrModule // ReSharper restore InconsistentNaming // ReSharper restore CheckNamespace { - - [RGiesecke.DllExport.DllExport("initclr", System.Runtime.InteropServices.CallingConvention.StdCall)] // ReSharper disable InconsistentNaming +#if (PYTHON32 || PYTHON33 || PYTHON34) + [RGiesecke.DllExport.DllExport("PyInit_clr", System.Runtime.InteropServices.CallingConvention.StdCall)] + public static IntPtr PyInit_clr() +#else + [RGiesecke.DllExport.DllExport("initclr", System.Runtime.InteropServices.CallingConvention.StdCall)] public static void initclr() +#endif // ReSharper restore InconsistentNaming { #if DEBUG_PRINT @@ -77,7 +81,7 @@ public static void initclr() System.Console.WriteLine("Success!"); #endif } - catch (System.IO.FileNotFoundException) + catch (System.IO.IOException) { try { @@ -104,13 +108,22 @@ public static void initclr() #if DEBUG_PRINT System.Console.WriteLine("Could not load Python.Runtime, so sad."); #endif +#if (PYTHON32 || PYTHON33 || PYTHON34) + return IntPtr.Zero; +#else return; +#endif } } // Once here, we've successfully loaded SOME version of Python.Runtime // So now we get the PythonEngine and execute the InitExt method on it. var pythonEngineType = pythonRuntime.GetType("Python.Runtime.PythonEngine"); + +#if (PYTHON32 || PYTHON33 || PYTHON34) + return (IntPtr)pythonEngineType.InvokeMember("InitExt", System.Reflection.BindingFlags.InvokeMethod, null, null, null); +#else pythonEngineType.InvokeMember("InitExt", System.Reflection.BindingFlags.InvokeMethod, null, null, null); +#endif } } diff --git a/pythonnet/src/clrmodule/clrmodule.csproj b/pythonnet/src/clrmodule/clrmodule.csproj index 9d9119f22..7651d3bca 100644 --- a/pythonnet/src/clrmodule/clrmodule.csproj +++ b/pythonnet/src/clrmodule/clrmodule.csproj @@ -18,7 +18,7 @@ full false bin\Debug\ - DEBUG;TRACE + DEBUG;TRACE prompt 4 x86 @@ -27,7 +27,7 @@ pdbonly true bin\Release\ - TRACE + TRACE prompt 4 x86 @@ -42,7 +42,7 @@ true bin\x86\Debug\ - DEBUG;TRACE + DEBUG;TRACE full x86 prompt @@ -52,7 +52,7 @@ bin\x86\Release\ - TRACE + TRACE true pdbonly x86 @@ -64,7 +64,7 @@ true bin\DebugMono_x86\ - DEBUG;TRACE + DEBUG;TRACE full x86 prompt @@ -74,7 +74,7 @@ true bin\x86\DebugMono_x86\ - DEBUG;TRACE + DEBUG;TRACE full x86 prompt @@ -85,7 +85,7 @@ true bin\x64\Debug\ - DEBUG;TRACE + DEBUG;TRACE full x64 prompt @@ -94,7 +94,7 @@ bin\x64\Release\ - TRACE + TRACE true pdbonly x64 @@ -105,7 +105,7 @@ true bin\x64\DebugMono_x86\ - DEBUG;TRACE + DEBUG;TRACE full x64 prompt diff --git a/pythonnet/src/runtime/Python.Runtime.csproj b/pythonnet/src/runtime/Python.Runtime.csproj index a3ceaf63c..be7ca1174 100644 --- a/pythonnet/src/runtime/Python.Runtime.csproj +++ b/pythonnet/src/runtime/Python.Runtime.csproj @@ -15,20 +15,20 @@ full true .\bin\Debug\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true pdbonly true .\bin\Release\ - TRACE;PYTHON27, UCS2 + TRACE;PYTHON27, UCS2 true true bin\EmbeddingTest\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -37,14 +37,14 @@ true bin\UnitTests\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full AnyCPU - true + false pythonnet.snk @@ -52,7 +52,7 @@ true bin\x86\Debug\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -60,7 +60,7 @@ bin\x86\Release\ - PYTHON27, UCS4 + PYTHON27, UCS4 true true pdbonly @@ -70,7 +70,7 @@ true - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -82,7 +82,7 @@ true bin\x86\UnitTests\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -91,7 +91,7 @@ true bin\DebugMono_x86\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -103,7 +103,7 @@ true bin\x86\DebugMono_x86\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -112,7 +112,7 @@ true bin\x64\Debug\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -121,7 +121,7 @@ bin\x64\Release\ - PYTHON27, UCS4 + PYTHON32, UCS2 true true pdbonly @@ -132,7 +132,7 @@ true bin\x64\EmbeddingTest\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -144,7 +144,7 @@ true bin\x64\UnitTests\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -156,7 +156,7 @@ true bin\x64\DebugMono_x86\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -244,4 +244,4 @@ copy "$(TargetDir)clr.pyd" "$(SolutionDir)" del "$(TargetDir)clr.pyd" - \ No newline at end of file + diff --git a/pythonnet/src/runtime/buildclrmodule.bat b/pythonnet/src/runtime/buildclrmodule.bat index 1a0ae5a02..2cabaa702 100644 --- a/pythonnet/src/runtime/buildclrmodule.bat +++ b/pythonnet/src/runtime/buildclrmodule.bat @@ -8,6 +8,7 @@ set INPUT_PATH="%INPUT_DIRECTORY%\clrmodule.il" set OUTPUT_PATH=%3 if %TARGET_PLATFORM%==AnyCPU goto SETUP32 +if %TARGET_PLATFORM%==x86 goto SETUP32 if %TARGET_PLATFORM%==x64 goto SETUP64 goto ERROR_BAD_PLATFORM diff --git a/pythonnet/src/runtime/classbase.cs b/pythonnet/src/runtime/classbase.cs index 1541b12cd..daad173c8 100644 --- a/pythonnet/src/runtime/classbase.cs +++ b/pythonnet/src/runtime/classbase.cs @@ -57,7 +57,43 @@ public virtual IntPtr type_subscript(IntPtr idx) { //==================================================================== // Standard comparison implementation for instances of reflected types. //==================================================================== +#if (PYTHON32 || PYTHON33 || PYTHON34) + public static IntPtr tp_richcompare(IntPtr ob, IntPtr other, int op) { + if (op != Runtime.Py_EQ && op != Runtime.Py_NE) + { + Runtime.Incref(Runtime.PyNotImplemented); + return Runtime.PyNotImplemented; + } + + IntPtr pytrue = Runtime.PyTrue; + IntPtr pyfalse = Runtime.PyFalse; + + // swap true and false for NE + if (op != Runtime.Py_EQ) + { + pytrue = Runtime.PyFalse; + pyfalse = Runtime.PyTrue; + } + + if (ob == other) { + Runtime.Incref(pytrue); + return pytrue; + } + + CLRObject co1 = GetManagedObject(ob) as CLRObject; + CLRObject co2 = GetManagedObject(other) as CLRObject; + Object o1 = co1.inst; + Object o2 = co2.inst; + if (Object.Equals(o1, o2)) { + Runtime.Incref(pytrue); + return pytrue; + } + + Runtime.Incref(pyfalse); + return pyfalse; + } +#else public static int tp_compare(IntPtr ob, IntPtr other) { if (ob == other) { return 0; @@ -73,6 +109,7 @@ public static int tp_compare(IntPtr ob, IntPtr other) { } return -1; } +#endif //==================================================================== @@ -154,7 +191,7 @@ public static int tp_is_gc(IntPtr type) { public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.ob_dict); + IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); if (dict != IntPtr.Zero) { Runtime.Decref(dict); } diff --git a/pythonnet/src/runtime/clrobject.cs b/pythonnet/src/runtime/clrobject.cs index c61f9523d..1de49aede 100644 --- a/pythonnet/src/runtime/clrobject.cs +++ b/pythonnet/src/runtime/clrobject.cs @@ -25,15 +25,15 @@ internal CLRObject(Object ob, IntPtr tp) : base() { int flags = (int)Marshal.ReadIntPtr(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.ob_dict); + IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.DictOffset(tp)); if (dict == IntPtr.Zero) { dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.ob_dict, dict); + Marshal.WriteIntPtr(py, ObjectOffset.DictOffset(tp), dict); } } GCHandle gc = GCHandle.Alloc(this); - Marshal.WriteIntPtr(py, ObjectOffset.magic(), (IntPtr)gc); + Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); this.tpHandle = tp; this.pyHandle = py; this.gcHandle = gc; diff --git a/pythonnet/src/runtime/converter.cs b/pythonnet/src/runtime/converter.cs index af7acd972..063a57294 100644 --- a/pythonnet/src/runtime/converter.cs +++ b/pythonnet/src/runtime/converter.cs @@ -381,6 +381,18 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, return true; case TypeCode.Byte: +#if (PYTHON32 || PYTHON33 || PYTHON34) + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) + { + if (Runtime.PyBytes_Size(value) == 1) + { + op = Runtime.PyBytes_AS_STRING(value); + result = (byte)Marshal.ReadByte(op); + return true; + } + goto type_error; + } +#else if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) { if (Runtime.PyString_Size(value) == 1) { op = Runtime.PyString_AS_STRING(value); @@ -389,6 +401,7 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, } goto type_error; } +#endif op = Runtime.PyNumber_Int(value); if (op == IntPtr.Zero) { @@ -408,6 +421,16 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, return true; case TypeCode.SByte: +#if (PYTHON32 || PYTHON33 || PYTHON34) + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { + if (Runtime.PyBytes_Size(value) == 1) { + op = Runtime.PyBytes_AS_STRING(value); + result = (byte)Marshal.ReadByte(op); + return true; + } + goto type_error; + } +#else if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) { if (Runtime.PyString_Size(value) == 1) { op = Runtime.PyString_AS_STRING(value); @@ -416,6 +439,7 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, } goto type_error; } +#endif op = Runtime.PyNumber_Int(value); if (op == IntPtr.Zero) { @@ -435,7 +459,16 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, return true; case TypeCode.Char: - +#if (PYTHON32 || PYTHON33 || PYTHON34) + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) { + if (Runtime.PyBytes_Size(value) == 1) { + op = Runtime.PyBytes_AS_STRING(value); + result = (byte)Marshal.ReadByte(op); + return true; + } + goto type_error; + } +#else if (Runtime.PyObject_TypeCheck(value, Runtime.PyStringType)) { if (Runtime.PyString_Size(value) == 1) { op = Runtime.PyString_AS_STRING(value); @@ -444,7 +477,7 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, } goto type_error; } - +#endif else if (Runtime.PyObject_TypeCheck(value, Runtime.PyUnicodeType)) { if (Runtime.PyUnicode_GetSize(value) == 1) { diff --git a/pythonnet/src/runtime/delegateobject.cs b/pythonnet/src/runtime/delegateobject.cs index 839fb71e5..473b2e81c 100644 --- a/pythonnet/src/runtime/delegateobject.cs +++ b/pythonnet/src/runtime/delegateobject.cs @@ -103,7 +103,36 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { //==================================================================== // Implements __cmp__ for reflected delegate types. //==================================================================== +#if (PYTHON32 || PYTHON33 || PYTHON34) + public static new IntPtr tp_richcompare(IntPtr ob, IntPtr other, int op) { + if (op != Runtime.Py_EQ && op != Runtime.Py_NE) + { + Runtime.Incref(Runtime.PyNotImplemented); + return Runtime.PyNotImplemented; + } + + IntPtr pytrue = Runtime.PyTrue; + IntPtr pyfalse = Runtime.PyFalse; + + // swap true and false for NE + if (op != Runtime.Py_EQ) + { + pytrue = Runtime.PyFalse; + pyfalse = Runtime.PyTrue; + } + + Delegate d1 = GetTrueDelegate(ob); + Delegate d2 = GetTrueDelegate(other); + if (d1 == d2) + { + Runtime.Incref(pytrue); + return pytrue; + } + Runtime.Incref(pyfalse); + return pyfalse; + } +#else public static new int tp_compare(IntPtr ob, IntPtr other) { Delegate d1 = GetTrueDelegate(ob); Delegate d2 = GetTrueDelegate(other); @@ -112,7 +141,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { } return -1; } - +#endif } diff --git a/pythonnet/src/runtime/exceptions.cs b/pythonnet/src/runtime/exceptions.cs index f08217dac..5857c1e35 100644 --- a/pythonnet/src/runtime/exceptions.cs +++ b/pythonnet/src/runtime/exceptions.cs @@ -31,7 +31,7 @@ internal class ExceptionClassObject : ClassObject { internal ExceptionClassObject(Type tp) : base(tp) { } -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) internal static Exception ToException(IntPtr ob) { CLRObject co = GetManagedObject(ob) as CLRObject; if (co == null) { @@ -114,7 +114,7 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return Runtime.PyObject_GenericGetAttr(ob, key); } -#endif // (PYTHON25 || PYTHON26 || PYTHON27) +#endif // (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) } /// @@ -136,7 +136,11 @@ private Exceptions() {} //=================================================================== internal static void Initialize() { +#if (PYTHON32 || PYTHON33 || PYTHON34) + exceptions_module = Runtime.PyImport_ImportModule("builtins"); +#else exceptions_module = Runtime.PyImport_ImportModule("exceptions"); +#endif Exceptions.ErrorCheck(exceptions_module); warnings_module = Runtime.PyImport_ImportModule("warnings"); Exceptions.ErrorCheck(warnings_module); @@ -565,15 +569,17 @@ internal static IntPtr RaiseTypeError(string message) { puplic static variables on the Exceptions class filled in from the python class using reflection in Initialize() looked up by name, not posistion. */ -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) public static IntPtr BaseException; #endif public static IntPtr Exception; public static IntPtr StopIteration; -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) public static IntPtr GeneratorExit; #endif +#if !(PYTHON32 || PYTHON33 || PYTHON34) public static IntPtr StandardError; +#endif public static IntPtr ArithmeticError; public static IntPtr LookupError; @@ -628,7 +634,7 @@ puplic static variables on the Exceptions class filled in from public static IntPtr SyntaxWarning; public static IntPtr RuntimeWarning; public static IntPtr FutureWarning; -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) public static IntPtr ImportWarning; public static IntPtr UnicodeWarning; //PyAPI_DATA(PyObject *) PyExc_BytesWarning; diff --git a/pythonnet/src/runtime/extensiontype.cs b/pythonnet/src/runtime/extensiontype.cs index b0499bb0a..75ac67e59 100644 --- a/pythonnet/src/runtime/extensiontype.cs +++ b/pythonnet/src/runtime/extensiontype.cs @@ -40,7 +40,7 @@ public ExtensionType() : base() { IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); GCHandle gc = GCHandle.Alloc(this); - Marshal.WriteIntPtr(py, ObjectOffset.magic(), (IntPtr)gc); + Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most diff --git a/pythonnet/src/runtime/importhook.cs b/pythonnet/src/runtime/importhook.cs index 9d618b60d..5a4176203 100644 --- a/pythonnet/src/runtime/importhook.cs +++ b/pythonnet/src/runtime/importhook.cs @@ -23,29 +23,53 @@ internal class ImportHook { static CLRModule root; static MethodWrapper hook; +#if (PYTHON32 || PYTHON33 || PYTHON34) + static IntPtr py_clr_module; + static IntPtr module_def; +#endif + //=================================================================== // Initialization performed on startup of the Python runtime. //=================================================================== internal static void Initialize() { - // Initialize the Python <--> CLR module hook. We replace the // built-in Python __import__ with our own. This isn't ideal, // but it provides the most "Pythonic" way of dealing with CLR // modules (Python doesn't provide a way to emulate packages). - IntPtr dict = Runtime.PyImport_GetModuleDict(); +#if (PYTHON32 || PYTHON33 || PYTHON34) + IntPtr mod = Runtime.PyImport_ImportModule("builtins"); + py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); +#else IntPtr mod = Runtime.PyDict_GetItemString(dict, "__builtin__"); py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); - +#endif hook = new MethodWrapper(typeof(ImportHook), "__import__"); Runtime.PyObject_SetAttrString(mod, "__import__", hook.ptr); Runtime.Decref(hook.ptr); root = new CLRModule(); + +#if (PYTHON32 || PYTHON33 || PYTHON34) + // create a python module with the same methods as the clr module-like object + module_def = ModuleDefOffset.AllocModuleDef("clr"); + py_clr_module = Runtime.PyModule_Create2(module_def, 3); + + // both dicts are borrowed references + IntPtr mod_dict = Runtime.PyModule_GetDict(py_clr_module); + IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject** + clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); + + Runtime.PyDict_Update(mod_dict, clr_dict); + Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); + Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); +#else Runtime.Incref(root.pyHandle); // we are using the module two times Runtime.PyDict_SetItemString(dict, "CLR", root.pyHandle); Runtime.PyDict_SetItemString(dict, "clr", root.pyHandle); +#endif + } @@ -54,11 +78,30 @@ internal static void Initialize() { //=================================================================== internal static void Shutdown() { +#if (PYTHON32 || PYTHON33 || PYTHON34) + Runtime.Decref(py_clr_module); Runtime.Decref(root.pyHandle); + ModuleDefOffset.FreeModuleDef(module_def); +#else Runtime.Decref(root.pyHandle); + Runtime.Decref(root.pyHandle); +#endif Runtime.Decref(py_import); } + //=================================================================== + // Return the clr python module (new reference) + //=================================================================== + public static IntPtr GetCLRModule() { + root.InitializePreload(); +#if (PYTHON32 || PYTHON33 || PYTHON34) + Runtime.Incref(py_clr_module); + return py_clr_module; +#else + Runtime.Incref(root.pyHandle); + return root.pyHandle; +#endif + } //=================================================================== // The actual import hook that ties Python to the managed world. @@ -102,16 +145,12 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { // do the Incref()ed return here, since we've already found // the module. if (mod_name == "clr") { - root.InitializePreload(); - Runtime.Incref(root.pyHandle); - return root.pyHandle; + return GetCLRModule(); } if (mod_name == "CLR") { Exceptions.deprecation("The CLR module is deprecated. " + "Please use 'clr'."); - root.InitializePreload(); - Runtime.Incref(root.pyHandle); - return root.pyHandle; + return GetCLRModule(); } string realname = mod_name; if (mod_name.StartsWith("CLR.")) { diff --git a/pythonnet/src/runtime/interop.cs b/pythonnet/src/runtime/interop.cs index 9aad4c6e4..ecbf60eb9 100644 --- a/pythonnet/src/runtime/interop.cs +++ b/pythonnet/src/runtime/interop.cs @@ -12,6 +12,7 @@ using System.Collections.Specialized; using System.Runtime.InteropServices; using System.Reflection; +using System.Text; namespace Python.Runtime { @@ -77,11 +78,37 @@ static ObjectOffset() { ob_data = (n+3) * size; } - public static int magic() { + public static int magic(IntPtr ob) { +#if (PYTHON32 || PYTHON33 || PYTHON33) + if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || + (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + { + return ExceptionOffset.ob_data; + } +#endif return ob_data; } - public static int Size() { + public static int DictOffset(IntPtr ob) + { +#if (PYTHON32 || PYTHON33 || PYTHON33) + if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || + (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + { + return ExceptionOffset.ob_dict; + } +#endif + return ob_dict; + } + + public static int Size(IntPtr ob) { +#if (PYTHON32 || PYTHON33 || PYTHON33) + if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || + (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + { + return ExceptionOffset.Size(); + } +#endif #if (Py_DEBUG) return 6 * IntPtr.Size; #else @@ -95,10 +122,43 @@ public static int Size() { #endif public static int ob_refcnt; public static int ob_type; + private static int ob_dict; + private static int ob_data; + } + +#if (PYTHON32 || PYTHON33 || PYTHON34) + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ExceptionOffset + { + static ExceptionOffset() + { + Type type = typeof(ExceptionOffset); + FieldInfo[] fi = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, (i * size) + ObjectOffset.ob_type + size); + } + } + + public static int Size() + { + return ob_data + IntPtr.Size; + } + + // PyException_HEAD + // (start after PyObject_HEAD) + public static int dict = 0; + public static int args = 0; + public static int traceback = 0; + public static int context = 0; + public static int cause = 0; + + // extra c# data public static int ob_dict; public static int ob_data; } - +#endif [StructLayout(LayoutKind.Sequential, CharSet=CharSet.Ansi)] internal class TypeOffset { @@ -139,7 +199,7 @@ public static int magic() { public static int tp_print = 0; public static int tp_getattr = 0; public static int tp_setattr = 0; - public static int tp_compare = 0; + public static int tp_compare = 0; /* tp_reserved in Python 3 */ public static int tp_repr = 0; /* Method suites for standard classes */ @@ -198,7 +258,7 @@ public static int magic() { public static int tp_subclasses = 0; public static int tp_weaklist = 0; public static int tp_del = 0; -#if (PYTHON26 || PYTHON27) +#if (PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) /* Type attribute cache version tag. Added in version 2.6 */ public static int tp_version_tag; #endif @@ -212,11 +272,14 @@ public static int magic() { public static int tp_next = 0; #endif //} PyTypeObject; + //typedef struct { public static int nb_add = 0; public static int nb_subtract = 0; public static int nb_multiply = 0; +#if !(PYTHON32 || PYTHON33 || PYTHON34) public static int nb_divide = 0; +#endif public static int nb_remainder = 0; public static int nb_divmod = 0; public static int nb_power = 0; @@ -230,17 +293,23 @@ public static int magic() { public static int nb_and = 0; public static int nb_xor = 0; public static int nb_or = 0; +#if !(PYTHON32 || PYTHON33 || PYTHON34) public static int nb_coerce = 0; +#endif public static int nb_int = 0; public static int nb_long = 0; public static int nb_float = 0; +#if !(PYTHON32 || PYTHON33 || PYTHON34) public static int nb_oct = 0; public static int nb_hex = 0; +#endif /* Added in release 2.0 */ public static int nb_inplace_add = 0; public static int nb_inplace_subtract = 0; public static int nb_inplace_multiply = 0; +#if !(PYTHON32 || PYTHON33 || PYTHON34) public static int nb_inplace_divide = 0; +#endif public static int nb_inplace_remainder = 0; public static int nb_inplace_power = 0; public static int nb_inplace_lshift = 0; @@ -254,7 +323,7 @@ public static int magic() { public static int nb_true_divide = 0; public static int nb_inplace_floor_divide = 0; public static int nb_inplace_true_divide = 0; -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) /* Added in release 2.5 */ public static int nb_index = 0; #endif @@ -278,11 +347,13 @@ public static int magic() { public static int sq_inplace_repeat = 0; //} PySequenceMethods; //typedef struct { +#if !(PYTHON32 || PYTHON33 || PYTHON34) public static int bf_getreadbuffer = 0; public static int bf_getwritebuffer = 0; public static int bf_getsegcount = 0; public static int bf_getcharbuffer = 0; -#if (PYTHON26 || PYTHON27) +#endif +#if (PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) // This addition is not actually noted in the 2.6.5 object.h public static int bf_getbuffer = 0; public static int bf_releasebuffer = 0; @@ -291,10 +362,107 @@ public static int magic() { //PyObject *ht_name, *ht_slots; public static int name = 0; public static int slots = 0; + +#if (PYTHON33 || PYTHON34) + public static int qualname = 0; + public static int cached_keys; +#endif + /* here are optional user slots, followed by the members. */ public static int members = 0; } +#if (PYTHON32 || PYTHON33 || PYTHON34) + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class BytesOffset + { + static BytesOffset() + { + Type type = typeof(BytesOffset); + FieldInfo[] fi = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, i * size); + } + } + + /* The *real* layout of a type object when allocated on the heap */ + //typedef struct _heaptypeobject { +#if (Py_DEBUG) // #ifdef Py_TRACE_REFS +/* _PyObject_HEAD_EXTRA defines pointers to support a doubly-linked list of all live heap objects. */ + public static int _ob_next = 0; + public static int _ob_prev = 0; +#endif + // PyObject_VAR_HEAD { + // PyObject_HEAD { + public static int ob_refcnt = 0; + public static int ob_type = 0; + // } + public static int ob_size = 0; /* Number of items in _VAR_iable part */ + // } + public static int ob_shash = 0; + public static int ob_sval = 0; /* start of data */ + + /* Invariants: + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. + * ob_shash is the hash of the string or -1 if not computed yet. + */ + //} PyBytesObject; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ModuleDefOffset + { + static ModuleDefOffset() + { + Type type = typeof(ModuleDefOffset); + FieldInfo[] fi = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, (i * size) + TypeOffset.ob_size); + } + } + + public static IntPtr AllocModuleDef(string modulename) { + byte[] ascii = Encoding.ASCII.GetBytes(modulename); + int size = name + ascii.Length + 1; + IntPtr ptr = Marshal.AllocHGlobal(size); + for (int i = 0; i <= m_free; i += IntPtr.Size) + Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); + Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); + Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); + Marshal.WriteByte(ptr, name + ascii.Length, 0); + return ptr; + } + + public static void FreeModuleDef(IntPtr ptr) { + Marshal.FreeHGlobal(ptr); + } + + // typedef struct PyModuleDef{ + // typedef struct PyModuleDef_Base { + // starts after PyObject_HEAD (TypeOffset.ob_type + 1) + public static int m_init = 0; + public static int m_index = 0; + public static int m_copy = 0; + // } PyModuleDef_Base + public static int m_name = 0; + public static int m_doc = 0; + public static int m_size = 0; + public static int m_methods = 0; + public static int m_reload = 0; + public static int m_traverse = 0; + public static int m_clear = 0; + public static int m_free = 0; + // } PyModuleDef + + public static int name = 0; + } +#endif // PYTHON3 + /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. @@ -321,10 +489,10 @@ internal class TypeFlags { /* XXX Reusing reserved constants */ public static int Managed = (1 << 15); // PythonNet specific public static int Subclass = (1 << 16); // PythonNet specific -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) public static int HaveIndex = (1 << 17); #endif -#if (PYTHON26 || PYTHON27) +#if (PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) /* Objects support nb_index in PyNumberMethods */ public static int HaveVersionTag = (1 << 18); public static int ValidVersionTag = (1 << 19); @@ -349,7 +517,7 @@ internal class TypeFlags { HaveIter | HaveClass | HaveStacklessExtension | -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) HaveIndex | #endif 0); @@ -410,7 +578,9 @@ static Interop() { pmap["nb_add"] = p["BinaryFunc"]; pmap["nb_subtract"] = p["BinaryFunc"]; pmap["nb_multiply"] = p["BinaryFunc"]; +#if !(PYTHON32 || PYTHON33 || PYTHON34) pmap["nb_divide"] = p["BinaryFunc"]; +#endif pmap["nb_remainder"] = p["BinaryFunc"]; pmap["nb_divmod"] = p["BinaryFunc"]; pmap["nb_power"] = p["TernaryFunc"]; @@ -433,7 +603,9 @@ static Interop() { pmap["nb_inplace_add"] = p["BinaryFunc"]; pmap["nb_inplace_subtract"] = p["BinaryFunc"]; pmap["nb_inplace_multiply"] = p["BinaryFunc"]; +#if !(PYTHON32 || PYTHON33 || PYTHON34) pmap["nb_inplace_divide"] = p["BinaryFunc"]; +#endif pmap["nb_inplace_remainder"] = p["BinaryFunc"]; pmap["nb_inplace_power"] = p["TernaryFunc"]; pmap["nb_inplace_lshift"] = p["BinaryFunc"]; @@ -445,7 +617,7 @@ static Interop() { pmap["nb_true_divide"] = p["BinaryFunc"]; pmap["nb_inplace_floor_divide"] = p["BinaryFunc"]; pmap["nb_inplace_true_divide"] = p["BinaryFunc"]; -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) pmap["nb_index"] = p["UnaryFunc"]; #endif @@ -541,5 +713,4 @@ public Thunk(Delegate d) { fn = d; } } - } diff --git a/pythonnet/src/runtime/managedtype.cs b/pythonnet/src/runtime/managedtype.cs index 670bcd2b3..78e29c2b6 100644 --- a/pythonnet/src/runtime/managedtype.cs +++ b/pythonnet/src/runtime/managedtype.cs @@ -42,7 +42,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) { if ((flags & TypeFlags.Managed) != 0) { IntPtr op = (tp == ob) ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) : - Marshal.ReadIntPtr(ob, ObjectOffset.magic()); + Marshal.ReadIntPtr(ob, ObjectOffset.magic(ob)); GCHandle gc = (GCHandle)op; return (ManagedType)gc.Target; } diff --git a/pythonnet/src/runtime/methodwrapper.cs b/pythonnet/src/runtime/methodwrapper.cs index 04a49d592..9b701f154 100644 --- a/pythonnet/src/runtime/methodwrapper.cs +++ b/pythonnet/src/runtime/methodwrapper.cs @@ -34,9 +34,13 @@ public MethodWrapper(Type type, string name) { // XXX - here we create a Python string object, then take the // char * of the internal string to pass to our methoddef // structure. Its a hack, and the name is leaked! - +#if (PYTHON32 || PYTHON33 || PYTHON34) + IntPtr ps = Runtime.PyBytes_FromString(name); + IntPtr sp = Runtime.PyBytes_AS_STRING(ps); +#else IntPtr ps = Runtime.PyString_FromString(name); IntPtr sp = Runtime.PyString_AS_STRING(ps); +#endif // Allocate and initialize a PyMethodDef structure to represent // the managed method, then create a PyCFunction. @@ -44,7 +48,7 @@ public MethodWrapper(Type type, string name) { mdef = Runtime.PyMem_Malloc(4 * IntPtr.Size); Marshal.WriteIntPtr(mdef, sp); Marshal.WriteIntPtr(mdef, (1 * IntPtr.Size), fp); - Marshal.WriteIntPtr(mdef, (2 * IntPtr.Size), (IntPtr)0x0002); + Marshal.WriteIntPtr(mdef, (2 * IntPtr.Size), (IntPtr)0x0003); // METH_VARARGS | METH_KEYWORDS Marshal.WriteIntPtr(mdef, (3 * IntPtr.Size), IntPtr.Zero); ptr = Runtime.PyCFunction_New(mdef, IntPtr.Zero); } diff --git a/pythonnet/src/runtime/moduleobject.cs b/pythonnet/src/runtime/moduleobject.cs index 3a3991947..b39135cdc 100644 --- a/pythonnet/src/runtime/moduleobject.cs +++ b/pythonnet/src/runtime/moduleobject.cs @@ -44,7 +44,7 @@ public ModuleObject(string name) : base() { Runtime.PyDict_SetItemString(dict, "__doc__", Runtime.PyNone); Runtime.Decref(pyname); - Marshal.WriteIntPtr(this.pyHandle, ObjectOffset.ob_dict, dict); + Marshal.WriteIntPtr(this.pyHandle, ObjectOffset.DictOffset(this.pyHandle), dict); InitializeModuleMembers(); } diff --git a/pythonnet/src/runtime/pythonengine.cs b/pythonnet/src/runtime/pythonengine.cs index 07326185f..460928f98 100644 --- a/pythonnet/src/runtime/pythonengine.cs +++ b/pythonnet/src/runtime/pythonengine.cs @@ -126,8 +126,11 @@ public static void Initialize() { // CPython interpreter process - this bootstraps the managed runtime // when it is imported by the CLR extension module. //==================================================================== - +#if (PYTHON32 || PYTHON33 || PYTHON34) + public static IntPtr InitExt() { +#else public static void InitExt() { +#endif Initialize(); // Trickery - when the import hook is installed into an already @@ -157,15 +160,18 @@ public static void InitExt() { " line.startswith('import clr') or \\\n" + " line.startswith('from clr') or \\\n" + " line.startswith('from CLR'):\n" + - " exec line\n" + + " exec(line)\n" + " break\n"; PyObject r = PythonEngine.RunString(code); if (r != null) { r.Dispose(); } - } +#if (PYTHON32 || PYTHON33 || PYTHON34) + return Python.Runtime.ImportHook.GetCLRModule(); +#endif + } /// /// Shutdown Method diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 1218f2b7d..7cf5b19b4 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -15,6 +15,10 @@ using Mono.Unix; #endif +#if (UCS2 && (PYTHON32 || PYTHON33 || PYTHON34)) +using System.Text; +#endif + namespace Python.Runtime { [SuppressUnmanagedCodeSecurityAttribute()] @@ -61,8 +65,23 @@ public class Runtime { public const string pyversion = "2.7"; public const int pyversionnumber = 27; #endif -#if ! (PYTHON23 || PYTHON24 || PYTHON25 || PYTHON26 || PYTHON27) -#error You must define one of PYTHON23 to PYTHON27 +#if (PYTHON32) + public const string dll = "python32"; + public const string pyversion = "3.2"; + public const int pyversionnumber = 32; +#endif +#if (PYTHON33) + public const string dll = "python33"; + public const string pyversion = "3.3"; + public const int pyversionnumber = 33; +#endif +#if (PYTHON34) + public const string dll = "python34"; + public const string pyversion = "3.4"; + public const int pyversionnumber = 34; +#endif +#if ! (PYTHON23 || PYTHON24 || PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) +#error You must define one of PYTHON23 to PYTHON34 #endif internal static bool wrap_exceptions; @@ -75,12 +94,20 @@ internal static void Initialize() { is32bit = IntPtr.Size == 4; + if (0 == Runtime.Py_IsInitialized()) + { Runtime.Py_Initialize(); Runtime.PyEval_InitThreads(); + } +#if (PYTHON32 || PYTHON33 || PYTHON34) + IntPtr op = Runtime.PyImport_ImportModule("builtins"); + IntPtr dict = Runtime.PyObject_GetAttrString(op, "__dict__"); + PyNotImplemented = Runtime.PyObject_GetAttrString(op, "NotImplemented"); +#else IntPtr dict = Runtime.PyImport_GetModuleDict(); IntPtr op = Runtime.PyDict_GetItemString(dict, "__builtin__"); - +#endif PyBaseObjectType = Runtime.PyObject_GetAttrString(op, "object"); PyModuleType = Runtime.PyObject_Type(op); @@ -96,6 +123,11 @@ internal static void Initialize() { PyMethodType = Runtime.PyObject_Type(op); Runtime.Decref(op); +#if (PYTHON32 || PYTHON33 || PYTHON34) + Runtime.Decref(dict); + Runtime.Decref(op); +#endif + op = Runtime.PyString_FromString("string"); PyStringType = Runtime.PyObject_Type(op); Runtime.Decref(op); @@ -104,6 +136,12 @@ internal static void Initialize() { PyUnicodeType = Runtime.PyObject_Type(op); Runtime.Decref(op); +#if (PYTHON32 || PYTHON33 || PYTHON34) + op = Runtime.PyBytes_FromString("bytes"); + PyBytesType = Runtime.PyObject_Type(op); + Runtime.Decref(op); +#endif + op = Runtime.PyTuple_New(0); PyTupleType = Runtime.PyObject_Type(op); Runtime.Decref(op); @@ -128,8 +166,13 @@ internal static void Initialize() { PyFloatType = Runtime.PyObject_Type(op); Runtime.Decref(op); +#if (PYTHON32 || PYTHON33 || PYTHON34) + PyClassType = IntPtr.Zero; + PyInstanceType = IntPtr.Zero; +#else IntPtr s = Runtime.PyString_FromString("_temp"); IntPtr d = Runtime.PyDict_New(); + IntPtr c = Runtime.PyClass_New(IntPtr.Zero, d, s); PyClassType = Runtime.PyObject_Type(c); @@ -140,6 +183,7 @@ internal static void Initialize() { Runtime.Decref(i); Runtime.Decref(c); Runtime.Decref(d); +#endif Error = new IntPtr(-1); @@ -147,7 +191,7 @@ internal static void Initialize() { // of the Python runtime that do not allow new-style classes to // be used as exceptions (Python versions 2.4 and lower). -#if (PYTHON25 || PYTHON26 || PYTHON27) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) wrap_exceptions = false; #else IntPtr m = PyImport_ImportModule("exceptions"); @@ -207,6 +251,13 @@ internal static void Shutdown() { internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; +#if (PYTHON32 || PYTHON33 || PYTHON34) + internal static IntPtr PyBytesType; + internal static IntPtr PyNotImplemented; + internal static int Py_EQ = 2; + internal static int Py_NE = 3; +#endif + internal static IntPtr PyTrue; internal static IntPtr PyFalse; internal static IntPtr PyNone; @@ -750,10 +801,18 @@ internal unsafe static extern IntPtr internal unsafe static extern IntPtr PyObject_Str(IntPtr pointer); +#if (PYTHON32 || PYTHON33 || PYTHON34) [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint="PyObject_Str", ExactSpelling = true, CharSet = CharSet.Ansi)] internal unsafe static extern IntPtr PyObject_Unicode(IntPtr pointer); +#else + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyObject_Unicode(IntPtr pointer); +#endif [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] @@ -795,14 +854,8 @@ internal static bool PyBool_Check(IntPtr ob) { return PyObject_TypeCheck(ob, Runtime.PyBoolType); } - - - [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, - ExactSpelling=true, CharSet=CharSet.Ansi)] - private unsafe static extern IntPtr - PyInt_FromLong(IntPtr value); - - internal static IntPtr PyInt_FromInt32(int value) { + internal static IntPtr PyInt_FromInt32(int value) + { IntPtr v = new IntPtr(value); return PyInt_FromLong(v); } @@ -812,22 +865,51 @@ internal static IntPtr PyInt_FromInt64(long value) { return PyInt_FromLong(v); } +#if (PYTHON32 || PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromLong", + ExactSpelling = true, CharSet = CharSet.Ansi)] + private unsafe static extern IntPtr + PyInt_FromLong(IntPtr value); [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + EntryPoint = "PyLong_AsLong", ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern int PyInt_AsLong(IntPtr value); [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + EntryPoint = "PyLong_FromString", ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + EntryPoint = "PyLong_GetMax", ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern int PyInt_GetMax(); +#else // Python 2 + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + private unsafe static extern IntPtr + PyInt_FromLong(IntPtr value); + + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern int + PyInt_AsLong(IntPtr value); + + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern IntPtr + PyInt_FromString(string value, IntPtr end, int radix); + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern int + PyInt_GetMax(); +#endif internal static bool PyLong_Check(IntPtr ob) { return PyObject_TYPE(ob) == Runtime.PyLongType; @@ -1001,6 +1083,57 @@ internal static IntPtr PyString_FromString(string value) { return PyString_FromStringAndSize(value, value.Length); } +#if (PYTHON32 || PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern IntPtr + PyBytes_FromString(string op); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern int + PyBytes_Size(IntPtr op); + + internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { + return ob + BytesOffset.ob_sval; + } + + internal static IntPtr PyString_FromStringAndSize(string value, int length) + { + // copy the string into an unmanaged UTF-8 buffer + int len = Encoding.UTF8.GetByteCount(value); + byte[] buffer = new byte[len + 1]; + Encoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0); + IntPtr nativeUtf8 = Marshal.AllocHGlobal(buffer.Length); + try { + Marshal.Copy(buffer, 0, nativeUtf8, buffer.Length); + return PyUnicode_FromStringAndSize(nativeUtf8, length); + } + finally { + Marshal.FreeHGlobal(nativeUtf8); + } + } + +#if (PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromStringAndSize(IntPtr value, int size); +#elif (UCS2) + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyUnicodeUCS2_FromStringAndSize", + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromStringAndSize(IntPtr value, int size); +#else + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyUnicodeUCS4_FromStringAndSize", + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyUnicode_FromStringAndSize(IntPtr value, int size); +#endif + +#else // Python2x [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern IntPtr @@ -1016,12 +1149,57 @@ internal unsafe static extern IntPtr ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern int PyString_Size(IntPtr pointer); +#endif internal static bool PyUnicode_Check(IntPtr ob) { return PyObject_TYPE(ob) == Runtime.PyUnicodeType; } #if (UCS2) +#if (PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromObject(IntPtr ob); + + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint="PyUnicode_FromKindAndData", + ExactSpelling=true, + CharSet=CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromKindAndString(int kind, string s, int size); + + internal static IntPtr PyUnicode_FromUnicode(string s, int size) { + return PyUnicode_FromKindAndString(2, s, size); + } + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern int + PyUnicode_GetSize(IntPtr ob); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern char * + PyUnicode_AsUnicode(IntPtr ob); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyUnicode_AsUnicode", + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_AS_UNICODE(IntPtr op); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromOrdinal(int c); + +#else [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, EntryPoint="PyUnicodeUCS2_FromObject", ExactSpelling=true, CharSet=CharSet.Unicode)] @@ -1063,6 +1241,7 @@ internal unsafe static extern IntPtr ExactSpelling=true, CharSet=CharSet.Unicode)] internal unsafe static extern IntPtr PyUnicode_FromOrdinal(int c); +#endif internal static IntPtr PyUnicode_FromString(string s) { @@ -1073,6 +1252,8 @@ internal unsafe static string GetManagedString(IntPtr op) { IntPtr type = PyObject_TYPE(op); +// Python 3 strings are all unicode +#if !(PYTHON32 || PYTHON33 || PYTHON34) if (type == Runtime.PyStringType) { return Marshal.PtrToStringAnsi( @@ -1080,6 +1261,7 @@ internal unsafe static string GetManagedString(IntPtr op) Runtime.PyString_Size(op) ); } +#endif if (type == Runtime.PyUnicodeType) { @@ -1093,6 +1275,52 @@ internal unsafe static string GetManagedString(IntPtr op) #endif #if (UCS4) +#if (PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromObject(IntPtr ob); + + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyUnicode_FromKindAndData", + ExactSpelling = true)] + internal unsafe static extern IntPtr + PyUnicode_FromKindAndString(int kind, + [MarshalAs (UnmanagedType.CustomMarshaler, + MarshalTypeRef=typeof(Utf32Marshaler))] string s, + int size); + + internal static IntPtr PyUnicode_FromUnicode(string s, int size) { + return PyUnicode_FromKindAndString(4, s, size); + } + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern int + PyUnicode_GetSize(IntPtr ob); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true)] + internal unsafe static extern IntPtr + PyUnicode_AsUnicode(IntPtr ob); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyUnicode_AsUnicode", + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_AS_UNICODE(IntPtr op); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Unicode)] + internal unsafe static extern IntPtr + PyUnicode_FromOrdinal(int c); + +#else [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicodeUCS4_FromObject", ExactSpelling = true, CharSet = CharSet.Unicode)] @@ -1138,6 +1366,8 @@ internal unsafe static extern IntPtr internal unsafe static extern IntPtr PyUnicode_FromOrdinal(int c); +#endif + internal static IntPtr PyUnicode_FromString(string s) { return PyUnicode_FromUnicode(s, (s.Length)); @@ -1147,6 +1377,8 @@ internal unsafe static string GetManagedString(IntPtr op) { IntPtr type = PyObject_TYPE(op); +// Python 3 strings are all unicode +#if !(PYTHON32 || PYTHON33 || PYTHON34) if (type == Runtime.PyStringType) { return Marshal.PtrToStringAnsi( @@ -1154,6 +1386,7 @@ internal unsafe static string GetManagedString(IntPtr op) Runtime.PyString_Size(op) ); } +#endif if (type == Runtime.PyUnicodeType) { @@ -1365,6 +1598,11 @@ internal unsafe static extern IntPtr // Python module API //==================================================================== + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern IntPtr + PyModule_New(string name); + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern string @@ -1380,6 +1618,13 @@ internal unsafe static extern IntPtr internal unsafe static extern string PyModule_GetFilename(IntPtr module); +#if (PYTHON32 || PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern IntPtr + PyModule_Create2(IntPtr module, int apiver); +#endif + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern IntPtr diff --git a/pythonnet/src/runtime/typemanager.cs b/pythonnet/src/runtime/typemanager.cs index 41b845737..e3ad807d1 100644 --- a/pythonnet/src/runtime/typemanager.cs +++ b/pythonnet/src/runtime/typemanager.cs @@ -26,11 +26,9 @@ internal class TypeManager { static BindingFlags tbFlags; static Dictionary cache; - static int obSize; static TypeManager() { tbFlags = BindingFlags.Public | BindingFlags.Static; - obSize = 5 * IntPtr.Size; cache = new Dictionary(128); } @@ -86,11 +84,12 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) { internal static IntPtr CreateType(Type impl) { IntPtr type = AllocateTypeObject(impl.Name); + int ob_size = ObjectOffset.Size(type); // Set tp_basicsize to the size of our managed instance objects. - Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - IntPtr offset = (IntPtr)ObjectOffset.ob_dict; + IntPtr offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); InitializeSlots(type, impl); @@ -124,11 +123,21 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) { } IntPtr base_ = IntPtr.Zero; + int ob_size = ObjectOffset.Size(Runtime.PyTypeType); + int tp_dictoffset = ObjectOffset.DictOffset(Runtime.PyTypeType); + // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* // subclass BaseException (or better Exception). -#if (PYTHON25 || PYTHON26 || PYTHON27) - if (clrType == typeof(System.Exception)) { +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) + if (typeof(System.Exception).IsAssignableFrom(clrType)) + { + ob_size = ObjectOffset.Size(Exceptions.BaseException); + tp_dictoffset = ObjectOffset.DictOffset(Exceptions.BaseException); + } + + if (clrType == typeof(System.Exception)) + { base_ = Exceptions.Exception; Runtime.Incref(base_); } else @@ -143,11 +152,9 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) { Marshal.WriteIntPtr(type,TypeOffset.ob_type,Runtime.PyCLRMetaType); Runtime.Incref(Runtime.PyCLRMetaType); - Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - - IntPtr offset = (IntPtr)ObjectOffset.ob_dict; - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); + Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); InitializeSlots(type, impl.GetType()); @@ -201,10 +208,11 @@ internal static IntPtr CreateSubType(IntPtr args) { Marshal.WriteIntPtr(type,TypeOffset.ob_type,Runtime.PyCLRMetaType); Runtime.Incref(Runtime.PyCLRMetaType); - Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)obSize); + int size = ObjectOffset.Size(type); + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - IntPtr offset = (IntPtr)ObjectOffset.ob_dict; + IntPtr offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); IntPtr dc = Runtime.PyDict_Copy(dict); @@ -338,12 +346,24 @@ internal static IntPtr AllocateTypeObject(string name) { // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. - +#if (PYTHON32 || PYTHON33 || PYTHON34) + // For python3 we leak two objects. One for the ascii representation + // required for tp_name, and another for the unicode representation + // for ht_name. + IntPtr temp = Runtime.PyBytes_FromString(name); + IntPtr raw = Runtime.PyBytes_AS_STRING(temp); + temp = Runtime.PyUnicode_FromString(name); +#else IntPtr temp = Runtime.PyString_FromString(name); IntPtr raw = Runtime.PyString_AS_STRING(temp); +#endif Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); +#if (PYTHON33 || PYTHON34) + Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); +#endif + long ptr = type.ToInt64(); // 64-bit safe temp = new IntPtr(ptr + TypeOffset.nb_add); @@ -355,8 +375,13 @@ internal static IntPtr AllocateTypeObject(string name) { temp = new IntPtr(ptr + TypeOffset.mp_length); Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); +#if (PYTHON32 || PYTHON33 || PYTHON34) + temp = new IntPtr(ptr + TypeOffset.bf_getbuffer); + Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); +#else temp = new IntPtr(ptr + TypeOffset.bf_getreadbuffer); Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); +#endif return type; } From acb1fac54ddbcccbe2116ee3ed912c5013714f09 Mon Sep 17 00:00:00 2001 From: David Lechner Date: Wed, 5 Jun 2013 11:54:24 -0500 Subject: [PATCH 02/77] Fix crash because Python 3 C API does not have PyNumber_Int --- pythonnet/src/runtime/runtime.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 7cf5b19b4..2efa2b0ee 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -824,10 +824,18 @@ internal unsafe static extern IntPtr // Python number API //==================================================================== +#if (PYTHON32 || PYTHON33 || PYTHON34) [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + EntryPoint = "PyNumber_Long", ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern IntPtr PyNumber_Int(IntPtr ob); +#else // Python 2 + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Int(IntPtr ob); +#endif [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] From 3f2f6774d173a87f0d2354a7ec932114b10a370a Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Tue, 2 Jul 2013 09:57:40 +0100 Subject: [PATCH 03/77] Convert IEnumerables to python lists. --- pythonnet/src/runtime/converter.cs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pythonnet/src/runtime/converter.cs b/pythonnet/src/runtime/converter.cs index 063a57294..45826d70c 100644 --- a/pythonnet/src/runtime/converter.cs +++ b/pythonnet/src/runtime/converter.cs @@ -12,6 +12,7 @@ using System.Runtime.InteropServices; using System.Globalization; using System.Security; +using System.Collections; namespace Python.Runtime { @@ -91,6 +92,16 @@ internal static IntPtr ToPython(Object value, Type type) { return result; } + if (value is IEnumerable) + { + var resultlist = new PyList(); + foreach (object o in (IEnumerable)value) + { + resultlist.Append(new PyObject(ToPython(o, o.GetType()))); + } + return resultlist.Handle; + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. @@ -738,10 +749,13 @@ static bool ToEnum(IntPtr value, Type obType, out Object result, return false; } - - - } - + public static class ConverterExtension + { + public static PyObject ToPython(this object o) + { + return new PyObject(Converter.ToPython(o, o.GetType())); + } + } } From f62eb634ff02f9ea2001e3af571c5b6897666c6c Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Tue, 2 Jul 2013 10:03:53 +0100 Subject: [PATCH 04/77] Update c# project --- pythonnet/src/runtime/Python.Runtime.csproj | 34 +++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pythonnet/src/runtime/Python.Runtime.csproj b/pythonnet/src/runtime/Python.Runtime.csproj index be7ca1174..3b0eec8a1 100644 --- a/pythonnet/src/runtime/Python.Runtime.csproj +++ b/pythonnet/src/runtime/Python.Runtime.csproj @@ -15,20 +15,20 @@ full true .\bin\Debug\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true pdbonly true .\bin\Release\ - TRACE;PYTHON27, UCS2 + TRACE;PYTHON27, UCS2 true true bin\EmbeddingTest\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -37,7 +37,7 @@ true bin\UnitTests\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -52,7 +52,7 @@ true bin\x86\Debug\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -60,17 +60,18 @@ bin\x86\Release\ - PYTHON27, UCS4 + PYTHON27, UCS4 true true pdbonly x86 false true + PYTHON27,UCS2 true - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -82,7 +83,7 @@ true bin\x86\UnitTests\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -91,7 +92,7 @@ true bin\DebugMono_x86\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -103,7 +104,7 @@ true bin\x86\DebugMono_x86\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -112,7 +113,7 @@ true bin\x64\Debug\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full @@ -121,7 +122,7 @@ bin\x64\Release\ - PYTHON32, UCS2 + PYTHON32, UCS2 true true pdbonly @@ -132,7 +133,7 @@ true bin\x64\EmbeddingTest\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -144,7 +145,7 @@ true bin\x64\UnitTests\ - TRACE;DEBUG;PYTHON27,UCS2 + TRACE;DEBUG;PYTHON27,UCS2 true true full @@ -156,13 +157,14 @@ true bin\x64\DebugMono_x86\ - TRACE;DEBUG;PYTHON27,UCS4 + TRACE;DEBUG;PYTHON27,UCS4 true true full x64 + @@ -244,4 +246,4 @@ copy "$(TargetDir)clr.pyd" "$(SolutionDir)" del "$(TargetDir)clr.pyd" - + \ No newline at end of file From 25481afdc18b75c70133c9cb886fed90d46a49fb Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Tue, 2 Jul 2013 10:05:00 +0100 Subject: [PATCH 05/77] Add more of the PyNumber API to support mathematical operations. --- pythonnet/src/runtime/runtime.cs | 133 ++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 2 deletions(-) diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 2efa2b0ee..f6c6ac37b 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -776,11 +776,16 @@ internal unsafe static extern int internal unsafe static extern int PyCallable_Check(IntPtr pointer); - [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, - ExactSpelling=true, CharSet=CharSet.Ansi)] + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] internal unsafe static extern int PyObject_IsTrue(IntPtr pointer); + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern int + PyObject_Not(IntPtr pointer); + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern int @@ -993,6 +998,130 @@ internal unsafe static extern IntPtr internal unsafe static extern double PyFloat_AsDouble(IntPtr ob); + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Add(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Subtract(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Multiply(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Divide(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_And(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Xor(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Or(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Lshift(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Rshift(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Power(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Remainder(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceAdd(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceSubtract(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceMultiply(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceDivide(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceAnd(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceXor(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceOr(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceLshift(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceRshift(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlacePower(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_InPlaceRemainder(IntPtr o1, IntPtr o2); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Negative(IntPtr o1); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Positive(IntPtr o1); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern IntPtr + PyNumber_Invert(IntPtr o1); //==================================================================== // Python sequence API From e6eac73f77dc0821cf9a837fea2233f92f88f761 Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Tue, 2 Jul 2013 10:07:22 +0100 Subject: [PATCH 06/77] Add convenience functions to new Py class. Supports "using (Py.GIL()) {}" blocks to setup the python interpreter and take/release the GIL, and Py.kw("key1", value1, "key2", value2, ...) to add keyword arguments. --- pythonnet/src/runtime/pythonengine.cs | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/pythonnet/src/runtime/pythonengine.cs b/pythonnet/src/runtime/pythonengine.cs index 460928f98..59d5a72bb 100644 --- a/pythonnet/src/runtime/pythonengine.cs +++ b/pythonnet/src/runtime/pythonengine.cs @@ -352,10 +352,61 @@ public static PyObject RunString(string code) { } return new PyObject(result); } + } + public static class Py + { + public static GILState GIL() + { + if (!PythonEngine.IsInitialized) + PythonEngine.Initialize(); + return new GILState(); + } - } + public class GILState : IDisposable + { + private IntPtr state; + internal GILState() + { + state = PythonEngine.AcquireLock(); + } + public void Dispose() + { + PythonEngine.ReleaseLock(state); + GC.SuppressFinalize(this); + } + ~GILState() + { + Dispose(); + } + } + public class KeywordArguments : PyDict { } + + public static KeywordArguments kw(params object[] kv) + { + var dict = new KeywordArguments(); + if (kv.Length % 2 != 0) + throw new ArgumentException("Must have an equal number of keys and values"); + for (int i = 0; i < kv.Length; i += 2) + { + IntPtr value; + if (kv[i + 1] is PyObject) + value = ((PyObject)kv[i + 1]).Handle; + else + value = Converter.ToPython(kv[i + 1], kv[i + 1].GetType()); + if (Runtime.PyDict_SetItemString(dict.Handle, (string)kv[i], value) != 0) + throw new ArgumentException(string.Format("Cannot add key '{0}' to dictionary.", (string)kv[i])); + if (!(kv[i + 1] is PyObject)) + Runtime.Decref(value); + } + return dict; + } + public static PyObject Import(string name) + { + return PythonEngine.ImportModule(name); + } + } } From 0bd8c47f9b7559015c0a366db31ae4185c802aa2 Mon Sep 17 00:00:00 2001 From: Patrick Stewart Date: Tue, 2 Jul 2013 10:11:15 +0100 Subject: [PATCH 07/77] Use the c# dynamic functionality for python objects. This means that python functions can be called directly, members can be accessed as normal (a.b) and mathematical operations work, and run in python (a = b*c). --- pythonnet/src/runtime/pyobject.cs | 224 +++++++++++++++++++++++++++++- 1 file changed, 220 insertions(+), 4 deletions(-) diff --git a/pythonnet/src/runtime/pyobject.cs b/pythonnet/src/runtime/pyobject.cs index 099b9fdf4..133b1d0a3 100644 --- a/pythonnet/src/runtime/pyobject.cs +++ b/pythonnet/src/runtime/pyobject.cs @@ -8,6 +8,8 @@ // ========================================================================== using System; +using System.Dynamic; +using System.Linq.Expressions; namespace Python.Runtime { @@ -17,7 +19,7 @@ namespace Python.Runtime { /// http://www.python.org/doc/current/api/object.html for details. /// - public class PyObject : IDisposable { + public class PyObject : DynamicObject, IDisposable { protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; @@ -95,7 +97,7 @@ public static PyObject FromManagedObject(object ob) { public object AsManagedObject(Type t) { Object result; - if (!Converter.ToManaged(this.Handle, t, out result, false)) { + if (!Converter.ToManaged(this.obj, t, out result, false)) { throw new InvalidCastException("cannot convert object to target type"); } return result; @@ -609,7 +611,7 @@ public PyObject Invoke(PyTuple args) { public PyObject Invoke(PyObject[] args, PyDict kw) { PyTuple t = new PyTuple(args); - IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw.obj); + IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw != null ? kw.obj : IntPtr.Zero); t.Dispose(); if (r == IntPtr.Zero) { throw new PythonException(); @@ -628,7 +630,7 @@ public PyObject Invoke(PyObject[] args, PyDict kw) { /// public PyObject Invoke(PyTuple args, PyDict kw) { - IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw.obj); + IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw != null ? kw.obj : IntPtr.Zero); if (r == IntPtr.Zero) { throw new PythonException(); } @@ -862,8 +864,222 @@ public override int GetHashCode() { return Runtime.PyObject_Hash(obj).ToInt32(); } + public override bool TryGetMember(GetMemberBinder binder, out object result) + { + if (this.HasAttr(binder.Name)) + { + result = this.GetAttr(binder.Name); + return true; + } + else + return base.TryGetMember(binder, out result); + } + + public override bool TrySetMember(SetMemberBinder binder, object value) + { + if (this.HasAttr(binder.Name)) + { + this.SetAttr(binder.Name, (PyObject)value); + return true; + } + else + return base.TrySetMember(binder, value); + } + + private void GetArgs(object[] inargs, out PyTuple args, out PyDict kwargs) + { + int arg_count; + for (arg_count = 0; arg_count < inargs.Length && !(inargs[arg_count] is Py.KeywordArguments); ++arg_count); + IntPtr argtuple = Runtime.PyTuple_New(arg_count); + for (int i = 0; i < arg_count; i++) + { + IntPtr ptr; + if (inargs[i] is PyObject) + { + ptr = ((PyObject)inargs[i]).Handle; + Runtime.Incref(ptr); + } + else + { + ptr = Converter.ToPython(inargs[i], inargs[i].GetType()); + } + if (Runtime.PyTuple_SetItem(argtuple, i, ptr) < 0) + throw new PythonException(); + } + args = new PyTuple(argtuple); + kwargs = null; + for (int i = arg_count; i < inargs.Length; i++) + { + if (!(inargs[i] is Py.KeywordArguments)) + throw new ArgumentException("Keyword arguments must come after normal arguments."); + if (kwargs == null) + kwargs = (Py.KeywordArguments)inargs[i]; + else + kwargs.Update((Py.KeywordArguments)inargs[i]); + } + } + public override bool TryInvokeMember(InvokeMemberBinder binder, object[] args, out object result) + { + if (this.HasAttr(binder.Name) && this.GetAttr(binder.Name).IsCallable()) + { + PyTuple pyargs; + PyDict kwargs; + GetArgs(args, out pyargs, out kwargs); + result = InvokeMethod(binder.Name, pyargs, kwargs); + return true; + } + else + return base.TryInvokeMember(binder, args, out result); } + public override bool TryInvoke(InvokeBinder binder, object[] args, out object result) + { + if (this.IsCallable()) + { + PyTuple pyargs; + PyDict kwargs; + GetArgs(args, out pyargs, out kwargs); + result = Invoke(pyargs, kwargs); + return true; + } + else + return base.TryInvoke(binder, args, out result); + } + + public override bool TryConvert(ConvertBinder binder, out object result) + { + return Converter.ToManaged(this.obj, binder.Type, out result, false); + } + public override bool TryBinaryOperation(BinaryOperationBinder binder, Object arg, out Object result) { + IntPtr res; + if (!(arg is PyObject)) + arg = arg.ToPython(); + + switch (binder.Operation) + { + case ExpressionType.Add: + res = Runtime.PyNumber_Add(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.AddAssign: + res = Runtime.PyNumber_InPlaceAdd(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Subtract: + res = Runtime.PyNumber_Subtract(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.SubtractAssign: + res = Runtime.PyNumber_InPlaceSubtract(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Multiply: + res = Runtime.PyNumber_Multiply(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.MultiplyAssign: + res = Runtime.PyNumber_InPlaceMultiply(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Divide: + res = Runtime.PyNumber_Divide(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.DivideAssign: + res = Runtime.PyNumber_InPlaceDivide(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.And: + res = Runtime.PyNumber_And(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.AndAssign: + res = Runtime.PyNumber_InPlaceAnd(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.ExclusiveOr: + res = Runtime.PyNumber_Xor(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.ExclusiveOrAssign: + res = Runtime.PyNumber_InPlaceXor(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.GreaterThan: + result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) > 0; + return true; + case ExpressionType.GreaterThanOrEqual: + result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) >= 0; + return true; + case ExpressionType.LeftShift: + res = Runtime.PyNumber_Lshift(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.LeftShiftAssign: + res = Runtime.PyNumber_InPlaceLshift(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.LessThan: + result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) < 0; + return true; + case ExpressionType.LessThanOrEqual: + result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) <= 0; + return true; + case ExpressionType.Modulo: + res = Runtime.PyNumber_Remainder(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.ModuloAssign: + res = Runtime.PyNumber_InPlaceRemainder(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.NotEqual: + result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) != 0; + return true; + case ExpressionType.Or: + res = Runtime.PyNumber_Or(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.OrAssign: + res = Runtime.PyNumber_InPlaceOr(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Power: + res = Runtime.PyNumber_Power(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.RightShift: + res = Runtime.PyNumber_Rshift(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.RightShiftAssign: + res = Runtime.PyNumber_InPlaceRshift(this.obj, ((PyObject)arg).obj); + break; + default: + result = null; + return false; + } + result = new PyObject(res); + return true; + } + + public override bool TryUnaryOperation(UnaryOperationBinder binder, out Object result) + { + int r; + IntPtr res; + switch (binder.Operation) + { + case ExpressionType.Negate: + res = Runtime.PyNumber_Negative(this.obj); + break; + case ExpressionType.UnaryPlus: + res = Runtime.PyNumber_Positive(this.obj); + break; + case ExpressionType.OnesComplement: + res = Runtime.PyNumber_Invert(this.obj); + break; + case ExpressionType.Not: + r = Runtime.PyObject_Not(this.obj); + result = r == 1; + return r != -1; + case ExpressionType.IsFalse: + r = Runtime.PyObject_IsTrue(this.obj); + result = r == 0; + return r != -1; + case ExpressionType.IsTrue: + r = Runtime.PyObject_IsTrue(this.obj); + result = r == 1; + return r != -1; + case ExpressionType.Decrement: + case ExpressionType.Increment: + default: + result = null; + return false; + } + result = new PyObject(res); + return true; + } + } } From eebe879fdf49e89f8bac52d0fc93eb86c8e8e463 Mon Sep 17 00:00:00 2001 From: patstew Date: Tue, 2 Jul 2013 11:40:58 +0100 Subject: [PATCH 08/77] Create README.md --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..39cc91440 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +pythonnet +========= + +This fork of http://sourceforge.net/projects/pythonnet/ allows easy calling of python functions from C#. + +All calls to python should be inside a "using (Py.GIL()) {/* Your code here */}" block. +Import python modules using dynamic mod = Py.Import("mod"), then you can call functions as normal, eg +mod.func(args). +Use mod.func(args, Py.kw("keywordargname", keywordargvalue)) to apply keyword arguments. +All python objects should be declared as 'dynamic' type. +Mathematical operations involving python and literal/managed types must have the python object first, eg np.pi*2 works, 2*np.pi doesn't. + +EG: + +static void Main(string[] args) +{ + using (Py.GIL()) { + dynamic np = Py.Import("numpy"); + dynamic sin = np.sin; + Console.WriteLine(np.cos(np.pi*2)); + Console.WriteLine(sin(5)); + Console.WriteLine(np.cos(5) + sin(5)); + dynamic a = np.array(new List { 1, 2, 3 }; + dynamic b = np.array(new List { 6, 5, 4 }, Py.kw("dtype", np.int32)); + Console.WriteLine(a.dtype); + Console.WriteLine(b.dtype); + Console.WriteLine(a * b); + Console.ReadKey(); + } +} + +which outputs: + +1.0 +-0.958924274663 +-0.6752620892 +float64 +int32 +[ 6. 10. 12.] From 2b1d8253b1200b739315ad5c503d468e67e652db Mon Sep 17 00:00:00 2001 From: patstew Date: Tue, 2 Jul 2013 11:49:50 +0100 Subject: [PATCH 09/77] Fix readme formatting. --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 39cc91440..23b181131 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,14 @@ pythonnet This fork of http://sourceforge.net/projects/pythonnet/ allows easy calling of python functions from C#. -All calls to python should be inside a "using (Py.GIL()) {/* Your code here */}" block. -Import python modules using dynamic mod = Py.Import("mod"), then you can call functions as normal, eg -mod.func(args). -Use mod.func(args, Py.kw("keywordargname", keywordargvalue)) to apply keyword arguments. -All python objects should be declared as 'dynamic' type. -Mathematical operations involving python and literal/managed types must have the python object first, eg np.pi*2 works, 2*np.pi doesn't. ++ All calls to python should be inside a "using (Py.GIL()) {/* Your code here */}" block. ++ Import python modules using dynamic mod = Py.Import("mod"), then you can call functions as normal, eg mod.func(args). ++ Use mod.func(args, Py.kw("keywordargname", keywordargvalue)) to apply keyword arguments. ++ All python objects should be declared as 'dynamic' type. ++ Mathematical operations involving python and literal/managed types must have the python object first, eg np.pi*2 works, 2*np.pi doesn't EG: - +```csharp static void Main(string[] args) { using (Py.GIL()) { @@ -28,12 +27,13 @@ static void Main(string[] args) Console.ReadKey(); } } - -which outputs: - -1.0 +``` +outputs: +``` +1.0 -0.958924274663 -0.6752620892 float64 int32 [ 6. 10. 12.] +``` From 11bc13a0b484dc353817f5ee7a21c79fbf096d1e Mon Sep 17 00:00:00 2001 From: patstew Date: Wed, 3 Jul 2013 15:30:20 +0100 Subject: [PATCH 10/77] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 23b181131..ca2225f62 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ static void Main(string[] args) dynamic sin = np.sin; Console.WriteLine(np.cos(np.pi*2)); Console.WriteLine(sin(5)); - Console.WriteLine(np.cos(5) + sin(5)); + double c = np.cos(5) + sin(5); + Console.WriteLine(c); dynamic a = np.array(new List { 1, 2, 3 }; dynamic b = np.array(new List { 6, 5, 4 }, Py.kw("dtype", np.int32)); Console.WriteLine(a.dtype); From 69d99332dcbb766740f7b65c0b68040a0e9a2a72 Mon Sep 17 00:00:00 2001 From: "Zane D. Purvis" Date: Fri, 13 Sep 2013 14:13:15 -0400 Subject: [PATCH 11/77] Add missing ) so sample code in README works --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca2225f62..515346d30 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ static void Main(string[] args) Console.WriteLine(sin(5)); double c = np.cos(5) + sin(5); Console.WriteLine(c); - dynamic a = np.array(new List { 1, 2, 3 }; + dynamic a = np.array(new List { 1, 2, 3 }); dynamic b = np.array(new List { 6, 5, 4 }, Py.kw("dtype", np.int32)); Console.WriteLine(a.dtype); Console.WriteLine(b.dtype); From 2b3868f32d71323b38c3461f90cab572a97759ef Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 14 Oct 2013 16:54:36 +0100 Subject: [PATCH 12/77] managed types can now be subclassed in python and override virtual methods --- pythonnet/src/runtime/Python.Runtime.csproj | 3 +- pythonnet/src/runtime/assemblymanager.cs | 2 +- pythonnet/src/runtime/classbase.cs | 5 + pythonnet/src/runtime/classderived.cs | 281 ++++++++++++++++++++ pythonnet/src/runtime/classmanager.cs | 7 +- pythonnet/src/runtime/genericutil.cs | 3 + pythonnet/src/runtime/metatype.cs | 42 +-- pythonnet/src/runtime/typemanager.cs | 78 +++--- pythonnet/src/testing/Python.Test.csproj | 1 + pythonnet/src/testing/subclasstest.cs | 44 +++ pythonnet/src/tests/test_subclass.py | 40 +++ 11 files changed, 421 insertions(+), 85 deletions(-) create mode 100644 pythonnet/src/runtime/classderived.cs create mode 100644 pythonnet/src/testing/subclasstest.cs create mode 100644 pythonnet/src/tests/test_subclass.py diff --git a/pythonnet/src/runtime/Python.Runtime.csproj b/pythonnet/src/runtime/Python.Runtime.csproj index 3b0eec8a1..cccb2919e 100644 --- a/pythonnet/src/runtime/Python.Runtime.csproj +++ b/pythonnet/src/runtime/Python.Runtime.csproj @@ -115,7 +115,7 @@ bin\x64\Debug\ TRACE;DEBUG;PYTHON27,UCS4 true - true + false full x64 false @@ -171,6 +171,7 @@ + diff --git a/pythonnet/src/runtime/assemblymanager.cs b/pythonnet/src/runtime/assemblymanager.cs index e723ca659..5ff2241d6 100644 --- a/pythonnet/src/runtime/assemblymanager.cs +++ b/pythonnet/src/runtime/assemblymanager.cs @@ -260,7 +260,7 @@ public static bool LoadImplicit(string name, out bool fromFile) { // be valid namespaces (to better match Python import semantics). //=================================================================== - static void ScanAssembly(Assembly assembly) { + internal static void ScanAssembly(Assembly assembly) { // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by diff --git a/pythonnet/src/runtime/classbase.cs b/pythonnet/src/runtime/classbase.cs index daad173c8..fb33817c2 100644 --- a/pythonnet/src/runtime/classbase.cs +++ b/pythonnet/src/runtime/classbase.cs @@ -82,6 +82,11 @@ public static IntPtr tp_richcompare(IntPtr ob, IntPtr other, int op) { CLRObject co1 = GetManagedObject(ob) as CLRObject; CLRObject co2 = GetManagedObject(other) as CLRObject; + if (null == co2) { + Runtime.Incref(pyfalse); + return pyfalse; + } + Object o1 = co1.inst; Object o2 = co2.inst; diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs new file mode 100644 index 000000000..4a3758726 --- /dev/null +++ b/pythonnet/src/runtime/classderived.cs @@ -0,0 +1,281 @@ +// ========================================================================== +// This software is subject to the provisions of the Zope Public License, +// Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +// FOR A PARTICULAR PURPOSE. +// ========================================================================== + +using System; +using System.Reflection; +using System.Reflection.Emit; +using System.Collections.Generic; +using System.Threading; +using System.Linq; + +namespace Python.Runtime +{ + + /// + /// Managed class that provides the implementation for reflected types. + /// Managed classes and value types are represented in Python by actual + /// Python type objects. Each of those type objects is associated with + /// an instance of ClassObject, which provides its implementation. + /// + + internal class ClassDerivedObject : ClassObject + { + static private Dictionary assemblyBuilders; + static private Dictionary, ModuleBuilder> moduleBuilders; + + static ClassDerivedObject() + { + assemblyBuilders = new Dictionary(); + moduleBuilders = new Dictionary, ModuleBuilder>(); + } + + internal ClassDerivedObject(Type tp) + : base(tp) + { + } + + //==================================================================== + // Implements __new__ for derived classes of reflected classes. + //==================================================================== + new public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) + { + // derived classes have a __pyobj__ field that points back to the python object + // (see Trampoline.InvokeMethod and CreateDerivedType) + IntPtr pyobj = ClassObject.tp_new(tp, args, kw); + CLRObject obj = (CLRObject)ManagedType.GetManagedObject(pyobj); + FieldInfo fi = obj.inst.GetType().GetField("__pyobj__"); + fi.SetValue(obj.inst, pyobj); + return pyobj; + } + + //==================================================================== + // Creates a new managed type derived from a base type with any virtual + // methods overriden to call out to python if the associated python + // object has overriden the method. + //==================================================================== + internal static Type CreateDerivedType(string name, + Type baseType, + string namespaceStr, + string assemblyName, + string moduleName="Python.Runtime.Dynamic.dll") + { + if (null != namespaceStr) + name = namespaceStr + "." + name; + + if (null == assemblyName) + assemblyName = Assembly.GetExecutingAssembly().FullName; + + ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); + TypeBuilder typeBuilder = moduleBuilder.DefineType(name, TypeAttributes.Public | TypeAttributes.Class); + typeBuilder.SetParent(baseType); + + // add a field for storing the python object pointer + FieldBuilder fb = typeBuilder.DefineField("__pyobj__", typeof(IntPtr), FieldAttributes.Public); + + // override any virtual methods + MethodInfo[] methods = baseType.GetMethods(); + List baseMethodNames = new List(); + foreach (MethodInfo method in methods) + { + if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | method.Attributes.HasFlag(MethodAttributes.Final)) + continue; + + ParameterInfo[] parameters = method.GetParameters(); + Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + + // create a method for calling the original method + string baseMethodName = "_" + baseType.Name + "__" + method.Name; + baseMethodNames.Add(baseMethodName); + MethodBuilder mb = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | MethodAttributes.Final | MethodAttributes.HideBySig, + method.ReturnType, + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator il = mb.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + for (int i = 0; i < parameters.Length; ++i) + il.Emit(OpCodes.Ldarg, i + 1); + il.Emit(OpCodes.Call, method); + il.Emit(OpCodes.Ret); + + // override the original method with a new one that dispatches to python + mb = typeBuilder.DefineMethod(method.Name, + MethodAttributes.Public | MethodAttributes.ReuseSlot | + MethodAttributes.Virtual | MethodAttributes.HideBySig, + method.CallingConvention, + method.ReturnType, + parameterTypes); + + il = mb.GetILGenerator(); + il.DeclareLocal(typeof(Object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, method.Name); + il.Emit(OpCodes.Ldstr, baseMethodName); + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(System.Object)); + il.Emit(OpCodes.Stloc_0); + for (int i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (parameterTypes[i].IsPrimitive) + il.Emit(OpCodes.Box, parameterTypes[i]); + il.Emit(OpCodes.Stelem, typeof(Object)); + } + il.Emit(OpCodes.Ldloc_0); + if (method.ReturnType == typeof(void)) + { + il.Emit(OpCodes.Call, typeof(Trampoline).GetMethod("InvokeMethodVoid")); + } + else + { + il.Emit(OpCodes.Call, typeof(Trampoline).GetMethod("InvokeMethod").MakeGenericMethod(method.ReturnType)); + } + il.Emit(OpCodes.Ret); + } + + Type type = typeBuilder.CreateType(); + + // scan the assembly so the newly added class can be imported + Assembly assembly = Assembly.GetAssembly(type); + AssemblyManager.ScanAssembly(assembly); + + return type; + } + + + private static ModuleBuilder GetModuleBuilder(string assemblyName, string moduleName) + { + // find or create a dynamic assembly and module + AppDomain domain = AppDomain.CurrentDomain; + ModuleBuilder moduleBuilder = null; + + if (moduleBuilders.ContainsKey(Tuple.Create(assemblyName, moduleName))) + { + moduleBuilder = moduleBuilders[Tuple.Create(assemblyName, moduleName)]; + } + else + { + AssemblyBuilder assemblyBuilder = null; + if (assemblyBuilders.ContainsKey(assemblyName)) + { + assemblyBuilder = assemblyBuilders[assemblyName]; + } + else + { + assemblyBuilder = domain.DefineDynamicAssembly(new AssemblyName(assemblyName), + AssemblyBuilderAccess.Run); + assemblyBuilders[assemblyName] = assemblyBuilder; + } + + moduleBuilder = assemblyBuilder.DefineDynamicModule(moduleName); + moduleBuilders[Tuple.Create(assemblyName, moduleName)] = moduleBuilder; + } + + return moduleBuilder; + } + + } + + // This has to be public as it's called from methods on dynamically built classes + // potentially in other assemblies + public class Trampoline + { + //==================================================================== + // This is the implementaion of the overriden methods in the derived + // type. It looks for a python method with the same name as the method + // on the managed base class and if it exists and isn't the managed + // method binding (ie it has been overriden in the derived python + // class) it calls it, otherwise it calls the base method. + //==================================================================== + public static T InvokeMethod(Object obj, string methodName, string origMethodName, Object[] args) + { + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + IntPtr ptr = (IntPtr)fi.GetValue(obj); + if (null != ptr) + { + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + PyObject pyobj = new PyObject(ptr); + PyObject method = pyobj.GetAttr(methodName, new PyObject(Runtime.PyNone)); + if (method.Handle != Runtime.PyNone) + { + // if the method hasn't been overriden then it will be a managed object + ManagedType managedMethod = ManagedType.GetManagedObject(method.Handle); + if (null == managedMethod) + { + PyObject[] pyargs = new PyObject[args.Length]; + for (int i = 0; i < args.Length; ++i) + { + pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + } + + PyObject py_result = method.Invoke(pyargs); + return (T)py_result.AsManagedObject(typeof(T)); + } + } + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + + return (T)obj.GetType().InvokeMember(origMethodName, + BindingFlags.InvokeMethod, + null, + obj, + args); + } + + public static void InvokeMethodVoid(Object obj, string methodName, string origMethodName, Object[] args) + { + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + IntPtr ptr = (IntPtr)fi.GetValue(obj); + if (null != ptr) + { + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + PyObject pyobj = new PyObject(ptr); + PyObject method = pyobj.GetAttr(methodName, new PyObject(Runtime.PyNone)); + if (method.Handle != Runtime.PyNone) + { + // if the method hasn't been overriden then it will be a managed object + ManagedType managedMethod = ManagedType.GetManagedObject(method.Handle); + if (null == managedMethod) + { + PyObject[] pyargs = new PyObject[args.Length]; + for (int i = 0; i < args.Length; ++i) + { + pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + } + + PyObject py_result = method.Invoke(pyargs); + return; + } + } + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + + obj.GetType().InvokeMember(origMethodName, + BindingFlags.InvokeMethod, + null, + obj, + args); + } + } +} diff --git a/pythonnet/src/runtime/classmanager.cs b/pythonnet/src/runtime/classmanager.cs index 2eb3cfd90..6bcd52ce5 100644 --- a/pythonnet/src/runtime/classmanager.cs +++ b/pythonnet/src/runtime/classmanager.cs @@ -105,7 +105,12 @@ private static ClassBase CreateClass(Type type) { impl = new ExceptionClassObject(type); } - else { + else if (null != type.GetField("__pyobj__")) { + impl = new ClassDerivedObject(type); + } + + else + { impl = new ClassObject(type); } diff --git a/pythonnet/src/runtime/genericutil.cs b/pythonnet/src/runtime/genericutil.cs index c3de0aa56..e646af098 100644 --- a/pythonnet/src/runtime/genericutil.cs +++ b/pythonnet/src/runtime/genericutil.cs @@ -37,6 +37,9 @@ static GenericUtil() { //==================================================================== internal static void Register(Type t) { + if (null == t.Namespace || null == t.Name) + return; + Dictionary> nsmap = null; mapping.TryGetValue(t.Namespace, out nsmap); if (nsmap == null) { diff --git a/pythonnet/src/runtime/metatype.cs b/pythonnet/src/runtime/metatype.cs index 305437c84..f2e611902 100644 --- a/pythonnet/src/runtime/metatype.cs +++ b/pythonnet/src/runtime/metatype.cs @@ -46,7 +46,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { return Exceptions.RaiseTypeError("invalid argument list"); } - //IntPtr name = Runtime.PyTuple_GetItem(args, 0); + IntPtr name = Runtime.PyTuple_GetItem(args, 0); IntPtr bases = Runtime.PyTuple_GetItem(args, 1); IntPtr dict = Runtime.PyTuple_GetItem(args, 2); @@ -88,45 +88,7 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { ); } - // hack for now... fix for 1.0 - //return TypeManager.CreateSubType(args); - - - // right way - - IntPtr func = Marshal.ReadIntPtr(Runtime.PyTypeType, - TypeOffset.tp_new); - IntPtr type = NativeCall.Call_3(func, tp, args, kw); - if (type == IntPtr.Zero) { - return IntPtr.Zero; - } - - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.BaseType; - flags |= TypeFlags.Subclass; - flags |= TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); - - TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); - - // Hmm - the standard subtype_traverse, clear look at ob_size to - // do things, so to allow gc to work correctly we need to move - // our hidden handle out of ob_size. Then, in theory we can - // comment this out and still not crash. - TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); - TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); - - - // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); - - //DebugUtil.DumpType(base_type); - //DebugUtil.DumpType(type); - - return type; + return TypeManager.CreateSubType(name, base_type, dict); } diff --git a/pythonnet/src/runtime/typemanager.cs b/pythonnet/src/runtime/typemanager.cs index e3ad807d1..eb0ef5025 100644 --- a/pythonnet/src/runtime/typemanager.cs +++ b/pythonnet/src/runtime/typemanager.cs @@ -195,59 +195,53 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) { return type; } - internal static IntPtr CreateSubType(IntPtr args) { - - IntPtr py_name = Runtime.PyTuple_GetItem(args, 0); - IntPtr bases = Runtime.PyTuple_GetItem(args, 1); - IntPtr dict = Runtime.PyTuple_GetItem(args, 2); - IntPtr base_ = Runtime.PyTuple_GetItem(bases, 0); - + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) + { + // Utility to create a subtype of a managed type with the ability for the + // a python subtype able to override the managed implementation string name = Runtime.GetManagedString(py_name); - IntPtr type = AllocateTypeObject(name); - Marshal.WriteIntPtr(type,TypeOffset.ob_type,Runtime.PyCLRMetaType); - Runtime.Incref(Runtime.PyCLRMetaType); + // the derived class can have class attributes __assembly__ and __module__ which + // control the name of the assembly and module the new type is created in. + object assembly = null; + object namespaceStr = null; - int size = ObjectOffset.Size(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)size); - Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - - IntPtr offset = (IntPtr)ObjectOffset.DictOffset(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - - IntPtr dc = Runtime.PyDict_Copy(dict); - Marshal.WriteIntPtr(type, TypeOffset.tp_dict, dc); - - Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); - Runtime.Incref(base_); - - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.BaseType; - flags |= TypeFlags.Subclass; - flags |= TypeFlags.HaveGC; - Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + PyObject assemblyKey = new PyObject(Converter.ToPython("__assembly__", typeof(String))); + if (0 != Runtime.PyMapping_HasKey(py_dict, assemblyKey.Handle)) + { + PyObject pyAssembly = new PyObject(Runtime.PyDict_GetItem(py_dict, assemblyKey.Handle)); + if (!Converter.ToManagedValue(pyAssembly.Handle, typeof(String), out assembly, false)) + throw new InvalidCastException("Couldn't convert __assembly__ value to string"); + } - CopySlot(base_, type, TypeOffset.tp_traverse); - CopySlot(base_, type, TypeOffset.tp_clear); - CopySlot(base_, type, TypeOffset.tp_is_gc); + PyObject namespaceKey = new PyObject(Converter.ToPythonImplicit("__namespace__")); + if (0 != Runtime.PyMapping_HasKey(py_dict, namespaceKey.Handle)) + { + PyObject pyNamespace = new PyObject(Runtime.PyDict_GetItem(py_dict, namespaceKey.Handle)); + if (!Converter.ToManagedValue(pyNamespace.Handle, typeof(String), out namespaceStr, false)) + throw new InvalidCastException("Couldn't convert __namespace__ value to string"); + } - Runtime.PyType_Ready(type); + // create the new managed type subclassing the base managed type + ClassObject baseClass = ManagedType.GetManagedObject(py_base_type) as ClassObject; + Type subType = ClassDerivedObject.CreateDerivedType(name, + baseClass.type, + (string)namespaceStr, + (string)assembly); - IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(tp_dict, "__module__", mod); + // create the new ManagedType and python type + ClassBase subClass = ClassManager.GetClass(subType); + IntPtr py_type = GetTypeHandle(subClass, subType); - // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); + // by default the class dict will have all the C# methods in it, but as this is a + // derived class we want the python overrides in there instead if they exist. + IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); + Runtime.PyDict_Update(cls_dict, py_dict); - return type; + return py_type; } - internal static IntPtr CreateMetaType(Type impl) { // The managed metatype is functionally little different than the diff --git a/pythonnet/src/testing/Python.Test.csproj b/pythonnet/src/testing/Python.Test.csproj index 97b7ade6f..11591e091 100644 --- a/pythonnet/src/testing/Python.Test.csproj +++ b/pythonnet/src/testing/Python.Test.csproj @@ -148,6 +148,7 @@ + diff --git a/pythonnet/src/testing/subclasstest.cs b/pythonnet/src/testing/subclasstest.cs new file mode 100644 index 000000000..6c2028dfb --- /dev/null +++ b/pythonnet/src/testing/subclasstest.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Python.Test +{ + public class SubClassTest + { + public SubClassTest() + { + } + + // simple test with no arguments + public virtual string foo() + { + return "foo"; + } + + // test passing objects and boxing primitives + public virtual string bar(string s, int i) + { + return s; + } + + // virtual methods that aren't overriden in python still work + public virtual string not_overriden() + { + return "not_overriden"; + } + + public static string test_foo(SubClassTest x) + { + // calls into python if foo is overriden + return x.foo(); + } + + public static string test_bar(SubClassTest x, string s, int i) + { + // calls into python if bar is overriden + return x.bar(s, i); + } + } +} diff --git a/pythonnet/src/tests/test_subclass.py b/pythonnet/src/tests/test_subclass.py new file mode 100644 index 000000000..6869c8bfa --- /dev/null +++ b/pythonnet/src/tests/test_subclass.py @@ -0,0 +1,40 @@ +# =========================================================================== +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# =========================================================================== +import clr +clr.AddReference('Python.Test') + +import sys, os, string, unittest, types +from Python.Test import SubClassTest + +class DerivedClass(SubClassTest): + def foo(self): + return "bar" + +class SubClassTests(unittest.TestCase): + """Test subclassing managed types""" + + def testSubClass(self): + """Test subclassing managed types""" + object = SubClassTest() + self.assertEqual(object.foo(), "foo") + self.assertEqual(SubClassTest.test(object), "foo") + + object = DerivedClass() + self.assertEqual(object.foo(), "bar") + self.assertEqual(SubClassTest.test(object), "bar") + +def test_suite(): + return unittest.makeSuite(SubClassTests) + +def main(): + for i in range(50): + unittest.TextTestRunner().run(test_suite()) + +if __name__ == '__main__': + main() From 7e0226ff51cc7f735056849ab6e7bc5498c8c178 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 16 Oct 2013 12:37:55 +0100 Subject: [PATCH 13/77] creating instances of classes that are managed types sub-classed in python from managed code now works --- pythonnet/src/runtime/classderived.cs | 327 +++++++++++++++++++++++--- pythonnet/src/runtime/converter.cs | 8 + pythonnet/src/runtime/runtime.cs | 11 + pythonnet/src/testing/subclasstest.cs | 23 ++ pythonnet/src/tests/test_subclass.py | 55 ++++- 5 files changed, 383 insertions(+), 41 deletions(-) diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index 4a3758726..4e0e8e0da 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -13,6 +13,8 @@ using System.Collections.Generic; using System.Threading; using System.Linq; +using System.Runtime.InteropServices; +using System.Threading.Tasks; namespace Python.Runtime { @@ -24,6 +26,11 @@ namespace Python.Runtime /// an instance of ClassObject, which provides its implementation. /// + // interface used to idenfity which C# types were dynamically created as python subclasses + public interface IPythonDerivedType + { + } + internal class ClassDerivedObject : ClassObject { static private Dictionary assemblyBuilders; @@ -45,13 +52,63 @@ internal ClassDerivedObject(Type tp) //==================================================================== new public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { - // derived classes have a __pyobj__ field that points back to the python object - // (see Trampoline.InvokeMethod and CreateDerivedType) - IntPtr pyobj = ClassObject.tp_new(tp, args, kw); - CLRObject obj = (CLRObject)ManagedType.GetManagedObject(pyobj); - FieldInfo fi = obj.inst.GetType().GetField("__pyobj__"); - fi.SetValue(obj.inst, pyobj); - return pyobj; + ClassDerivedObject cls = GetManagedObject(tp) as ClassDerivedObject; + + // call the managed constructor + Object obj = cls.binder.InvokeRaw(IntPtr.Zero, args, kw); + if (obj == null) + return IntPtr.Zero; + + // return the pointer to the python object + // (this indirectly calls ClassDerivedObject.ToPython) + return Converter.ToPython(obj, cls.GetType()); + } + + new public static void tp_dealloc(IntPtr ob) + { + CLRObject self = (CLRObject)GetManagedObject(ob); + + // don't let the python GC destroy this object + Runtime.PyObject_GC_UnTrack(self.pyHandle); + + // The python should now have a ref count of 0, but we don't actually want to + // deallocate the object until the C# object that references it is destroyed. + // So we don't call PyObject_GC_Del here and instead we set the python + // reference to a weak reference so that the C# object can be collected. + GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); + Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + self.gcHandle.Free(); + self.gcHandle = gc; + } + + // Called from Converter.ToPython for types that are python subclasses of managed types. + // The referenced python object is returned instead of a new wrapper. + internal static IntPtr ToPython(IPythonDerivedType obj) + { + // derived types have a __pyobj__ field that gets set to the python + // object in the overriden constructor + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + CLRObject self = (CLRObject)fi.GetValue(obj); + + Runtime.Incref(self.pyHandle); + + // when the C# constructor creates the python object it starts as a weak + // reference with a reference count of 0. Now we're passing this object + // to Python the reference count needs to be incremented and the reference + // needs to be replaced with a strong reference to stop the C# object being + // collected while Python still has a reference to it. + if (Runtime.Refcount(self.pyHandle) == 1) + { + GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); + Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); + self.gcHandle.Free(); + self.gcHandle = gc; + + // now the object has a python reference it's safe for the python GC to track it + Runtime.PyObject_GC_Track(self.pyHandle); + } + + return self.pyHandle; } //==================================================================== @@ -72,15 +129,70 @@ internal static Type CreateDerivedType(string name, assemblyName = Assembly.GetExecutingAssembly().FullName; ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); - TypeBuilder typeBuilder = moduleBuilder.DefineType(name, TypeAttributes.Public | TypeAttributes.Class); - typeBuilder.SetParent(baseType); + TypeBuilder typeBuilder = moduleBuilder.DefineType(name, + TypeAttributes.Public | TypeAttributes.Class, + baseType, + new Type[] { typeof(IPythonDerivedType) }); + + ILGenerator il; + MethodBuilder mb; // add a field for storing the python object pointer - FieldBuilder fb = typeBuilder.DefineField("__pyobj__", typeof(IntPtr), FieldAttributes.Public); + FieldBuilder fb = typeBuilder.DefineField("__pyobj__", typeof(CLRObject), FieldAttributes.Public); + + // override any constructors + ConstructorInfo[] constructors = baseType.GetConstructors(); + foreach (ConstructorInfo ctor in constructors) + { + ParameterInfo[] parameters = ctor.GetParameters(); + Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + + // create a method for calling the original constructor + string baseCtorName = "_" + baseType.Name + "__cinit__"; + mb = typeBuilder.DefineMethod(baseCtorName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + typeof(void), + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + il = mb.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + for (int i = 0; i < parameters.Length; ++i) + il.Emit(OpCodes.Ldarg, i + 1); + il.Emit(OpCodes.Call, ctor); + il.Emit(OpCodes.Ret); + + // override the original method with a new one that dispatches to python + ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig, + ctor.CallingConvention, + parameterTypes); + il = cb.GetILGenerator(); + il.DeclareLocal(typeof(Object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, baseCtorName); + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(System.Object)); + il.Emit(OpCodes.Stloc_0); + for (int i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (parameterTypes[i].IsValueType) + il.Emit(OpCodes.Box, parameterTypes[i]); + il.Emit(OpCodes.Stelem, typeof(Object)); + } + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeCtor")); + il.Emit(OpCodes.Ret); + } // override any virtual methods MethodInfo[] methods = baseType.GetMethods(); - List baseMethodNames = new List(); foreach (MethodInfo method in methods) { if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | method.Attributes.HasFlag(MethodAttributes.Final)) @@ -91,14 +203,15 @@ internal static Type CreateDerivedType(string name, // create a method for calling the original method string baseMethodName = "_" + baseType.Name + "__" + method.Name; - baseMethodNames.Add(baseMethodName); - MethodBuilder mb = typeBuilder.DefineMethod(baseMethodName, - MethodAttributes.Public | MethodAttributes.Final | MethodAttributes.HideBySig, + mb = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, method.ReturnType, parameterTypes); // emit the assembly for calling the original method using call instead of callvirt - ILGenerator il = mb.GetILGenerator(); + il = mb.GetILGenerator(); il.Emit(OpCodes.Ldarg_0); for (int i = 0; i < parameters.Length; ++i) il.Emit(OpCodes.Ldarg, i + 1); @@ -107,12 +220,13 @@ internal static Type CreateDerivedType(string name, // override the original method with a new one that dispatches to python mb = typeBuilder.DefineMethod(method.Name, - MethodAttributes.Public | MethodAttributes.ReuseSlot | - MethodAttributes.Virtual | MethodAttributes.HideBySig, + MethodAttributes.Public | + MethodAttributes.ReuseSlot | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, method.CallingConvention, method.ReturnType, parameterTypes); - il = mb.GetILGenerator(); il.DeclareLocal(typeof(Object[])); il.Emit(OpCodes.Ldarg_0); @@ -126,22 +240,37 @@ internal static Type CreateDerivedType(string name, il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Ldc_I4, i); il.Emit(OpCodes.Ldarg, i + 1); - if (parameterTypes[i].IsPrimitive) + if (parameterTypes[i].IsValueType) il.Emit(OpCodes.Box, parameterTypes[i]); il.Emit(OpCodes.Stelem, typeof(Object)); } il.Emit(OpCodes.Ldloc_0); if (method.ReturnType == typeof(void)) { - il.Emit(OpCodes.Call, typeof(Trampoline).GetMethod("InvokeMethodVoid")); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid")); } else { - il.Emit(OpCodes.Call, typeof(Trampoline).GetMethod("InvokeMethod").MakeGenericMethod(method.ReturnType)); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(method.ReturnType)); } il.Emit(OpCodes.Ret); } + // add the destructor so the python object created in the constructor gets destroyed + mb = typeBuilder.DefineMethod("Finalize", + MethodAttributes.Family | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, + CallingConventions.Standard, + typeof(void), + Type.EmptyTypes); + il = mb.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("Finalize")); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, baseType.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + il.Emit(OpCodes.Ret); + Type type = typeBuilder.CreateType(); // scan the assembly so the newly added class can be imported @@ -185,9 +314,16 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module } + // + // PythonDerivedType contains static methods used by the dynamically created + // derived type that allow it to call back into python from overriden virtual + // methods, and also handle the construction and destruction of the python + // object. + // // This has to be public as it's called from methods on dynamically built classes - // potentially in other assemblies - public class Trampoline + // potentially in other assemblies. + // + public class PythonDerivedType { //==================================================================== // This is the implementaion of the overriden methods in the derived @@ -196,17 +332,26 @@ public class Trampoline // method binding (ie it has been overriden in the derived python // class) it calls it, otherwise it calls the base method. //==================================================================== - public static T InvokeMethod(Object obj, string methodName, string origMethodName, Object[] args) + public static T InvokeMethod(IPythonDerivedType obj, string methodName, string origMethodName, Object[] args) { FieldInfo fi = obj.GetType().GetField("__pyobj__"); - IntPtr ptr = (IntPtr)fi.GetValue(obj); - if (null != ptr) + CLRObject self = (CLRObject)fi.GetValue(obj); + if (null != self) { + List disposeList = new List(); IntPtr gs = Runtime.PyGILState_Ensure(); try { - PyObject pyobj = new PyObject(ptr); - PyObject method = pyobj.GetAttr(methodName, new PyObject(Runtime.PyNone)); + Runtime.Incref(self.pyHandle); + PyObject pyself = new PyObject(self.pyHandle); + disposeList.Add(pyself); + + Runtime.Incref(Runtime.PyNone); + PyObject pynone = new PyObject(Runtime.PyNone); + disposeList.Add(pynone); + + PyObject method = pyself.GetAttr(methodName, pynone); + disposeList.Add(method); if (method.Handle != Runtime.PyNone) { // if the method hasn't been overriden then it will be a managed object @@ -217,15 +362,21 @@ public static T InvokeMethod(Object obj, string methodName, string origMethod for (int i = 0; i < args.Length; ++i) { pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + disposeList.Add(pyargs[i]); } PyObject py_result = method.Invoke(pyargs); + disposeList.Add(py_result); return (T)py_result.AsManagedObject(typeof(T)); } } } finally { + foreach (PyObject x in disposeList) { + if (x != null) + x.Dispose(); + } Runtime.PyGILState_Release(gs); } } @@ -237,17 +388,26 @@ public static T InvokeMethod(Object obj, string methodName, string origMethod args); } - public static void InvokeMethodVoid(Object obj, string methodName, string origMethodName, Object[] args) + public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName, Object[] args) { FieldInfo fi = obj.GetType().GetField("__pyobj__"); - IntPtr ptr = (IntPtr)fi.GetValue(obj); - if (null != ptr) + CLRObject self = (CLRObject)fi.GetValue(obj); + if (null != self) { + List disposeList = new List(); IntPtr gs = Runtime.PyGILState_Ensure(); try { - PyObject pyobj = new PyObject(ptr); - PyObject method = pyobj.GetAttr(methodName, new PyObject(Runtime.PyNone)); + Runtime.Incref(self.pyHandle); + PyObject pyself = new PyObject(self.pyHandle); + disposeList.Add(pyself); + + Runtime.Incref(Runtime.PyNone); + PyObject pynone = new PyObject(Runtime.PyNone); + disposeList.Add(pynone); + + PyObject method = pyself.GetAttr(methodName, pynone); + disposeList.Add(method); if (method.Handle != Runtime.PyNone) { // if the method hasn't been overriden then it will be a managed object @@ -258,15 +418,21 @@ public static void InvokeMethodVoid(Object obj, string methodName, string origMe for (int i = 0; i < args.Length; ++i) { pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + disposeList.Add(pyargs[i]); } PyObject py_result = method.Invoke(pyargs); + disposeList.Add(py_result); return; } } } finally { + foreach (PyObject x in disposeList) { + if (x != null) + x.Dispose(); + } Runtime.PyGILState_Release(gs); } } @@ -277,5 +443,100 @@ public static void InvokeMethodVoid(Object obj, string methodName, string origMe obj, args); } + + public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, Object[] args) + { + // call the base constructor + obj.GetType().InvokeMember(origCtorName, + BindingFlags.InvokeMethod, + null, + obj, + args); + + List disposeList = new List(); + CLRObject self = null; + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + // create the python object + IntPtr type = TypeManager.GetTypeHandle(obj.GetType()); + self = new CLRObject(obj, type); + + // set __pyobj__ to self and deref the python object which will allow this + // object to be collected. + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + fi.SetValue(obj, self); + + Runtime.Incref(self.pyHandle); + PyObject pyself = new PyObject(self.pyHandle); + disposeList.Add(pyself); + + Runtime.Incref(Runtime.PyNone); + PyObject pynone = new PyObject(Runtime.PyNone); + disposeList.Add(pynone); + + // call __init__ + PyObject init = pyself.GetAttr("__init__", pynone); + disposeList.Add(init); + if (init.Handle != Runtime.PyNone) + { + // if __init__ hasn't been overriden then it will be a managed object + ManagedType managedMethod = ManagedType.GetManagedObject(init.Handle); + if (null == managedMethod) + { + PyObject[] pyargs = new PyObject[args.Length]; + for (int i = 0; i < args.Length; ++i) + { + pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + disposeList.Add(pyargs[i]); + } + + disposeList.Add(init.Invoke(pyargs)); + } + } + } + finally + { + foreach (PyObject x in disposeList) { + if (x != null) + x.Dispose(); + } + + // Decrement the python object's reference count. + // This doesn't actually destroy the object, it just sets the reference to this object + // to be a weak reference and it will be destroyed when the C# object is destroyed. + if (null != self) + Runtime.Decref(self.pyHandle); + + Runtime.PyGILState_Release(gs); + } + } + + public static void Finalize(IPythonDerivedType obj) + { + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + CLRObject self = (CLRObject)fi.GetValue(obj); + + // delete the python object in an asnyc task as we may not be able to acquire + // the GIL immediately and we don't want to block the GC thread. + var t = Task.Factory.StartNew(() => { + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + // the C# object is being destroyed which must mean there are no more + // references to the Python object as well so now we can dealloc the + // python object. + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + if (dict != IntPtr.Zero) + Runtime.Decref(dict); + Runtime.PyObject_GC_Del(self.pyHandle); + self.gcHandle.Free(); + } + finally + { + Runtime.PyGILState_Release(gs); + } + }); + } } } diff --git a/pythonnet/src/runtime/converter.cs b/pythonnet/src/runtime/converter.cs index 45826d70c..43ac0a206 100644 --- a/pythonnet/src/runtime/converter.cs +++ b/pythonnet/src/runtime/converter.cs @@ -102,6 +102,14 @@ internal static IntPtr ToPython(Object value, Type type) { return resultlist.Handle; } + // it the type is a python subclass of a managed type then return the + // underying python object rather than construct a new wrapper object. + IPythonDerivedType pyderived = value as IPythonDerivedType; + if (null != pyderived) + { + return ClassDerivedObject.ToPython(pyderived); + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index f6c6ac37b..75db49c0e 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -409,6 +409,17 @@ internal unsafe static void Decref(IntPtr op) { #endif } + internal unsafe static long Refcount(IntPtr op) + { + void* p = (void*)op; + if ((void*)0 != p) + { + if (is32bit) { return (*(int*)p); } + else { return (*(long*)p); } + } + return 0; + } + #if (Py_DEBUG) // Py_IncRef and Py_DecRef are taking care of the extra payload // in Py_DEBUG builds of Python like _Py_RefTotal diff --git a/pythonnet/src/testing/subclasstest.cs b/pythonnet/src/testing/subclasstest.cs index 6c2028dfb..3927b104e 100644 --- a/pythonnet/src/testing/subclasstest.cs +++ b/pythonnet/src/testing/subclasstest.cs @@ -29,6 +29,11 @@ public virtual string not_overriden() return "not_overriden"; } + public virtual IList return_list() + { + return new List { "a", "b", "c" }; + } + public static string test_foo(SubClassTest x) { // calls into python if foo is overriden @@ -40,5 +45,23 @@ public static string test_bar(SubClassTest x, string s, int i) // calls into python if bar is overriden return x.bar(s, i); } + + public static IList test_list(SubClassTest x) + { + // calls into python if return_list is overriden + return x.return_list(); + } + + // test instances can be constructed in managed code + public static SubClassTest create_instance(Type t) + { + return (SubClassTest)t.GetConstructor(new Type[] {}).Invoke(new Object[] {}); + } + + // test instances pass through managed code unchanged + public static SubClassTest pass_through(SubClassTest s) + { + return s; + } } } diff --git a/pythonnet/src/tests/test_subclass.py b/pythonnet/src/tests/test_subclass.py index 6869c8bfa..bc29e9c0a 100644 --- a/pythonnet/src/tests/test_subclass.py +++ b/pythonnet/src/tests/test_subclass.py @@ -8,33 +8,72 @@ # =========================================================================== import clr clr.AddReference('Python.Test') +clr.AddReference('System') import sys, os, string, unittest, types from Python.Test import SubClassTest +from System.Collections.Generic import List class DerivedClass(SubClassTest): - def foo(self): - return "bar" + + def foo(self): + return "bar" + + def bar(self, x, i): + return "_".join([x] * i) + + def return_list(self): + l = List[str]() + l.Add("A") + l.Add("B") + l.Add("C") + return l class SubClassTests(unittest.TestCase): """Test subclassing managed types""" - def testSubClass(self): - """Test subclassing managed types""" + def testBaseClass(self): + """Test base class managed type""" object = SubClassTest() self.assertEqual(object.foo(), "foo") - self.assertEqual(SubClassTest.test(object), "foo") + self.assertEqual(SubClassTest.test_foo(object), "foo") + self.assertEqual(object.bar("bar", 2), "bar") + self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar") + self.assertEqual(object.not_overriden(), "not_overriden") + self.assertEqual(list(object.return_list()), ["a", "b", "c"]) + self.assertEqual(list(SubClassTest.test_list(object)), ["a", "b", "c"]) + def testDerivedClass(self): + """Test python class derived from managed type""" object = DerivedClass() self.assertEqual(object.foo(), "bar") - self.assertEqual(SubClassTest.test(object), "bar") + self.assertEqual(SubClassTest.test_foo(object), "bar") + self.assertEqual(object.bar("bar", 2), "bar_bar") + self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar_bar") + self.assertEqual(object.not_overriden(), "not_overriden") + self.assertEqual(list(object.return_list()), ["A", "B", "C"]) + self.assertEqual(list(SubClassTest.test_list(object)), ["A", "B", "C"]) + + x = SubClassTest.pass_through(object) + self.assertEqual(id(x), id(object)) + + def testCreateInstance(self): + """Test derived instances can be created from managed code""" + object = SubClassTest.create_instance(DerivedClass) + self.assertEqual(object.foo(), "bar") + self.assertEqual(SubClassTest.test_foo(object), "bar") + self.assertEqual(object.bar("bar", 2), "bar_bar") + self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar_bar") + self.assertEqual(object.not_overriden(), "not_overriden") + + x = SubClassTest.pass_through(object) + self.assertEqual(id(x), id(object)) def test_suite(): return unittest.makeSuite(SubClassTests) def main(): - for i in range(50): - unittest.TextTestRunner().run(test_suite()) + unittest.TextTestRunner().run(test_suite()) if __name__ == '__main__': main() From 95f123ce1d6830d1bee141c713bc623116289214 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 16 Oct 2013 15:29:44 +0100 Subject: [PATCH 14/77] - Ensure python threads are always initialized, even if the main interpreter hasn't initialized them, otherwise attempting to dereference objects in finalizers called by the concurrent GC will fail. - Dispose of PyObject instances after use in CreateSubType rather than wait for the GC - Don't try and dereference derived class instances after python has shutdown if called from the GC --- pythonnet/src/runtime/classderived.cs | 7 +++++- pythonnet/src/runtime/runtime.cs | 13 ++++++++-- pythonnet/src/runtime/typemanager.cs | 35 ++++++++++++++++++--------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index 4e0e8e0da..d1c6424fa 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -519,7 +519,12 @@ public static void Finalize(IPythonDerivedType obj) // delete the python object in an asnyc task as we may not be able to acquire // the GIL immediately and we don't want to block the GC thread. - var t = Task.Factory.StartNew(() => { + var t = Task.Factory.StartNew(() => + { + // If python's been terminated then there's nothing to do + if (0 == Runtime.Py_IsInitialized()) + return; + IntPtr gs = Runtime.PyGILState_Ensure(); try { diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 75db49c0e..d51299275 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -96,8 +96,12 @@ internal static void Initialize() { if (0 == Runtime.Py_IsInitialized()) { - Runtime.Py_Initialize(); - Runtime.PyEval_InitThreads(); + Runtime.Py_Initialize(); + } + + if (0 == Runtime.PyEval_ThreadsInitialized()) + { + Runtime.PyEval_InitThreads(); } #if (PYTHON32 || PYTHON33 || PYTHON34) @@ -516,6 +520,11 @@ public unsafe static extern int internal unsafe static extern void PyEval_InitThreads(); + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern int + PyEval_ThreadsInitialized(); + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern void diff --git a/pythonnet/src/runtime/typemanager.cs b/pythonnet/src/runtime/typemanager.cs index eb0ef5025..3d1fd9a4b 100644 --- a/pythonnet/src/runtime/typemanager.cs +++ b/pythonnet/src/runtime/typemanager.cs @@ -206,20 +206,33 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr object assembly = null; object namespaceStr = null; - PyObject assemblyKey = new PyObject(Converter.ToPython("__assembly__", typeof(String))); - if (0 != Runtime.PyMapping_HasKey(py_dict, assemblyKey.Handle)) + List disposeList = new List(); + try { - PyObject pyAssembly = new PyObject(Runtime.PyDict_GetItem(py_dict, assemblyKey.Handle)); - if (!Converter.ToManagedValue(pyAssembly.Handle, typeof(String), out assembly, false)) - throw new InvalidCastException("Couldn't convert __assembly__ value to string"); - } + PyObject assemblyKey = new PyObject(Converter.ToPython("__assembly__", typeof(String))); + disposeList.Add(assemblyKey); + if (0 != Runtime.PyMapping_HasKey(py_dict, assemblyKey.Handle)) + { + PyObject pyAssembly = new PyObject(Runtime.PyDict_GetItem(py_dict, assemblyKey.Handle)); + disposeList.Add(pyAssembly); + if (!Converter.ToManagedValue(pyAssembly.Handle, typeof(String), out assembly, false)) + throw new InvalidCastException("Couldn't convert __assembly__ value to string"); + } - PyObject namespaceKey = new PyObject(Converter.ToPythonImplicit("__namespace__")); - if (0 != Runtime.PyMapping_HasKey(py_dict, namespaceKey.Handle)) + PyObject namespaceKey = new PyObject(Converter.ToPythonImplicit("__namespace__")); + disposeList.Add(namespaceKey); + if (0 != Runtime.PyMapping_HasKey(py_dict, namespaceKey.Handle)) + { + PyObject pyNamespace = new PyObject(Runtime.PyDict_GetItem(py_dict, namespaceKey.Handle)); + disposeList.Add(pyNamespace); + if (!Converter.ToManagedValue(pyNamespace.Handle, typeof(String), out namespaceStr, false)) + throw new InvalidCastException("Couldn't convert __namespace__ value to string"); + } + } + finally { - PyObject pyNamespace = new PyObject(Runtime.PyDict_GetItem(py_dict, namespaceKey.Handle)); - if (!Converter.ToManagedValue(pyNamespace.Handle, typeof(String), out namespaceStr, false)) - throw new InvalidCastException("Couldn't convert __namespace__ value to string"); + foreach (PyObject o in disposeList) + o.Dispose(); } // create the new managed type subclassing the base managed type From f3ff88d5ab3834807a405f90ac13b69475b8af60 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 18 Oct 2013 13:38:20 +0100 Subject: [PATCH 15/77] merge from patstew-master --- pythonnet/src/runtime/converter.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pythonnet/src/runtime/converter.cs b/pythonnet/src/runtime/converter.cs index 43ac0a206..b3d420ff6 100644 --- a/pythonnet/src/runtime/converter.cs +++ b/pythonnet/src/runtime/converter.cs @@ -110,6 +110,16 @@ internal static IntPtr ToPython(Object value, Type type) { return ClassDerivedObject.ToPython(pyderived); } + if (value is IEnumerable) + { + var resultlist = new PyList(); + foreach (object o in (IEnumerable)value) + { + resultlist.Append(new PyObject(ToPython(o, o.GetType()))); + } + return resultlist.Handle; + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. From 471673a1fa4691b3becb4d9fee6941308535bb04 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 21 Oct 2013 15:50:22 +0100 Subject: [PATCH 16/77] don't convert IEnumerables to python lists unless nothing else matches (strings shouldn't be converted to lists, for example) --- pythonnet/src/runtime/converter.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pythonnet/src/runtime/converter.cs b/pythonnet/src/runtime/converter.cs index b3d420ff6..4ceba15da 100644 --- a/pythonnet/src/runtime/converter.cs +++ b/pythonnet/src/runtime/converter.cs @@ -110,16 +110,6 @@ internal static IntPtr ToPython(Object value, Type type) { return ClassDerivedObject.ToPython(pyderived); } - if (value is IEnumerable) - { - var resultlist = new PyList(); - foreach (object o in (IEnumerable)value) - { - resultlist.Append(new PyObject(ToPython(o, o.GetType()))); - } - return resultlist.Handle; - } - // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. @@ -194,6 +184,15 @@ internal static IntPtr ToPython(Object value, Type type) { return Runtime.PyLong_FromUnsignedLongLong((ulong)value); default: + if (value is IEnumerable) + { + var resultlist = new PyList(); + foreach (object o in (IEnumerable)value) + { + resultlist.Append(new PyObject(ToPython(o, o.GetType()))); + } + return resultlist.Handle; + } result = CLRObject.GetInstHandle(value, type); return result; } From f9109a09a0c0bea0296b770b01a998bbcb7091b1 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 21 Oct 2013 15:52:47 +0100 Subject: [PATCH 17/77] allow base class methods to be called from derived classes either from the class directly or using super --- pythonnet/src/runtime/classmanager.cs | 2 +- pythonnet/src/runtime/methodbinding.cs | 83 +++++++++++++++---- pythonnet/src/runtime/methodobject.cs | 31 +++++-- pythonnet/src/runtime/modulefunctionobject.cs | 4 +- pythonnet/src/runtime/moduleobject.cs | 2 +- pythonnet/src/runtime/typemanager.cs | 2 +- pythonnet/src/runtime/typemethod.cs | 8 +- pythonnet/src/tests/test_subclass.py | 8 ++ 8 files changed, 106 insertions(+), 34 deletions(-) diff --git a/pythonnet/src/runtime/classmanager.cs b/pythonnet/src/runtime/classmanager.cs index 6bcd52ce5..1e2aceac1 100644 --- a/pythonnet/src/runtime/classmanager.cs +++ b/pythonnet/src/runtime/classmanager.cs @@ -348,7 +348,7 @@ private static ClassInfo GetClassInfo(Type type) { typeof(MethodInfo) ); - ob = new MethodObject(name, mlist); + ob = new MethodObject(type, name, mlist); ci.members[name] = ob; } diff --git a/pythonnet/src/runtime/methodbinding.cs b/pythonnet/src/runtime/methodbinding.cs index 0459d36b2..0658e0095 100644 --- a/pythonnet/src/runtime/methodbinding.cs +++ b/pythonnet/src/runtime/methodbinding.cs @@ -9,6 +9,7 @@ using System; using System.Reflection; +using System.Collections.Generic; namespace Python.Runtime { @@ -23,14 +24,25 @@ internal class MethodBinding : ExtensionType { internal MethodInfo info; internal MethodObject m; internal IntPtr target; + internal IntPtr targetType; - public MethodBinding(MethodObject m, IntPtr target) : base() { + public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) : base() { Runtime.Incref(target); this.target = target; + + Runtime.Incref(targetType); + if (targetType == IntPtr.Zero) + targetType = Runtime.PyObject_Type(target); + this.targetType = targetType; + this.info = null; this.m = m; } + public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zero) + { + } + //==================================================================== // Implement binding of generic methods using the subscript syntax []. //==================================================================== @@ -114,25 +126,59 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { // as the first argument. Note that this is not supported if any // of the overloads are static since we can't know if the intent // was to call the static method or the unbound instance method. + List disposeList = new List(); + try + { + IntPtr target = self.target; + + if ((target == IntPtr.Zero) && (!self.m.IsStatic())) + { + int len = Runtime.PyTuple_Size(args); + if (len < 1) + { + Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); + return IntPtr.Zero; + } + target = Runtime.PyTuple_GetItem(args, 0); + Runtime.Incref(target); + disposeList.Add(target); + + args = Runtime.PyTuple_GetSlice(args, 1, len); + disposeList.Add(args); + } - if ((self.target == IntPtr.Zero) && (!self.m.IsStatic())) - { - int len = Runtime.PyTuple_Size(args); - if (len < 1) - { - Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); - return IntPtr.Zero; - } - IntPtr uargs = Runtime.PyTuple_GetSlice(args, 1, len); - IntPtr inst = Runtime.PyTuple_GetItem(args, 0); - Runtime.Incref(inst); - IntPtr r = self.m.Invoke(inst, uargs, kw, self.info); - Runtime.Decref(inst); - Runtime.Decref(uargs); - return r; - } + // if the class is a IPythonDerivedClass and target is not the same as self.targetType + // (eg if calling the base class method) then call the original base class method instead + // of the target method. + IntPtr superType = IntPtr.Zero; + if (Runtime.PyObject_TYPE(target) != self.targetType) + { + CLRObject inst = CLRObject.GetManagedObject(target) as CLRObject; + if (inst != null && (inst.inst as IPythonDerivedType) != null) + { + ClassBase baseType = GetManagedObject(self.targetType) as ClassBase; + if (baseType != null) + { + string baseMethodName = "_" + baseType.type.Name + "__" + self.m.name; + IntPtr baseMethod = Runtime.PyObject_GetAttrString(target, baseMethodName); + if (baseMethod != null) + { + MethodBinding baseSelf = GetManagedObject(baseMethod) as MethodBinding; + if (baseSelf != null) + self = baseSelf; + } + Runtime.Decref(baseMethod); + } + } + } - return self.m.Invoke(self.target, args, kw, self.info); + return self.m.Invoke(target, args, kw, self.info); + } + finally + { + foreach (IntPtr ptr in disposeList) + Runtime.Decref(ptr); + } } @@ -184,6 +230,7 @@ public static IntPtr tp_repr(IntPtr ob) { public static new void tp_dealloc(IntPtr ob) { MethodBinding self = (MethodBinding)GetManagedObject(ob); Runtime.Decref(self.target); + Runtime.Decref(self.targetType); ExtensionType.FinalizeObject(self); } diff --git a/pythonnet/src/runtime/methodobject.cs b/pythonnet/src/runtime/methodobject.cs index 15a5cd547..0298751ca 100644 --- a/pythonnet/src/runtime/methodobject.cs +++ b/pythonnet/src/runtime/methodobject.cs @@ -27,19 +27,21 @@ internal class MethodObject : ExtensionType { internal MethodBinder binder; internal bool is_static = false; internal IntPtr doc; + internal Type type; - public MethodObject(string name, MethodInfo[] info) : base() { - _MethodObject(name, info); + public MethodObject(Type type, string name, MethodInfo[] info) : base() { + _MethodObject(type, name, info); } - public MethodObject(string name, MethodInfo[] info, bool allow_threads) : base() + public MethodObject(Type type, string name, MethodInfo[] info, bool allow_threads) : base() { - _MethodObject(name, info); + _MethodObject(type, name, info); binder.allow_threads = allow_threads; } - private void _MethodObject(string name, MethodInfo[] info) + private void _MethodObject(Type type, string name, MethodInfo[] info) { + this.type = type; this.name = name; this.info = info; binder = new MethodBinder(); @@ -144,7 +146,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { if (ob == IntPtr.Zero) { if (self.unbound == null) { - self.unbound = new MethodBinding(self, IntPtr.Zero); + self.unbound = new MethodBinding(self, IntPtr.Zero, tp); } binding = self.unbound; Runtime.Incref(binding.pyHandle);; @@ -155,7 +157,22 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return Exceptions.RaiseTypeError("invalid argument"); } - binding = new MethodBinding(self, ob); + // The the object this descriptor is being called with is a subclass of the type + // this descriptor was defined on then it will be because the base class method + // is being called via super(Derived, self).method(...). + // In which case create a MethodBinding bound to the base class. + CLRObject obj = GetManagedObject(ob) as CLRObject; + if (obj != null + && obj.inst.GetType() != self.type + && obj.inst is IPythonDerivedType + && self.type.IsAssignableFrom(obj.inst.GetType())) + { + ClassBase basecls = ClassManager.GetClass(self.type); + binding = new MethodBinding(self, ob, basecls.pyHandle); + return binding.pyHandle; + } + + binding = new MethodBinding(self, ob, tp); return binding.pyHandle; } diff --git a/pythonnet/src/runtime/modulefunctionobject.cs b/pythonnet/src/runtime/modulefunctionobject.cs index 5c9a4de21..2aa8c2306 100644 --- a/pythonnet/src/runtime/modulefunctionobject.cs +++ b/pythonnet/src/runtime/modulefunctionobject.cs @@ -19,8 +19,8 @@ namespace Python.Runtime internal class ModuleFunctionObject : MethodObject { - public ModuleFunctionObject(string name, MethodInfo[] info, bool allow_threads) - : base(name, info, allow_threads) + public ModuleFunctionObject(Type type, string name, MethodInfo[] info, bool allow_threads) + : base(type, name, info, allow_threads) { for (int i = 0; i < info.Length; i++) { diff --git a/pythonnet/src/runtime/moduleobject.cs b/pythonnet/src/runtime/moduleobject.cs index b39135cdc..6d5241786 100644 --- a/pythonnet/src/runtime/moduleobject.cs +++ b/pythonnet/src/runtime/moduleobject.cs @@ -222,7 +222,7 @@ internal void InitializeModuleMembers() string name = method.Name; MethodInfo[] mi = new MethodInfo[1]; mi[0] = method; - ModuleFunctionObject m = new ModuleFunctionObject(name, mi, allow_threads); + ModuleFunctionObject m = new ModuleFunctionObject(type, name, mi, allow_threads); StoreAttribute(name, m); } } diff --git a/pythonnet/src/runtime/typemanager.cs b/pythonnet/src/runtime/typemanager.cs index 3d1fd9a4b..1bc612b8c 100644 --- a/pythonnet/src/runtime/typemanager.cs +++ b/pythonnet/src/runtime/typemanager.cs @@ -458,7 +458,7 @@ private static void InitMethods(IntPtr pytype, Type type) { string method_name = method.Name; MethodInfo[] mi = new MethodInfo[1]; mi[0] = method; - MethodObject m = new TypeMethod(method_name, mi); + MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); } diff --git a/pythonnet/src/runtime/typemethod.cs b/pythonnet/src/runtime/typemethod.cs index ab95f28ed..9170e5a4c 100644 --- a/pythonnet/src/runtime/typemethod.cs +++ b/pythonnet/src/runtime/typemethod.cs @@ -19,11 +19,11 @@ namespace Python.Runtime { internal class TypeMethod : MethodObject { - public TypeMethod(string name, MethodInfo[] info) : - base(name, info) {} + public TypeMethod(Type type, string name, MethodInfo[] info) : + base(type, name, info) {} - public TypeMethod(string name, MethodInfo[] info, bool allow_threads) : - base(name, info, allow_threads) { } + public TypeMethod(Type type, string name, MethodInfo[] info, bool allow_threads) : + base(type, name, info, allow_threads) { } public override IntPtr Invoke(IntPtr ob, IntPtr args, IntPtr kw) { MethodInfo mi = this.info[0]; diff --git a/pythonnet/src/tests/test_subclass.py b/pythonnet/src/tests/test_subclass.py index bc29e9c0a..882627e8b 100644 --- a/pythonnet/src/tests/test_subclass.py +++ b/pythonnet/src/tests/test_subclass.py @@ -19,6 +19,12 @@ class DerivedClass(SubClassTest): def foo(self): return "bar" + def base_foo(self): + return SubClassTest.foo(self) + + def super_foo(self): + return super(DerivedClass, self).foo() + def bar(self, x, i): return "_".join([x] * i) @@ -47,6 +53,8 @@ def testDerivedClass(self): """Test python class derived from managed type""" object = DerivedClass() self.assertEqual(object.foo(), "bar") + self.assertEqual(object.base_foo(), "foo") + self.assertEqual(object.super_foo(), "foo") self.assertEqual(SubClassTest.test_foo(object), "bar") self.assertEqual(object.bar("bar", 2), "bar_bar") self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar_bar") From 93e9ba34bcbd048a2b603a08946f7bf1b92b2890 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 10 Jan 2014 17:37:55 +0000 Subject: [PATCH 18/77] fix derived types so managed classes can be declared in python by subclassing interfaces --- pythonnet/src/runtime/classderived.cs | 23 ++++++++--- pythonnet/src/runtime/methodobject.cs | 2 +- pythonnet/src/runtime/typemanager.cs | 56 +++++++++++++++++---------- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index d1c6424fa..04f4ff480 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -129,10 +129,23 @@ internal static Type CreateDerivedType(string name, assemblyName = Assembly.GetExecutingAssembly().FullName; ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); - TypeBuilder typeBuilder = moduleBuilder.DefineType(name, - TypeAttributes.Public | TypeAttributes.Class, - baseType, - new Type[] { typeof(IPythonDerivedType) }); + TypeBuilder typeBuilder; + + Type baseClass = baseType; + List interfaces = new List { typeof(IPythonDerivedType) }; + + // if the base type is an interface then use System.Object as the base class + // and add the base type to the list of interfaces this new class will implement. + if (baseType.IsInterface) + { + interfaces.Add(baseType); + baseClass = typeof(System.Object); + } + + typeBuilder = moduleBuilder.DefineType(name, + TypeAttributes.Public | TypeAttributes.Class, + baseClass, + interfaces.ToArray()); ILGenerator il; MethodBuilder mb; @@ -268,7 +281,7 @@ internal static Type CreateDerivedType(string name, il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("Finalize")); il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Call, baseType.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); il.Emit(OpCodes.Ret); Type type = typeBuilder.CreateType(); diff --git a/pythonnet/src/runtime/methodobject.cs b/pythonnet/src/runtime/methodobject.cs index 0298751ca..45e7de709 100644 --- a/pythonnet/src/runtime/methodobject.cs +++ b/pythonnet/src/runtime/methodobject.cs @@ -157,7 +157,7 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) { return Exceptions.RaiseTypeError("invalid argument"); } - // The the object this descriptor is being called with is a subclass of the type + // If the object this descriptor is being called with is a subclass of the type // this descriptor was defined on then it will be because the base class method // is being called via super(Derived, self).method(...). // In which case create a MethodBinding bound to the base class. diff --git a/pythonnet/src/runtime/typemanager.cs b/pythonnet/src/runtime/typemanager.cs index 1bc612b8c..d51c60818 100644 --- a/pythonnet/src/runtime/typemanager.cs +++ b/pythonnet/src/runtime/typemanager.cs @@ -236,23 +236,34 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr } // create the new managed type subclassing the base managed type - ClassObject baseClass = ManagedType.GetManagedObject(py_base_type) as ClassObject; + ClassBase baseClass = ManagedType.GetManagedObject(py_base_type) as ClassBase; + if (null == baseClass) + { + return Exceptions.RaiseTypeError("invalid base class, expected CLR class type"); + } - Type subType = ClassDerivedObject.CreateDerivedType(name, - baseClass.type, - (string)namespaceStr, - (string)assembly); + try + { + Type subType = ClassDerivedObject.CreateDerivedType(name, + baseClass.type, + (string)namespaceStr, + (string)assembly); - // create the new ManagedType and python type - ClassBase subClass = ClassManager.GetClass(subType); - IntPtr py_type = GetTypeHandle(subClass, subType); + // create the new ManagedType and python type + ClassBase subClass = ClassManager.GetClass(subType); + IntPtr py_type = GetTypeHandle(subClass, subType); - // by default the class dict will have all the C# methods in it, but as this is a - // derived class we want the python overrides in there instead if they exist. - IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); - Runtime.PyDict_Update(cls_dict, py_dict); + // by default the class dict will have all the C# methods in it, but as this is a + // derived class we want the python overrides in there instead if they exist. + IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); + Runtime.PyDict_Update(cls_dict, py_dict); - return py_type; + return py_type; + } + catch (Exception e) + { + return Exceptions.RaiseTypeError(e.Message); + } } internal static IntPtr CreateMetaType(Type impl) { @@ -448,19 +459,22 @@ private static void InitMethods(IntPtr pytype, Type type) { Type marker = typeof(PythonMethodAttribute); BindingFlags flags = BindingFlags.Public | BindingFlags.Static; + HashSet addedMethods = new HashSet(); while (type != null) { MethodInfo[] methods = type.GetMethods(flags); for (int i = 0; i < methods.Length; i++) { MethodInfo method = methods[i]; - object[] attrs = method.GetCustomAttributes(marker, false); - if (attrs.Length > 0) { - string method_name = method.Name; - MethodInfo[] mi = new MethodInfo[1]; - mi[0] = method; - MethodObject m = new TypeMethod(type, method_name, mi); - Runtime.PyDict_SetItemString(dict, method_name, - m.pyHandle); + if (!addedMethods.Contains(method.Name)) { + object[] attrs = method.GetCustomAttributes(marker, false); + if (attrs.Length > 0) { + string method_name = method.Name; + MethodInfo[] mi = new MethodInfo[1]; + mi[0] = method; + MethodObject m = new TypeMethod(type, method_name, mi); + Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + addedMethods.Add(method_name); + } } } type = type.BaseType; From 7417342ae02408a98a5f0cbec2effe44ce74eca2 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 10 Jan 2014 20:04:24 +0000 Subject: [PATCH 19/77] get the constructors from the base class rather than the type when building a derived class, as the type could be an interface --- pythonnet/src/runtime/classderived.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index 04f4ff480..cabc37e1e 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -154,7 +154,7 @@ internal static Type CreateDerivedType(string name, FieldBuilder fb = typeBuilder.DefineField("__pyobj__", typeof(CLRObject), FieldAttributes.Public); // override any constructors - ConstructorInfo[] constructors = baseType.GetConstructors(); + ConstructorInfo[] constructors = baseClass.GetConstructors(); foreach (ConstructorInfo ctor in constructors) { ParameterInfo[] parameters = ctor.GetParameters(); @@ -349,6 +349,7 @@ public static T InvokeMethod(IPythonDerivedType obj, string methodName, strin { FieldInfo fi = obj.GetType().GetField("__pyobj__"); CLRObject self = (CLRObject)fi.GetValue(obj); + if (null != self) { List disposeList = new List(); From 686eba29441c369f47b57ecc96b8022138e89749 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 13 Jan 2014 13:52:12 +0000 Subject: [PATCH 20/77] add unit tests to check python classes can derive from C# interfaces --- pythonnet/src/testing/subclasstest.cs | 36 ++++++++++++------ pythonnet/src/tests/test_subclass.py | 55 ++++++++++++++++++++------- 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/pythonnet/src/testing/subclasstest.cs b/pythonnet/src/testing/subclasstest.cs index 3927b104e..64cea87c6 100644 --- a/pythonnet/src/testing/subclasstest.cs +++ b/pythonnet/src/testing/subclasstest.cs @@ -5,7 +5,16 @@ namespace Python.Test { - public class SubClassTest + public interface IInterfaceTest + { + // simple test with no arguments + string foo(); + + // test passing objects and boxing primitives + string bar(string s, int i); + } + + public class SubClassTest : IInterfaceTest { public SubClassTest() { @@ -34,32 +43,35 @@ public virtual IList return_list() return new List { "a", "b", "c" }; } - public static string test_foo(SubClassTest x) + public static IList test_list(SubClassTest x) + { + // calls into python if return_list is overriden + return x.return_list(); + } + } + + public class TestFunctions + { + public static string test_foo(IInterfaceTest x) { // calls into python if foo is overriden return x.foo(); } - public static string test_bar(SubClassTest x, string s, int i) + public static string test_bar(IInterfaceTest x, string s, int i) { // calls into python if bar is overriden return x.bar(s, i); } - public static IList test_list(SubClassTest x) - { - // calls into python if return_list is overriden - return x.return_list(); - } - // test instances can be constructed in managed code - public static SubClassTest create_instance(Type t) + public static IInterfaceTest create_instance(Type t) { - return (SubClassTest)t.GetConstructor(new Type[] {}).Invoke(new Object[] {}); + return (IInterfaceTest)t.GetConstructor(new Type[] {}).Invoke(new Object[] {}); } // test instances pass through managed code unchanged - public static SubClassTest pass_through(SubClassTest s) + public static IInterfaceTest pass_through(IInterfaceTest s) { return s; } diff --git a/pythonnet/src/tests/test_subclass.py b/pythonnet/src/tests/test_subclass.py index 882627e8b..5a8e0fb8d 100644 --- a/pythonnet/src/tests/test_subclass.py +++ b/pythonnet/src/tests/test_subclass.py @@ -11,13 +11,22 @@ clr.AddReference('System') import sys, os, string, unittest, types -from Python.Test import SubClassTest +from Python.Test import TestFunctions, SubClassTest, IInterfaceTest from System.Collections.Generic import List +# class that implements the test interface +class InterfaceTestClass(IInterfaceTest): + def foo(self): + return "InterfaceTestClass" + + def bar(self, x, i): + return "/".join([x] * i) + +# class that derives from a class deriving from IInterfaceTest class DerivedClass(SubClassTest): def foo(self): - return "bar" + return "DerivedClass" def base_foo(self): return SubClassTest.foo(self) @@ -42,41 +51,61 @@ def testBaseClass(self): """Test base class managed type""" object = SubClassTest() self.assertEqual(object.foo(), "foo") - self.assertEqual(SubClassTest.test_foo(object), "foo") + self.assertEqual(TestFunctions.test_foo(object), "foo") self.assertEqual(object.bar("bar", 2), "bar") - self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar") self.assertEqual(object.not_overriden(), "not_overriden") self.assertEqual(list(object.return_list()), ["a", "b", "c"]) self.assertEqual(list(SubClassTest.test_list(object)), ["a", "b", "c"]) + def testInterface(self): + """Test python classes can derive from C# interfaces""" + object = InterfaceTestClass() + self.assertEqual(object.foo(), "InterfaceTestClass") + self.assertEqual(TestFunctions.test_foo(object), "InterfaceTestClass") + self.assertEqual(object.bar("bar", 2), "bar/bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar/bar") + + x = TestFunctions.pass_through(object) + self.assertEqual(id(x), id(object)) + def testDerivedClass(self): """Test python class derived from managed type""" object = DerivedClass() - self.assertEqual(object.foo(), "bar") + self.assertEqual(object.foo(), "DerivedClass") self.assertEqual(object.base_foo(), "foo") self.assertEqual(object.super_foo(), "foo") - self.assertEqual(SubClassTest.test_foo(object), "bar") + self.assertEqual(TestFunctions.test_foo(object), "DerivedClass") self.assertEqual(object.bar("bar", 2), "bar_bar") - self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar_bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar_bar") self.assertEqual(object.not_overriden(), "not_overriden") self.assertEqual(list(object.return_list()), ["A", "B", "C"]) self.assertEqual(list(SubClassTest.test_list(object)), ["A", "B", "C"]) - x = SubClassTest.pass_through(object) + x = TestFunctions.pass_through(object) self.assertEqual(id(x), id(object)) def testCreateInstance(self): """Test derived instances can be created from managed code""" - object = SubClassTest.create_instance(DerivedClass) - self.assertEqual(object.foo(), "bar") - self.assertEqual(SubClassTest.test_foo(object), "bar") + object = TestFunctions.create_instance(DerivedClass) + self.assertEqual(object.foo(), "DerivedClass") + self.assertEqual(TestFunctions.test_foo(object), "DerivedClass") self.assertEqual(object.bar("bar", 2), "bar_bar") - self.assertEqual(SubClassTest.test_bar(object, "bar", 2), "bar_bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar_bar") self.assertEqual(object.not_overriden(), "not_overriden") - x = SubClassTest.pass_through(object) + x = TestFunctions.pass_through(object) self.assertEqual(id(x), id(object)) + object2 = TestFunctions.create_instance(InterfaceTestClass) + self.assertEqual(object2.foo(), "InterfaceTestClass") + self.assertEqual(TestFunctions.test_foo(object2), "InterfaceTestClass") + self.assertEqual(object2.bar("bar", 2), "bar/bar") + self.assertEqual(TestFunctions.test_bar(object2, "bar", 2), "bar/bar") + + y = TestFunctions.pass_through(object2) + self.assertEqual(id(y), id(object2)) + def test_suite(): return unittest.makeSuite(SubClassTests) From afce8dc9dd7eb96cf1534e0fc28e422b4d211b8c Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 12 Feb 2014 09:36:26 +0000 Subject: [PATCH 21/77] Update README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 515346d30..6205329fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ pythonnet ========= +**NOTE** The *official* repo is now https://github.com/pythonnet/pythonnet. Changes from this fork of the original sourceforge project will be integrated back into that main repo in due course. + +**Features not yet integrated into the main repo**: +- Python 3 support +- Subclassing managed types in Python + +-------------------------------------------------------------------------------------------------------- + This fork of http://sourceforge.net/projects/pythonnet/ allows easy calling of python functions from C#. + All calls to python should be inside a "using (Py.GIL()) {/* Your code here */}" block. From f0d0aeb810308dc4061821076562ad37a05301e8 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 14 Feb 2014 14:53:51 +0000 Subject: [PATCH 22/77] set __file__, __doc__ and __class__ on ModuleObject to make the managed modules behave better in ipython (%autoreload expects __file__ to be set and the ipython help ? expects __class__ to be set). --- pythonnet/src/runtime/assemblymanager.cs | 10 ++++++++++ pythonnet/src/runtime/moduleobject.cs | 19 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pythonnet/src/runtime/assemblymanager.cs b/pythonnet/src/runtime/assemblymanager.cs index 5ff2241d6..7b8a58c97 100644 --- a/pythonnet/src/runtime/assemblymanager.cs +++ b/pythonnet/src/runtime/assemblymanager.cs @@ -314,6 +314,16 @@ public static bool IsValidNamespace(string name) { return namespaces.ContainsKey(name); } + //=================================================================== + // Returns list of assemblies that declare types in a given namespace + //=================================================================== + + public static IEnumerable GetAssemblies(string nsname) { + if (!namespaces.ContainsKey(nsname)) + return new List(); + + return namespaces[nsname].Keys; + } //=================================================================== // Returns the current list of valid names for the input namespace. diff --git a/pythonnet/src/runtime/moduleobject.cs b/pythonnet/src/runtime/moduleobject.cs index 6d5241786..ce716d635 100644 --- a/pythonnet/src/runtime/moduleobject.cs +++ b/pythonnet/src/runtime/moduleobject.cs @@ -37,12 +37,27 @@ public ModuleObject(string name) : base() { cache = new Dictionary(); _namespace = name; + // Use the filename from any of the assemblies just so there's something for + // anything that expects __file__ to be set. + string filename = "unknown"; + string docstring = "Namespace containing types from the following assemblies:\n\n"; + foreach (Assembly a in AssemblyManager.GetAssemblies(name)) { + filename = a.Location; + docstring += "- " + a.FullName + "\n"; + } + dict = Runtime.PyDict_New(); IntPtr pyname = Runtime.PyString_FromString(moduleName); + IntPtr pyfilename = Runtime.PyString_FromString(filename); + IntPtr pydocstring = Runtime.PyString_FromString(docstring); + IntPtr pycls = TypeManager.GetTypeHandle(this.GetType()); Runtime.PyDict_SetItemString(dict, "__name__", pyname); - Runtime.PyDict_SetItemString(dict, "__file__", Runtime.PyNone); - Runtime.PyDict_SetItemString(dict, "__doc__", Runtime.PyNone); + Runtime.PyDict_SetItemString(dict, "__file__", pyfilename); + Runtime.PyDict_SetItemString(dict, "__doc__", pydocstring); + Runtime.PyDict_SetItemString(dict, "__class__", pycls); Runtime.Decref(pyname); + Runtime.Decref(pyfilename); + Runtime.Decref(pydocstring); Marshal.WriteIntPtr(this.pyHandle, ObjectOffset.DictOffset(this.pyHandle), dict); From ecf3237d27a88f503e28c9f94ebf0bce0bb3c107 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 27 May 2014 17:47:58 +0100 Subject: [PATCH 23/77] Stop PythonDerivedType.Finalize from trying to delete any Python objects once Py_Finalize has started. --- pythonnet/src/runtime/classderived.cs | 52 +++++++++++++++++---------- pythonnet/src/runtime/moduleobject.cs | 6 ++++ pythonnet/src/runtime/pythonengine.cs | 10 ++++++ pythonnet/src/runtime/runtime.cs | 12 +++++++ 4 files changed, 62 insertions(+), 18 deletions(-) diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index cabc37e1e..46b62a225 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -531,29 +531,45 @@ public static void Finalize(IPythonDerivedType obj) FieldInfo fi = obj.GetType().GetField("__pyobj__"); CLRObject self = (CLRObject)fi.GetValue(obj); - // delete the python object in an asnyc task as we may not be able to acquire - // the GIL immediately and we don't want to block the GC thread. - var t = Task.Factory.StartNew(() => + // If python's been terminated then just free the gchandle. + lock (Runtime.IsFinalizingLock) { - // If python's been terminated then there's nothing to do - if (0 == Runtime.Py_IsInitialized()) - return; - - IntPtr gs = Runtime.PyGILState_Ensure(); - try + if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) { - // the C# object is being destroyed which must mean there are no more - // references to the Python object as well so now we can dealloc the - // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); - if (dict != IntPtr.Zero) - Runtime.Decref(dict); - Runtime.PyObject_GC_Del(self.pyHandle); self.gcHandle.Free(); + return; } - finally + } + + // delete the python object in an asnyc task as we may not be able to acquire + // the GIL immediately and we don't want to block the GC thread. + var t = Task.Factory.StartNew(() => + { + lock (Runtime.IsFinalizingLock) { - Runtime.PyGILState_Release(gs); + // If python's been terminated then just free the gchandle. + if (0 == Runtime.Py_IsInitialized() || Runtime.IsFinalizing) + { + self.gcHandle.Free(); + return; + } + + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + // the C# object is being destroyed which must mean there are no more + // references to the Python object as well so now we can dealloc the + // python object. + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + if (dict != IntPtr.Zero) + Runtime.Decref(dict); + Runtime.PyObject_GC_Del(self.pyHandle); + self.gcHandle.Free(); + } + finally + { + Runtime.PyGILState_Release(gs); + } } }); } diff --git a/pythonnet/src/runtime/moduleobject.cs b/pythonnet/src/runtime/moduleobject.cs index ce716d635..b038c28a3 100644 --- a/pythonnet/src/runtime/moduleobject.cs +++ b/pythonnet/src/runtime/moduleobject.cs @@ -442,6 +442,12 @@ public static String[] ListAssemblies(bool verbose) return names; } + [ModuleFunctionAttribute()] + public static int _AtExit() + { + return Runtime.AtExit(); + } + } } diff --git a/pythonnet/src/runtime/pythonengine.cs b/pythonnet/src/runtime/pythonengine.cs index 59d5a72bb..b29950896 100644 --- a/pythonnet/src/runtime/pythonengine.cs +++ b/pythonnet/src/runtime/pythonengine.cs @@ -117,6 +117,16 @@ public static void Initialize() { Runtime.Initialize(); initialized = true; Exceptions.Clear(); + + // register the atexit callback (this doesn't use Py_AtExit as the C atexit + // callbacks are called after python is fully finalized but the python ones + // are called while the python engine is still running). + string code = + "import atexit, clr\n" + + "atexit.register(clr._AtExit)\n"; + PyObject r = PythonEngine.RunString(code); + if (r != null) + r.Dispose(); } } diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index d51299275..5abfff441 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -84,6 +84,10 @@ public class Runtime { #error You must define one of PYTHON23 to PYTHON34 #endif + // set to true when python is finalizing + internal static Object IsFinalizingLock = new Object(); + internal static bool IsFinalizing = false; + internal static bool wrap_exceptions; internal static bool is32bit; @@ -232,6 +236,14 @@ internal static void Shutdown() { Py_Finalize(); } + // called *without* the GIL aquired by clr._AtExit + internal static int AtExit() { + lock (IsFinalizingLock) { + IsFinalizing = true; + } + return 0; + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; internal static IntPtr Py_eval_input = (IntPtr)258; From 14591b62cb1d21fbc39759bef9c892d2e5b9aaae Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 28 May 2014 10:59:09 +0100 Subject: [PATCH 24/77] Python 3.4 compatibility (fixes #8) --- pythonnet/src/runtime/interop.cs | 25 ++++++++++++++++++++++--- pythonnet/src/runtime/methodwrapper.cs | 2 +- pythonnet/src/runtime/runtime.cs | 5 ----- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pythonnet/src/runtime/interop.cs b/pythonnet/src/runtime/interop.cs index ecbf60eb9..e3227f6aa 100644 --- a/pythonnet/src/runtime/interop.cs +++ b/pythonnet/src/runtime/interop.cs @@ -261,6 +261,9 @@ public static int magic() { #if (PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) /* Type attribute cache version tag. Added in version 2.6 */ public static int tp_version_tag; +#endif +#if (PYTHON34) + public static int tp_finalize = 0; #endif // COUNT_ALLOCS adds some more stuff to PyTypeObject #if (Py_COUNT_ALLOCS) @@ -470,6 +473,8 @@ public static void FreeModuleDef(IntPtr ptr) { /// to good use as PythonNet specific flags (Managed and Subclass) /// internal class TypeFlags { +#if (PYTHON23 || PYTHON24 || PYTHON25 || PYTHON26 || PYTHON27) + // these flags were removed in Python 3 public static int HaveGetCharBuffer = (1 << 0); public static int HaveSequenceIn = (1 << 1); public static int GC = 0; @@ -479,6 +484,7 @@ internal class TypeFlags { public static int HaveWeakRefs = (1 << 6); public static int HaveIter = (1 << 7); public static int HaveClass = (1 << 8); +#endif public static int HeapType = (1 << 9); public static int BaseType = (1 << 10); public static int Ready = (1 << 12); @@ -509,7 +515,11 @@ internal class TypeFlags { public static int BaseExceptionSubclass = (1 << 30); public static int TypeSubclass = (1 << 31); #endif - public static int Default = (HaveGetCharBuffer | + +// Default flags for Python 2 +#if (PYTHON23 || PYTHON24 || PYTHON25 || PYTHON26 || PYTHON27) + public static int Default = ( + HaveGetCharBuffer | HaveSequenceIn | HaveInPlaceOps | HaveRichCompare | @@ -517,10 +527,19 @@ internal class TypeFlags { HaveIter | HaveClass | HaveStacklessExtension | -#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) + #if (PYTHON25 || PYTHON26 || PYTHON27) HaveIndex | -#endif + #endif 0); +#endif + +// Default flags for Python 3 +#if (PYTHON32 || PYTHON33 || PYTHON34) + public static int Default = ( + HaveStacklessExtension | + HaveVersionTag); +#endif + } diff --git a/pythonnet/src/runtime/methodwrapper.cs b/pythonnet/src/runtime/methodwrapper.cs index 9b701f154..486e5c59d 100644 --- a/pythonnet/src/runtime/methodwrapper.cs +++ b/pythonnet/src/runtime/methodwrapper.cs @@ -50,7 +50,7 @@ public MethodWrapper(Type type, string name) { Marshal.WriteIntPtr(mdef, (1 * IntPtr.Size), fp); Marshal.WriteIntPtr(mdef, (2 * IntPtr.Size), (IntPtr)0x0003); // METH_VARARGS | METH_KEYWORDS Marshal.WriteIntPtr(mdef, (3 * IntPtr.Size), IntPtr.Zero); - ptr = Runtime.PyCFunction_New(mdef, IntPtr.Zero); + ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } public IntPtr Call(IntPtr args, IntPtr kw) { diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 5abfff441..cc362e5d0 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -648,11 +648,6 @@ internal unsafe static extern IntPtr internal unsafe static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); - [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, - ExactSpelling=true, CharSet=CharSet.Ansi)] - internal unsafe static extern IntPtr - PyCFunction_New(IntPtr ml, IntPtr self); - [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern IntPtr From 201137aff28ad6f7995fcc1c8f62269389591b42 Mon Sep 17 00:00:00 2001 From: sdpython Date: Sat, 5 Jul 2014 17:26:36 +0200 Subject: [PATCH 25/77] fix an issue with Python 3.4, it does not crash anynore when throwing an exception --- pythonnet/src/runtime/exceptions.cs | 17 ++++++++++++++--- pythonnet/src/runtime/interop.cs | 6 +++--- pythonnet/src/runtime/runtime.cs | 12 ++++++++++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pythonnet/src/runtime/exceptions.cs b/pythonnet/src/runtime/exceptions.cs index 5857c1e35..1da9f0390 100644 --- a/pythonnet/src/runtime/exceptions.cs +++ b/pythonnet/src/runtime/exceptions.cs @@ -222,12 +222,21 @@ internal unsafe static void ErrorOccurredCheck(IntPtr pointer) { /// /// internal static void SetupExceptionHack() { + DebugUtil.Print("SetupExceptionHack"); ns_exc = ClassManager.GetClass(typeof(Exception)).pyHandle; cache = new Hashtable(); + string module_exception = +#if(PYTHON34) + "builtins" +#else + "exceptions" +#endif + ; + string code = - "import exceptions\n" + - "class Exception(exceptions.Exception):\n" + + "import {0}\n" + + "class Exception({0}.Exception):\n" + " _class = None\n" + " _inner = None\n" + " \n" + @@ -239,7 +248,7 @@ internal static void SetupExceptionHack() { " def __init__(self, *args, **kw):\n" + " inst = self.__class__._class(*args, **kw)\n" + " self.__dict__['_inner'] = inst\n" + - " exceptions.Exception.__init__(self, *args, **kw)\n" + + " {0}.Exception.__init__(self, *args, **kw)\n" + "\n" + " def __getattr__(self, name, _marker=[]):\n" + " inner = self.__dict__['_inner']\n" + @@ -268,6 +277,8 @@ internal static void SetupExceptionHack() { " name = self.__class__.__name__\n" + " return '%s(\\'%s\\',)' % (name, msg) \n" + "\n"; + code = string.Format(code, module_exception); + DebugUtil.Print(code); IntPtr dict = Runtime.PyDict_New(); diff --git a/pythonnet/src/runtime/interop.cs b/pythonnet/src/runtime/interop.cs index e3227f6aa..401926082 100644 --- a/pythonnet/src/runtime/interop.cs +++ b/pythonnet/src/runtime/interop.cs @@ -79,7 +79,7 @@ static ObjectOffset() { } public static int magic(IntPtr ob) { -#if (PYTHON32 || PYTHON33 || PYTHON33) +#if (PYTHON32 || PYTHON33 || PYTHON34) if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) { @@ -91,7 +91,7 @@ public static int magic(IntPtr ob) { public static int DictOffset(IntPtr ob) { -#if (PYTHON32 || PYTHON33 || PYTHON33) +#if (PYTHON32 || PYTHON33 || PYTHON34) if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) { @@ -102,7 +102,7 @@ public static int DictOffset(IntPtr ob) } public static int Size(IntPtr ob) { -#if (PYTHON32 || PYTHON33 || PYTHON33) +#if (PYTHON32 || PYTHON33 || PYTHON34) if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) { diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index cc362e5d0..0522e743a 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -199,10 +199,18 @@ internal static void Initialize() { // of the Python runtime that do not allow new-style classes to // be used as exceptions (Python versions 2.4 and lower). -#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34 ) wrap_exceptions = false; #else - IntPtr m = PyImport_ImportModule("exceptions"); + + IntPtr m = PyImport_ImportModule( +#if(PYTHON34) + "builtins" +#else + "exceptions" +#endif + ); + Exceptions.ErrorCheck(m); op = Runtime.PyObject_GetAttrString(m, "Exception"); Exceptions.ErrorCheck(op); From c122b621e1ad672aab4f349d3ec89737551bdd77 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 8 Jul 2014 13:56:27 +0100 Subject: [PATCH 26/77] revert exceptions.cs and runtime.cs as the changes in them aren't necessary --- pythonnet/src/runtime/exceptions.cs | 17 +++-------------- pythonnet/src/runtime/runtime.cs | 12 ++---------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/pythonnet/src/runtime/exceptions.cs b/pythonnet/src/runtime/exceptions.cs index 1da9f0390..5857c1e35 100644 --- a/pythonnet/src/runtime/exceptions.cs +++ b/pythonnet/src/runtime/exceptions.cs @@ -222,21 +222,12 @@ internal unsafe static void ErrorOccurredCheck(IntPtr pointer) { /// /// internal static void SetupExceptionHack() { - DebugUtil.Print("SetupExceptionHack"); ns_exc = ClassManager.GetClass(typeof(Exception)).pyHandle; cache = new Hashtable(); - string module_exception = -#if(PYTHON34) - "builtins" -#else - "exceptions" -#endif - ; - string code = - "import {0}\n" + - "class Exception({0}.Exception):\n" + + "import exceptions\n" + + "class Exception(exceptions.Exception):\n" + " _class = None\n" + " _inner = None\n" + " \n" + @@ -248,7 +239,7 @@ internal static void SetupExceptionHack() { " def __init__(self, *args, **kw):\n" + " inst = self.__class__._class(*args, **kw)\n" + " self.__dict__['_inner'] = inst\n" + - " {0}.Exception.__init__(self, *args, **kw)\n" + + " exceptions.Exception.__init__(self, *args, **kw)\n" + "\n" + " def __getattr__(self, name, _marker=[]):\n" + " inner = self.__dict__['_inner']\n" + @@ -277,8 +268,6 @@ internal static void SetupExceptionHack() { " name = self.__class__.__name__\n" + " return '%s(\\'%s\\',)' % (name, msg) \n" + "\n"; - code = string.Format(code, module_exception); - DebugUtil.Print(code); IntPtr dict = Runtime.PyDict_New(); diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 0522e743a..cc362e5d0 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -199,18 +199,10 @@ internal static void Initialize() { // of the Python runtime that do not allow new-style classes to // be used as exceptions (Python versions 2.4 and lower). -#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34 ) +#if (PYTHON25 || PYTHON26 || PYTHON27 || PYTHON32 || PYTHON33 || PYTHON34) wrap_exceptions = false; #else - - IntPtr m = PyImport_ImportModule( -#if(PYTHON34) - "builtins" -#else - "exceptions" -#endif - ); - + IntPtr m = PyImport_ImportModule("exceptions"); Exceptions.ErrorCheck(m); op = Runtime.PyObject_GetAttrString(m, "Exception"); Exceptions.ErrorCheck(op); From 909a537b9a5321b48f39cc9fd69c829feaf2ac51 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 31 Jul 2014 13:54:29 +0100 Subject: [PATCH 27/77] Fix bug when calling methods with null arguments on derived types. --- pythonnet/src/runtime/classderived.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index 46b62a225..5acc31fa4 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -375,7 +375,7 @@ public static T InvokeMethod(IPythonDerivedType obj, string methodName, strin PyObject[] pyargs = new PyObject[args.Length]; for (int i = 0; i < args.Length; ++i) { - pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + pyargs[i] = new PyObject(Converter.ToPythonImplicit(args[i])); disposeList.Add(pyargs[i]); } @@ -431,7 +431,7 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s PyObject[] pyargs = new PyObject[args.Length]; for (int i = 0; i < args.Length; ++i) { - pyargs[i] = new PyObject(Converter.ToPython(args[i], args[i].GetType())); + pyargs[i] = new PyObject(Converter.ToPythonImplicit(args[i])); disposeList.Add(pyargs[i]); } From 79457feff75265f6c2588b6eb48c42aeaeb36295 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 19 Aug 2014 16:02:05 +0100 Subject: [PATCH 28/77] catch exceptions in tp_str when calling ToString. --- pythonnet/src/runtime/classbase.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pythonnet/src/runtime/classbase.cs b/pythonnet/src/runtime/classbase.cs index fb33817c2..09c4d65b5 100644 --- a/pythonnet/src/runtime/classbase.cs +++ b/pythonnet/src/runtime/classbase.cs @@ -170,7 +170,17 @@ public static IntPtr tp_str(IntPtr ob) { if (co == null) { return Exceptions.RaiseTypeError("invalid object"); } - return Runtime.PyString_FromString(co.inst.ToString()); + try { + return Runtime.PyString_FromString(co.inst.ToString()); + } + catch (Exception e) + { + if (e.InnerException != null) { + e = e.InnerException; + } + Exceptions.SetError(e); + return IntPtr.Zero; + } } From b64091b2d19b718232370868ee2bb4ee793970b4 Mon Sep 17 00:00:00 2001 From: sweinst Date: Sun, 31 Aug 2014 20:07:14 +0100 Subject: [PATCH 29/77] Fixed call to Py_SetPythonHome and Py_SetProgramName. Added support for Py_SetPythonPath --- pythonnet/src/runtime/pythonengine.cs | 16 +++++++++ pythonnet/src/runtime/runtime.cs | 49 +++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/pythonnet/src/runtime/pythonengine.cs b/pythonnet/src/runtime/pythonengine.cs index b29950896..b60ba1644 100644 --- a/pythonnet/src/runtime/pythonengine.cs +++ b/pythonnet/src/runtime/pythonengine.cs @@ -64,6 +64,22 @@ public static string PythonHome { } } + public static string PythonPath { + get + { + string result = Runtime.Py_GetPath(); + if (result == null) + { + return ""; + } + return result; + } + set + { + Runtime.Py_SetPath(value); + } + } + public static string Version { get { return Runtime.Py_GetVersion(); diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index cc362e5d0..1c38ebaf5 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -583,25 +583,70 @@ internal unsafe static extern IntPtr PyEval_GetLocals(); +#if PYTHON32 || PYTHON33 || PYTHON34 [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] + [return: MarshalAs(UnmanagedType.LPWStr)] internal unsafe static extern string Py_GetProgramName(); [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern void - Py_SetProgramName(string name); + Py_SetProgramName([MarshalAsAttribute(UnmanagedType.LPWStr)]string name); [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] + [return: MarshalAs(UnmanagedType.LPWStr)] internal unsafe static extern string Py_GetPythonHome(); [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern void - Py_SetPythonHome(string home); + Py_SetPythonHome([MarshalAsAttribute(UnmanagedType.LPWStr)]string home); + + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + [return: MarshalAs(UnmanagedType.LPWStr)] + internal unsafe static extern string + Py_GetPath(); + + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern void + Py_SetPath([MarshalAsAttribute(UnmanagedType.LPWStr)]string home); +#else + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern string + Py_GetProgramName(); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern void + Py_SetProgramName(string name); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern string + Py_GetPythonHome(); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern void + Py_SetPythonHome(string home); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern string + Py_GetPath(); + + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern void + Py_SetPath(string home); +#endif [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] From cfc3ef3517a943612a803a1390d224306a0bb5fe Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 2 Sep 2014 17:31:24 +0100 Subject: [PATCH 30/77] Subclasses written in Python can now have their properties and methods reflected back into .NET. Added decorators 'clrproperty' and 'clrmethod' to the clr module which are used to declare which properties and methods should be reflected, and to supply the .net argument and return types. --- pythonnet/src/runtime/Python.Runtime.csproj | 3 + pythonnet/src/runtime/classderived.cs | 506 +++++++++++++++----- pythonnet/src/runtime/converter.cs | 55 ++- pythonnet/src/runtime/pyobject.cs | 32 ++ pythonnet/src/runtime/pysequence.cs | 9 - pythonnet/src/runtime/pythonengine.cs | 133 +++-- pythonnet/src/runtime/pythonexception.cs | 12 + pythonnet/src/runtime/resources/clr.py | 83 ++++ pythonnet/src/runtime/runtime.cs | 36 ++ pythonnet/src/runtime/typemanager.cs | 1 + 10 files changed, 709 insertions(+), 161 deletions(-) create mode 100644 pythonnet/src/runtime/resources/clr.py diff --git a/pythonnet/src/runtime/Python.Runtime.csproj b/pythonnet/src/runtime/Python.Runtime.csproj index cccb2919e..880baf806 100644 --- a/pythonnet/src/runtime/Python.Runtime.csproj +++ b/pythonnet/src/runtime/Python.Runtime.csproj @@ -234,6 +234,9 @@ + + + diff --git a/pythonnet/src/runtime/classderived.cs b/pythonnet/src/runtime/classderived.cs index 5acc31fa4..ae18d8551 100644 --- a/pythonnet/src/runtime/classderived.cs +++ b/pythonnet/src/runtime/classderived.cs @@ -8,6 +8,7 @@ // ========================================================================== using System; +using System.IO; using System.Reflection; using System.Reflection.Emit; using System.Collections.Generic; @@ -47,9 +48,9 @@ internal ClassDerivedObject(Type tp) { } - //==================================================================== - // Implements __new__ for derived classes of reflected classes. - //==================================================================== + /// + /// Implements __new__ for derived classes of reflected classes. + /// new public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { ClassDerivedObject cls = GetManagedObject(tp) as ClassDerivedObject; @@ -111,13 +112,14 @@ internal static IntPtr ToPython(IPythonDerivedType obj) return self.pyHandle; } - //==================================================================== - // Creates a new managed type derived from a base type with any virtual - // methods overriden to call out to python if the associated python - // object has overriden the method. - //==================================================================== + /// + /// Creates a new managed type derived from a base type with any virtual + /// methods overriden to call out to python if the associated python + /// object has overriden the method. + /// internal static Type CreateDerivedType(string name, Type baseType, + IntPtr py_dict, string namespaceStr, string assemblyName, string moduleName="Python.Runtime.Dynamic.dll") @@ -147,9 +149,6 @@ internal static Type CreateDerivedType(string name, baseClass, interfaces.ToArray()); - ILGenerator il; - MethodBuilder mb; - // add a field for storing the python object pointer FieldBuilder fb = typeBuilder.DefineField("__pyobj__", typeof(CLRObject), FieldAttributes.Public); @@ -157,127 +156,82 @@ internal static Type CreateDerivedType(string name, ConstructorInfo[] constructors = baseClass.GetConstructors(); foreach (ConstructorInfo ctor in constructors) { - ParameterInfo[] parameters = ctor.GetParameters(); - Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); - - // create a method for calling the original constructor - string baseCtorName = "_" + baseType.Name + "__cinit__"; - mb = typeBuilder.DefineMethod(baseCtorName, - MethodAttributes.Public | - MethodAttributes.Final | - MethodAttributes.HideBySig, - typeof(void), - parameterTypes); - - // emit the assembly for calling the original method using call instead of callvirt - il = mb.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - for (int i = 0; i < parameters.Length; ++i) - il.Emit(OpCodes.Ldarg, i + 1); - il.Emit(OpCodes.Call, ctor); - il.Emit(OpCodes.Ret); + AddConstructor(ctor, baseType, typeBuilder); + } - // override the original method with a new one that dispatches to python - ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | - MethodAttributes.ReuseSlot | - MethodAttributes.HideBySig, - ctor.CallingConvention, - parameterTypes); - il = cb.GetILGenerator(); - il.DeclareLocal(typeof(Object[])); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, baseCtorName); - il.Emit(OpCodes.Ldc_I4, parameters.Length); - il.Emit(OpCodes.Newarr, typeof(System.Object)); - il.Emit(OpCodes.Stloc_0); - for (int i = 0; i < parameters.Length; ++i) + // Override any properties explicitly overriden in python + HashSet pyProperties = new HashSet(); + if (py_dict != IntPtr.Zero && Runtime.PyDict_Check(py_dict)) + { + Runtime.Incref(py_dict); + PyDict dict = new PyDict(py_dict); + + foreach (PyObject pyKey in dict.Keys()) { - il.Emit(OpCodes.Ldloc_0); - il.Emit(OpCodes.Ldc_I4, i); - il.Emit(OpCodes.Ldarg, i + 1); - if (parameterTypes[i].IsValueType) - il.Emit(OpCodes.Box, parameterTypes[i]); - il.Emit(OpCodes.Stelem, typeof(Object)); + PyObject value = dict[pyKey]; + if (value.HasAttr("_clr_property_type_")) + { + string propertyName = pyKey.ToString(); + pyProperties.Add(propertyName); + + // Add the property to the type + AddPythonProperty(propertyName, value, typeBuilder); + } } - il.Emit(OpCodes.Ldloc_0); - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeCtor")); - il.Emit(OpCodes.Ret); } - // override any virtual methods + // override any virtual methods not already overriden by the properties above MethodInfo[] methods = baseType.GetMethods(); + HashSet virtualMethods = new HashSet(); foreach (MethodInfo method in methods) { if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | method.Attributes.HasFlag(MethodAttributes.Final)) continue; - ParameterInfo[] parameters = method.GetParameters(); - Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + // skip if this property has already been overriden + if ((method.Name.StartsWith("get_") || method.Name.StartsWith("set_")) + && pyProperties.Contains(method.Name.Substring(4))) + continue; - // create a method for calling the original method - string baseMethodName = "_" + baseType.Name + "__" + method.Name; - mb = typeBuilder.DefineMethod(baseMethodName, - MethodAttributes.Public | - MethodAttributes.Final | - MethodAttributes.HideBySig, - method.ReturnType, - parameterTypes); + // keep track of the virtual methods redirected to the python instance + virtualMethods.Add(method.Name); - // emit the assembly for calling the original method using call instead of callvirt - il = mb.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - for (int i = 0; i < parameters.Length; ++i) - il.Emit(OpCodes.Ldarg, i + 1); - il.Emit(OpCodes.Call, method); - il.Emit(OpCodes.Ret); + // override the virtual method to call out to the python method, if there is one. + AddVirtualMethod(method, baseType, typeBuilder); + } - // override the original method with a new one that dispatches to python - mb = typeBuilder.DefineMethod(method.Name, - MethodAttributes.Public | - MethodAttributes.ReuseSlot | - MethodAttributes.Virtual | - MethodAttributes.HideBySig, - method.CallingConvention, - method.ReturnType, - parameterTypes); - il = mb.GetILGenerator(); - il.DeclareLocal(typeof(Object[])); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, method.Name); - il.Emit(OpCodes.Ldstr, baseMethodName); - il.Emit(OpCodes.Ldc_I4, parameters.Length); - il.Emit(OpCodes.Newarr, typeof(System.Object)); - il.Emit(OpCodes.Stloc_0); - for (int i = 0; i < parameters.Length; ++i) - { - il.Emit(OpCodes.Ldloc_0); - il.Emit(OpCodes.Ldc_I4, i); - il.Emit(OpCodes.Ldarg, i + 1); - if (parameterTypes[i].IsValueType) - il.Emit(OpCodes.Box, parameterTypes[i]); - il.Emit(OpCodes.Stelem, typeof(Object)); - } - il.Emit(OpCodes.Ldloc_0); - if (method.ReturnType == typeof(void)) - { - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid")); - } - else + // Add any additional methods and properties explicitly exposed from Python. + if (py_dict != IntPtr.Zero && Runtime.PyDict_Check(py_dict)) + { + Runtime.Incref(py_dict); + PyDict dict = new PyDict(py_dict); + + foreach (PyObject pyKey in dict.Keys()) { - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(method.ReturnType)); + PyObject value = dict[pyKey]; + if (value.HasAttr("_clr_return_type_") && value.HasAttr("_clr_arg_types_")) + { + string methodName = pyKey.ToString(); + + // if this method has already been redirected to the python method skip it + if (virtualMethods.Contains(methodName)) + continue; + + // Add the method to the type + AddPythonMethod(methodName, value, typeBuilder); + } } - il.Emit(OpCodes.Ret); } // add the destructor so the python object created in the constructor gets destroyed - mb = typeBuilder.DefineMethod("Finalize", - MethodAttributes.Family | - MethodAttributes.Virtual | - MethodAttributes.HideBySig, - CallingConventions.Standard, - typeof(void), - Type.EmptyTypes); - il = mb.GetILGenerator(); + MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize", + MethodAttributes.Family | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, + CallingConventions.Standard, + typeof(void), + Type.EmptyTypes); + ILGenerator il = methodBuilder.GetILGenerator(); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("Finalize")); il.Emit(OpCodes.Ldarg_0); @@ -290,9 +244,265 @@ internal static Type CreateDerivedType(string name, Assembly assembly = Assembly.GetAssembly(type); AssemblyManager.ScanAssembly(assembly); + AssemblyBuilder assemblyBuilder = assemblyBuilders[assemblyName]; + return type; } + /// + /// Add a constructor override that calls the python ctor after calling the base type constructor. + /// + /// constructor to be called before calling the python ctor + /// Python callable object + /// TypeBuilder for the new type the ctor is to be added to + private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuilder typeBuilder) + { + ParameterInfo[] parameters = ctor.GetParameters(); + Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + + // create a method for calling the original constructor + string baseCtorName = "_" + baseType.Name + "__cinit__"; + MethodBuilder methodBuilder = typeBuilder.DefineMethod(baseCtorName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + typeof(void), + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + for (int i = 0; i < parameters.Length; ++i) + il.Emit(OpCodes.Ldarg, i + 1); + il.Emit(OpCodes.Call, ctor); + il.Emit(OpCodes.Ret); + + // override the original method with a new one that dispatches to python + ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig, + ctor.CallingConvention, + parameterTypes); + il = cb.GetILGenerator(); + il.DeclareLocal(typeof(Object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, baseCtorName); + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(System.Object)); + il.Emit(OpCodes.Stloc_0); + for (int i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (parameterTypes[i].IsValueType) + il.Emit(OpCodes.Box, parameterTypes[i]); + il.Emit(OpCodes.Stelem, typeof(Object)); + } + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeCtor")); + il.Emit(OpCodes.Ret); + } + + /// + /// Add a virtual method override that checks for an override on the python instance + /// and calls it, otherwise fall back to the base class method. + /// + /// virtual method to be overriden + /// Python callable object + /// TypeBuilder for the new type the method is to be added to + private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuilder typeBuilder) + { + + ParameterInfo[] parameters = method.GetParameters(); + Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + + // create a method for calling the original method + string baseMethodName = "_" + baseType.Name + "__" + method.Name; + MethodBuilder methodBuilder = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + method.ReturnType, + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + for (int i = 0; i < parameters.Length; ++i) + il.Emit(OpCodes.Ldarg, i + 1); + il.Emit(OpCodes.Call, method); + il.Emit(OpCodes.Ret); + + // override the original method with a new one that dispatches to python + methodBuilder = typeBuilder.DefineMethod(method.Name, + MethodAttributes.Public | + MethodAttributes.ReuseSlot | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, + method.CallingConvention, + method.ReturnType, + parameterTypes); + il = methodBuilder.GetILGenerator(); + il.DeclareLocal(typeof(Object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, method.Name); + il.Emit(OpCodes.Ldstr, baseMethodName); + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(System.Object)); + il.Emit(OpCodes.Stloc_0); + for (int i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (parameterTypes[i].IsValueType) + il.Emit(OpCodes.Box, parameterTypes[i]); + il.Emit(OpCodes.Stelem, typeof(Object)); + } + il.Emit(OpCodes.Ldloc_0); + if (method.ReturnType == typeof(void)) + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid")); + } + else + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(method.ReturnType)); + } + il.Emit(OpCodes.Ret); + } + + /// + /// Python method may have the following function attributes set to control how they're exposed: + /// - _clr_return_type_ - method return type (required) + /// - _clr_arg_types_ - list of method argument types (required) + /// - _clr_method_name_ - method name, if different from the python method name (optional) + /// + /// Method name to add to the type + /// Python callable object + /// TypeBuilder for the new type the method/property is to be added to + private static void AddPythonMethod(string methodName, PyObject func, TypeBuilder typeBuilder) + { + if (func.HasAttr("_clr_method_name_")) + methodName = func.GetAttr("_clr_method_name_").ToString(); + + PyObject pyReturnType = func.GetAttr("_clr_return_type_"); + Type returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; + if (returnType == null) + returnType = typeof(void); + + PyObject pyArgTypes = func.GetAttr("_clr_arg_types_"); + if (!pyArgTypes.IsIterable()) + throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types"); + + List argTypes = new List(); + foreach (PyObject pyArgType in pyArgTypes) + { + Type argType = pyArgType.AsManagedObject(typeof(Type)) as Type; + if (argType == null) + throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types"); + argTypes.Add(argType); + } + + // add the method to call back into python + MethodAttributes methodAttribs = MethodAttributes.Public | + MethodAttributes.Virtual | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig; + + MethodBuilder methodBuilder = typeBuilder.DefineMethod(methodName, + methodAttribs, + returnType, + argTypes.ToArray()); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.DeclareLocal(typeof(Object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, methodName); + il.Emit(OpCodes.Ldnull); // don't fall back to the base type's method + il.Emit(OpCodes.Ldc_I4, argTypes.Count); + il.Emit(OpCodes.Newarr, typeof(System.Object)); + il.Emit(OpCodes.Stloc_0); + for (int i = 0; i < argTypes.Count; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (argTypes[i].IsValueType) + il.Emit(OpCodes.Box, argTypes[i]); + il.Emit(OpCodes.Stelem, typeof(Object)); + } + il.Emit(OpCodes.Ldloc_0); + if (returnType == typeof(void)) + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid")); + } + else + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(returnType)); + } + il.Emit(OpCodes.Ret); + } + + /// + /// Python properties may have the following function attributes set to control how they're exposed: + /// - _clr_property_type_ - property type (required) + /// + /// Property name to add to the type + /// Python property object + /// TypeBuilder for the new type the method/property is to be added to + private static void AddPythonProperty(string propertyName, PyObject func, TypeBuilder typeBuilder) + { + // add the method to call back into python + MethodAttributes methodAttribs = MethodAttributes.Public | + MethodAttributes.Virtual | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig | + MethodAttributes.SpecialName; + + PyObject pyPropertyType = func.GetAttr("_clr_property_type_"); + Type propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; + if (propertyType == null) + throw new ArgumentException("_clr_property_type must be a CLR type"); + + PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, + PropertyAttributes.None, + propertyType, + null); + + if (func.HasAttr("fget") && func.GetAttr("fget").IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("get_" + propertyName, + methodAttribs, + propertyType, + null); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeGetProperty").MakeGenericMethod(propertyType)); + il.Emit(OpCodes.Ret); + + propertyBuilder.SetGetMethod(methodBuilder); + } + + if (func.HasAttr("fset") && func.GetAttr("fset").IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("set_" + propertyName, + methodAttribs, + null, + new Type[]{propertyType}); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Ldarg_1); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeSetProperty").MakeGenericMethod(propertyType)); + il.Emit(OpCodes.Ret); + + propertyBuilder.SetSetMethod(methodBuilder); + } + } private static ModuleBuilder GetModuleBuilder(string assemblyName, string moduleName) { @@ -324,7 +534,6 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module return moduleBuilder; } - } // @@ -395,6 +604,9 @@ public static T InvokeMethod(IPythonDerivedType obj, string methodName, strin } } + if (origMethodName == null) + throw new NullReferenceException("Python object does not have a '" + methodName + "' method"); + return (T)obj.GetType().InvokeMember(origMethodName, BindingFlags.InvokeMethod, null, @@ -451,6 +663,9 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s } } + if (origMethodName == null) + throw new NullReferenceException("Python object does not have a '" + methodName + "' method"); + obj.GetType().InvokeMember(origMethodName, BindingFlags.InvokeMethod, null, @@ -458,6 +673,69 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s args); } + public static T InvokeGetProperty(IPythonDerivedType obj, string propertyName) + { + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + CLRObject self = (CLRObject)fi.GetValue(obj); + + if (null == self) + throw new NullReferenceException("Instance must be specified when getting a property"); + + List disposeList = new List(); + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + Runtime.Incref(self.pyHandle); + PyObject pyself = new PyObject(self.pyHandle); + disposeList.Add(pyself); + + PyObject pyvalue = pyself.GetAttr(propertyName); + disposeList.Add(pyvalue); + return (T)pyvalue.AsManagedObject(typeof(T)); + } + finally + { + foreach (PyObject x in disposeList) + { + if (x != null) + x.Dispose(); + } + Runtime.PyGILState_Release(gs); + } + } + + public static void InvokeSetProperty(IPythonDerivedType obj, string propertyName, T value) + { + FieldInfo fi = obj.GetType().GetField("__pyobj__"); + CLRObject self = (CLRObject)fi.GetValue(obj); + + if (null == self) + throw new NullReferenceException("Instance must be specified when setting a property"); + + List disposeList = new List(); + IntPtr gs = Runtime.PyGILState_Ensure(); + try + { + Runtime.Incref(self.pyHandle); + PyObject pyself = new PyObject(self.pyHandle); + disposeList.Add(pyself); + + PyObject pyvalue = new PyObject(Converter.ToPythonImplicit(value)); + disposeList.Add(pyvalue); + + pyself.SetAttr(propertyName, pyvalue); + } + finally + { + foreach (PyObject x in disposeList) + { + if (x != null) + x.Dispose(); + } + Runtime.PyGILState_Release(gs); + } + } + public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, Object[] args) { // call the base constructor diff --git a/pythonnet/src/runtime/converter.cs b/pythonnet/src/runtime/converter.cs index 4ceba15da..f8bfb1483 100644 --- a/pythonnet/src/runtime/converter.cs +++ b/pythonnet/src/runtime/converter.cs @@ -34,7 +34,7 @@ private Converter() {} static Type int64Type; static Type flagsType; static Type boolType; - //static Type typeType; + static Type typeType; static Converter () { nfi = NumberFormatInfo.InvariantInfo; @@ -45,7 +45,7 @@ static Converter () { doubleType = typeof(Double); flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); - //typeType = typeof(Type); + typeType = typeof(Type); } @@ -335,6 +335,57 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + // Conversion to 'Type' is done using the same mappings as above + // for objects. + + if (obType == typeType) + { + if (value == Runtime.PyStringType) + { + result = stringType; + return true; + } + + else if (value == Runtime.PyBoolType) + { + result = boolType; + return true; + } + + else if (value == Runtime.PyIntType) + { + result = int32Type; + return true; + } + + else if (value == Runtime.PyLongType) + { + result = int64Type; + return true; + } + + else if (value == Runtime.PyFloatType) + { + result = doubleType; + return true; + } + + else if (value == Runtime.PyListType || value == Runtime.PyTupleType) + { + result = typeof(object[]); + return true; + } + + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, + "value cannot be converted to Type" + ); + } + + return false; + } + return ToPrimitive(value, obType, out result, setError); } diff --git a/pythonnet/src/runtime/pyobject.cs b/pythonnet/src/runtime/pyobject.cs index 133b1d0a3..37aa65e78 100644 --- a/pythonnet/src/runtime/pyobject.cs +++ b/pythonnet/src/runtime/pyobject.cs @@ -10,6 +10,7 @@ using System; using System.Dynamic; using System.Linq.Expressions; +using System.Collections; namespace Python.Runtime { @@ -561,6 +562,21 @@ public PyObject GetIterator() { return new PyObject(r); } + /// + /// GetEnumerator Method + /// + /// + /// + /// Return a new PyIter object for the object. This allows any iterable + /// python object to be iterated over in C#. A PythonException will be + /// raised if the object is not iterable. + /// + + public IEnumerator GetEnumerator() + { + return new PyIter(this); + } + /// /// Invoke Method @@ -760,6 +776,22 @@ public bool IsCallable() { } + /// + /// IsIterable Method + /// + /// + /// + /// Returns true if the object is iterable object. This method + /// always succeeds. + /// + + public bool IsIterable() + { + return Runtime.PyIter_Check(obj); + } + + + /// /// IsTrue Method /// diff --git a/pythonnet/src/runtime/pysequence.cs b/pythonnet/src/runtime/pysequence.cs index 9b41c308b..855cad72e 100644 --- a/pythonnet/src/runtime/pysequence.cs +++ b/pythonnet/src/runtime/pysequence.cs @@ -158,15 +158,6 @@ public PyObject Repeat(int count) { } return new PyObject(op); } - - #region IEnumerable Members - - public IEnumerator GetEnumerator() - { - return new PyIter(this); - } - - #endregion } } diff --git a/pythonnet/src/runtime/pythonengine.cs b/pythonnet/src/runtime/pythonengine.cs index b60ba1644..d66cb0dbe 100644 --- a/pythonnet/src/runtime/pythonengine.cs +++ b/pythonnet/src/runtime/pythonengine.cs @@ -8,7 +8,9 @@ // ========================================================================== using System; +using System.IO; using System.Threading; +using System.Reflection; namespace Python.Runtime { @@ -143,10 +145,47 @@ public static void Initialize() { PyObject r = PythonEngine.RunString(code); if (r != null) r.Dispose(); + + // Load the clr.py resource into the clr module + IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); + IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + + IntPtr globals = Runtime.PyEval_GetGlobals(); + PyDict locals = new PyDict(); + try + { + IntPtr builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(locals.Handle, "__builtins__", builtins); + + var assembly = Assembly.GetExecutingAssembly(); + using (Stream stream = assembly.GetManifestResourceStream("Python.Runtime.resources.clr.py")) + using (StreamReader reader = new StreamReader(stream)) + { + // add the contents of clr.py to the module + string clr_py = reader.ReadToEnd(); + PyObject result = RunString(clr_py, globals, locals.Handle); + if (null == result) + throw new PythonException(); + result.Dispose(); + } + + foreach (PyObject key in locals.Keys()) + { + if (!key.ToString().StartsWith("_")){ + PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + value.Dispose(); + } + key.Dispose(); + } + } + finally + { + locals.Dispose(); + } } } - //==================================================================== // A helper to perform initialization from the context of an active // CPython interpreter process - this bootstraps the managed runtime @@ -157,41 +196,51 @@ public static IntPtr InitExt() { #else public static void InitExt() { #endif - Initialize(); - - // Trickery - when the import hook is installed into an already - // running Python, the standard import machinery is still in - // control for the duration of the import that caused bootstrap. - // - // That is problematic because the std machinery tries to get - // sub-names directly from the module __dict__ rather than going - // through our module object's getattr hook. This workaround is - // evil ;) We essentially climb up the stack looking for the - // import that caused the bootstrap to happen, then re-execute - // the import explicitly after our hook has been installed. By - // doing this, the original outer import should work correctly. - // - // Note that this is only needed during the execution of the - // first import that installs the CLR import hook. This hack - // still doesn't work if you use the interactive interpreter, - // since there is no line info to get the import line ;( - - string code = - - "import traceback\n" + - "for item in traceback.extract_stack():\n" + - " line = item[3]\n" + - " if line is not None:\n" + - " if line.startswith('import CLR') or \\\n" + - " line.startswith('import clr') or \\\n" + - " line.startswith('from clr') or \\\n" + - " line.startswith('from CLR'):\n" + - " exec(line)\n" + - " break\n"; - - PyObject r = PythonEngine.RunString(code); - if (r != null) { - r.Dispose(); + try + { + Initialize(); + + // Trickery - when the import hook is installed into an already + // running Python, the standard import machinery is still in + // control for the duration of the import that caused bootstrap. + // + // That is problematic because the std machinery tries to get + // sub-names directly from the module __dict__ rather than going + // through our module object's getattr hook. This workaround is + // evil ;) We essentially climb up the stack looking for the + // import that caused the bootstrap to happen, then re-execute + // the import explicitly after our hook has been installed. By + // doing this, the original outer import should work correctly. + // + // Note that this is only needed during the execution of the + // first import that installs the CLR import hook. This hack + // still doesn't work if you use the interactive interpreter, + // since there is no line info to get the import line ;( + + string code = + + "import traceback\n" + + "for item in traceback.extract_stack():\n" + + " line = item[3]\n" + + " if line is not None:\n" + + " if line.startswith('import CLR') or \\\n" + + " line.startswith('import clr') or \\\n" + + " line.startswith('from clr') or \\\n" + + " line.startswith('from CLR'):\n" + + " exec(line)\n" + + " break\n"; + + PyObject r = PythonEngine.RunString(code); + if (r != null) { + r.Dispose(); + } + } + catch (PythonException e) + { + e.Restore(); +#if (PYTHON32 || PYTHON33 || PYTHON34) + return IntPtr.Zero; +#endif } #if (PYTHON32 || PYTHON33 || PYTHON34) @@ -378,6 +427,18 @@ public static PyObject RunString(string code) { } return new PyObject(result); } + + public static PyObject RunString(string code, IntPtr globals, IntPtr locals) + { + IntPtr flag = (IntPtr)257; /* Py_file_input */ + IntPtr result = Runtime.PyRun_String(code, flag, globals, locals); + Runtime.Decref(locals); + if (result == IntPtr.Zero) + { + return null; + } + return new PyObject(result); + } } public static class Py diff --git a/pythonnet/src/runtime/pythonexception.cs b/pythonnet/src/runtime/pythonexception.cs index 592e9ea37..e85c17651 100644 --- a/pythonnet/src/runtime/pythonexception.cs +++ b/pythonnet/src/runtime/pythonexception.cs @@ -53,6 +53,18 @@ public PythonException() : base() Dispose(); } + /// + /// Restores python error. + /// + public void Restore() + { + IntPtr gs = PythonEngine.AcquireLock(); + Runtime.PyErr_Restore(_pyType, _pyValue, _pyTB); + _pyType = IntPtr.Zero; + _pyValue = IntPtr.Zero; + _pyTB = IntPtr.Zero; + PythonEngine.ReleaseLock(gs); + } /// /// PyType Property diff --git a/pythonnet/src/runtime/resources/clr.py b/pythonnet/src/runtime/resources/clr.py new file mode 100644 index 000000000..1fbd272b5 --- /dev/null +++ b/pythonnet/src/runtime/resources/clr.py @@ -0,0 +1,83 @@ +""" +Code in this module gets loaded into the main clr module. +""" + +class clrproperty(object): + """ + Property decorator for exposing python properties to .NET. + The property type must be specified as the only argument to clrproperty. + + e.g.:: + + class X(object): + @clrproperty(string) + def test(self): + return "x" + + Properties decorated this way can be called from .NET, e.g.:: + + dynamic x = getX(); // get an instance of X declared in Python + string z = x.test; // calls into python and returns "x" + """ + + def __init__(self, type_, fget=None, fset=None): + self.__name__ = getattr(fget, "__name__", None) + self._clr_property_type_ = type_ + self.fget = fget + self.fset = fset + + def __call__(self, fget): + return self.__class__(self._clr_property_type_, + fget=fget, + fset=self.fset) + + def setter(self, fset): + self.fset = fset + return self + + def getter(self, fget): + self.fget = fget + return self + + def __get__(self, instance, owner): + return self.fget.__get__(instance, owner)() + + def __set__(self, instance, value): + if not self.fset: + raise AttributeError("%s is read-only" % self.__name__) + return self.fset.__get__(instance, None)(value) + + +class clrmethod(object): + """ + Method decorator for exposing python methods to .NET. + The argument and return types must be specified as arguments to clrmethod. + + e.g.:: + + class X(object): + @clrmethod(int, [str]) + def test(self, x): + return len(x) + + Methods decorated this way can be called from .NET, e.g.:: + + dynamic x = getX(); // get an instance of X declared in Python + int z = x.test("hello"); // calls into python and returns len("hello") + """ + + def __init__(self, return_type, arg_types, clrname=None, func=None): + self.__name__ = getattr(func, "__name__", None) + self._clr_return_type_ = return_type + self._clr_arg_types_ = arg_types + self._clr_method_name_ = clrname or self.__name__ + self.__func = func + + def __call__(self, func): + return self.__class__(self._clr_return_type_, + self._clr_arg_types_, + clrname=self._clr_method_name_, + func=func) + + def __get__(self, instance, owner): + return self.__func.__get__(instance, owner) diff --git a/pythonnet/src/runtime/runtime.cs b/pythonnet/src/runtime/runtime.cs index 1c38ebaf5..9900c96f1 100644 --- a/pythonnet/src/runtime/runtime.cs +++ b/pythonnet/src/runtime/runtime.cs @@ -23,6 +23,19 @@ namespace Python.Runtime { [SuppressUnmanagedCodeSecurityAttribute()] + static class NativeMethods + { + [DllImport("kernel32.dll")] + public static extern IntPtr LoadLibrary(string dllToLoad); + + [DllImport("kernel32.dll")] + public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + + + [DllImport("kernel32.dll")] + public static extern bool FreeLibrary(IntPtr hModule); + } + public class Runtime { /// @@ -195,6 +208,13 @@ internal static void Initialize() { Error = new IntPtr(-1); +#if (PYTHON32 || PYTHON33 || PYTHON34) + IntPtr dll = NativeMethods.LoadLibrary(Runtime.dll); + _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dll, "_PyObject_NextNotImplemented"); + NativeMethods.FreeLibrary(dll); +#endif + + // Determine whether we need to wrap exceptions for versions of // of the Python runtime that do not allow new-style classes to // be used as exceptions (Python versions 2.4 and lower). @@ -272,6 +292,7 @@ internal static int AtExit() { internal static IntPtr PyNotImplemented; internal static int Py_EQ = 2; internal static int Py_NE = 3; + internal static IntPtr _PyObject_NextNotImplemented; #endif internal static IntPtr PyTrue; @@ -1650,6 +1671,11 @@ internal unsafe static extern int internal unsafe static extern int PyDict_DelItem(IntPtr pointer, IntPtr key); + [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, + ExactSpelling = true, CharSet = CharSet.Ansi)] + internal unsafe static extern int + PyDict_DelItemString(IntPtr pointer, string key); + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern int @@ -1793,10 +1819,20 @@ internal unsafe static extern int // Python iterator API //==================================================================== +#if !(PYTHON32 || PYTHON33 || PYTHON34) [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Ansi)] internal unsafe static extern bool PyIter_Check(IntPtr pointer); +#else + internal static bool + PyIter_Check(IntPtr pointer) + { + IntPtr ob_type = (IntPtr)Marshal.PtrToStructure(pointer + ObjectOffset.ob_type, typeof(IntPtr)); + IntPtr tp_iternext = ob_type + TypeOffset.tp_iternext; + return tp_iternext != null && tp_iternext != _PyObject_NextNotImplemented; + } +#endif [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Ansi)] diff --git a/pythonnet/src/runtime/typemanager.cs b/pythonnet/src/runtime/typemanager.cs index d51c60818..866bbbb78 100644 --- a/pythonnet/src/runtime/typemanager.cs +++ b/pythonnet/src/runtime/typemanager.cs @@ -246,6 +246,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr { Type subType = ClassDerivedObject.CreateDerivedType(name, baseClass.type, + py_dict, (string)namespaceStr, (string)assembly); From 02778b82d52c223e4f0c79e54e952ce09a7bd7fd Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 3 Sep 2014 15:07:08 +0100 Subject: [PATCH 31/77] Fix bug in __call__ implementation when calling a base class method on python subclass of a managed type. --- pythonnet/src/runtime/methodbinding.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pythonnet/src/runtime/methodbinding.cs b/pythonnet/src/runtime/methodbinding.cs index 0658e0095..64ab3edbe 100644 --- a/pythonnet/src/runtime/methodbinding.cs +++ b/pythonnet/src/runtime/methodbinding.cs @@ -161,13 +161,17 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { { string baseMethodName = "_" + baseType.type.Name + "__" + self.m.name; IntPtr baseMethod = Runtime.PyObject_GetAttrString(target, baseMethodName); - if (baseMethod != null) + if (baseMethod != IntPtr.Zero) { MethodBinding baseSelf = GetManagedObject(baseMethod) as MethodBinding; if (baseSelf != null) self = baseSelf; + Runtime.Decref(baseMethod); + } + else + { + Runtime.PyErr_Clear(); } - Runtime.Decref(baseMethod); } } } From 4c53587410a35208165f51b9e70fed970bb5e291 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 15 Sep 2014 13:50:49 +0100 Subject: [PATCH 32/77] fix setup.py to work with python3 --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 35c726d89..65657684b 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,10 @@ def _find_msbuild_tool(tool="msbuild.exe", use_windows_sdk=False): """Return full path to one of the microsoft build tools""" - import _winreg + try: + import _winreg + except ImportError: + import winreg as _winreg if use_windows_sdk: value_name = "InstallationFolder" From eb825dfd37e27afc842170f3d0fe0c437f67383b Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 15 Sep 2014 14:59:42 +0100 Subject: [PATCH 33/77] fix merge error (converting IEnumerable) --- src/runtime/converter.cs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 432d2ccd9..8a62baf42 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -92,16 +92,6 @@ internal static IntPtr ToPython(Object value, Type type) { return result; } - if (value is IEnumerable) - { - var resultlist = new PyList(); - foreach (object o in (IEnumerable)value) - { - resultlist.Append(new PyObject(ToPython(o, o.GetType()))); - } - return resultlist.Handle; - } - // it the type is a python subclass of a managed type then return the // underying python object rather than construct a new wrapper object. IPythonDerivedType pyderived = value as IPythonDerivedType; @@ -184,19 +174,17 @@ internal static IntPtr ToPython(Object value, Type type) { return Runtime.PyLong_FromUnsignedLongLong((ulong)value); default: - if (value is IEnumerable) - { + if (value is IEnumerable) { var resultlist = new PyList(); - foreach (object o in (IEnumerable)value) - { + foreach (object o in (IEnumerable)value) { resultlist.Append(new PyObject(ToPython(o, o.GetType()))); } return resultlist.Handle; } + result = CLRObject.GetInstHandle(value, type); return result; } - } From 61e83fe310b9dfb2a1c87f49be91d453bfe8e77a Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 4 Nov 2014 09:39:13 +0000 Subject: [PATCH 34/77] Update appveyor.yml add python 3.2 and 3.4 to test matrix --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 18f9761c0..3dd7c3c04 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,6 +9,10 @@ environment: - pythonurl: http://www.python.org/ftp/python/2.7.6/python-2.7.6.msi - pythonurl: http://www.python.org/ftp/python/2.6.6/python-2.6.6.msi - pythonurl: http://www.python.org/ftp/python/2.6.6/python-2.6.6.amd64.msi + - pythonurl: http://www.python.org/ftp/python/3.2.3/python-3.2.3.msi + - pythonurl: http://www.python.org/ftp/python/3.2.3/python-3.2.3.amd64.msi + - pythonurl: http://www.python.org/ftp/python/3.4.2/python-3.4.2.msi + - pythonurl: http://www.python.org/ftp/python/3.4.2/python-3.4.2.amd64.msi install: - ps: (new-object net.webclient).DownloadFile($env:pythonurl, 'C:\python.msi') From 3ba653e11881a1ad6f57574e61f385c880e1a8d0 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 4 Nov 2014 16:57:11 +0000 Subject: [PATCH 35/77] Only create a managed sub class if the python type has __namespace__ or __assembly__ set. This preserves the behavior from the develop branch while still making it possible to override .net methods in python if desired. --- src/runtime/metatype.cs | 43 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index f2e611902..0c4f8c3b5 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -88,7 +88,48 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { ); } - return TypeManager.CreateSubType(name, base_type, dict); + // If __assembly__ or __namespace__ are in the class dictionary then create + // a managed sub type. + // This creates a new managed type that can be used from .net to call back + // into python. + if (IntPtr.Zero != dict) { + Runtime.Incref(dict); + PyDict clsDict = new PyDict(dict); + if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) + return TypeManager.CreateSubType(name, base_type, dict); + } + + // otherwise just create a basic type without reflecting back into the managed side. + IntPtr func = Marshal.ReadIntPtr(Runtime.PyTypeType, + TypeOffset.tp_new); + IntPtr type = NativeCall.Call_3(func, tp, args, kw); + if (type == IntPtr.Zero) { + return IntPtr.Zero; + } + + int flags = TypeFlags.Default; + flags |= TypeFlags.Managed; + flags |= TypeFlags.HeapType; + flags |= TypeFlags.BaseType; + flags |= TypeFlags.Subclass; + flags |= TypeFlags.HaveGC; + Marshal.WriteIntPtr(type, TypeOffset.tp_flags, (IntPtr)flags); + + TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); + + // Hmm - the standard subtype_traverse, clear look at ob_size to + // do things, so to allow gc to work correctly we need to move + // our hidden handle out of ob_size. Then, in theory we can + // comment this out and still not crash. + TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); + TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); + + + // for now, move up hidden handle... + IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); + Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); + + return type; } From 30cc22a6590989b08835f312ad2d34e6f6a1eac0 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 4 Nov 2014 17:00:07 +0000 Subject: [PATCH 36/77] Add new tests for subclassing .net classes. --- src/testing/Python.Test.csproj | 1 + src/testing/subclasstest.cs | 79 ++++++++++++++++++++++ src/tests/test_subclass.py | 119 +++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 src/testing/subclasstest.cs create mode 100644 src/tests/test_subclass.py diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index b0c41a866..672a86300 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -120,6 +120,7 @@ + diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs new file mode 100644 index 000000000..64cea87c6 --- /dev/null +++ b/src/testing/subclasstest.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Python.Test +{ + public interface IInterfaceTest + { + // simple test with no arguments + string foo(); + + // test passing objects and boxing primitives + string bar(string s, int i); + } + + public class SubClassTest : IInterfaceTest + { + public SubClassTest() + { + } + + // simple test with no arguments + public virtual string foo() + { + return "foo"; + } + + // test passing objects and boxing primitives + public virtual string bar(string s, int i) + { + return s; + } + + // virtual methods that aren't overriden in python still work + public virtual string not_overriden() + { + return "not_overriden"; + } + + public virtual IList return_list() + { + return new List { "a", "b", "c" }; + } + + public static IList test_list(SubClassTest x) + { + // calls into python if return_list is overriden + return x.return_list(); + } + } + + public class TestFunctions + { + public static string test_foo(IInterfaceTest x) + { + // calls into python if foo is overriden + return x.foo(); + } + + public static string test_bar(IInterfaceTest x, string s, int i) + { + // calls into python if bar is overriden + return x.bar(s, i); + } + + // test instances can be constructed in managed code + public static IInterfaceTest create_instance(Type t) + { + return (IInterfaceTest)t.GetConstructor(new Type[] {}).Invoke(new Object[] {}); + } + + // test instances pass through managed code unchanged + public static IInterfaceTest pass_through(IInterfaceTest s) + { + return s; + } + } +} diff --git a/src/tests/test_subclass.py b/src/tests/test_subclass.py new file mode 100644 index 000000000..113397dba --- /dev/null +++ b/src/tests/test_subclass.py @@ -0,0 +1,119 @@ +# =========================================================================== +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# =========================================================================== +import clr +clr.AddReference('Python.Test') +clr.AddReference('System') + +import sys, os, string, unittest, types +from Python.Test import TestFunctions, SubClassTest, IInterfaceTest +from System.Collections.Generic import List + +# class that implements the test interface +class InterfaceTestClass(IInterfaceTest): + __namespace__ = "Python.Test" + + def foo(self): + return "InterfaceTestClass" + + def bar(self, x, i): + return "/".join([x] * i) + +# class that derives from a class deriving from IInterfaceTest +class DerivedClass(SubClassTest): + __namespace__ = "Python.Test" + + def foo(self): + return "DerivedClass" + + def base_foo(self): + return SubClassTest.foo(self) + + def super_foo(self): + return super(DerivedClass, self).foo() + + def bar(self, x, i): + return "_".join([x] * i) + + def return_list(self): + l = List[str]() + l.Add("A") + l.Add("B") + l.Add("C") + return l + +class SubClassTests(unittest.TestCase): + """Test subclassing managed types""" + + def testBaseClass(self): + """Test base class managed type""" + object = SubClassTest() + self.assertEqual(object.foo(), "foo") + self.assertEqual(TestFunctions.test_foo(object), "foo") + self.assertEqual(object.bar("bar", 2), "bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar") + self.assertEqual(object.not_overriden(), "not_overriden") + self.assertEqual(list(object.return_list()), ["a", "b", "c"]) + self.assertEqual(list(SubClassTest.test_list(object)), ["a", "b", "c"]) + + def testInterface(self): + """Test python classes can derive from C# interfaces""" + object = InterfaceTestClass() + self.assertEqual(object.foo(), "InterfaceTestClass") + self.assertEqual(TestFunctions.test_foo(object), "InterfaceTestClass") + self.assertEqual(object.bar("bar", 2), "bar/bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar/bar") + + x = TestFunctions.pass_through(object) + self.assertEqual(id(x), id(object)) + + def testDerivedClass(self): + """Test python class derived from managed type""" + object = DerivedClass() + self.assertEqual(object.foo(), "DerivedClass") + self.assertEqual(object.base_foo(), "foo") + self.assertEqual(object.super_foo(), "foo") + self.assertEqual(TestFunctions.test_foo(object), "DerivedClass") + self.assertEqual(object.bar("bar", 2), "bar_bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar_bar") + self.assertEqual(object.not_overriden(), "not_overriden") + self.assertEqual(list(object.return_list()), ["A", "B", "C"]) + self.assertEqual(list(SubClassTest.test_list(object)), ["A", "B", "C"]) + + x = TestFunctions.pass_through(object) + self.assertEqual(id(x), id(object)) + + def testCreateInstance(self): + """Test derived instances can be created from managed code""" + object = TestFunctions.create_instance(DerivedClass) + self.assertEqual(object.foo(), "DerivedClass") + self.assertEqual(TestFunctions.test_foo(object), "DerivedClass") + self.assertEqual(object.bar("bar", 2), "bar_bar") + self.assertEqual(TestFunctions.test_bar(object, "bar", 2), "bar_bar") + self.assertEqual(object.not_overriden(), "not_overriden") + + x = TestFunctions.pass_through(object) + self.assertEqual(id(x), id(object)) + + object2 = TestFunctions.create_instance(InterfaceTestClass) + self.assertEqual(object2.foo(), "InterfaceTestClass") + self.assertEqual(TestFunctions.test_foo(object2), "InterfaceTestClass") + self.assertEqual(object2.bar("bar", 2), "bar/bar") + self.assertEqual(TestFunctions.test_bar(object2, "bar", 2), "bar/bar") + + y = TestFunctions.pass_through(object2) + self.assertEqual(id(y), id(object2)) + +def test_suite(): + return unittest.makeSuite(SubClassTests) + +def main(): + unittest.TextTestRunner().run(test_suite()) + +if __name__ == '__main__': + main() From 263e2491a7788ad79401dddca08b167a585462ab Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 4 Nov 2014 17:09:28 +0000 Subject: [PATCH 37/77] Update tests for module.__file__ and __doc__ as they now both return something sensible. --- src/tests/test_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_module.py b/src/tests/test_module.py index 62ea78311..c9a4a2f24 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -62,8 +62,8 @@ def testModuleInterface(self): import System self.assertEquals(type(System.__dict__), type({})) self.assertEquals(System.__name__, 'System') - self.assertEquals(System.__file__, None) - self.assertEquals(System.__doc__, None) + self.assertTrue(System.__file__.endswith("System.dll")) + self.assertTrue(System.__doc__.startswith("Namespace containing types from the following assemblies:")) self.assertTrue(self.isCLRClass(System.String)) self.assertTrue(self.isCLRClass(System.Int32)) From 5354bc9ac5ac21b50b2e9a64ba538a6e35c4cb49 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 5 Nov 2014 10:09:25 +0000 Subject: [PATCH 38/77] Create a new module for importing the embedded clr.py file into as the globals dictionary doesn't exist when not being run inside a python process (i.e. when embedded). --- src/runtime/pythonengine.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index d66cb0dbe..d7fb10774 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -150,12 +150,13 @@ public static void Initialize() { IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); IntPtr clr_dict = Runtime.PyModule_GetDict(clr); - IntPtr globals = Runtime.PyEval_GetGlobals(); PyDict locals = new PyDict(); try { + IntPtr module = Runtime.PyImport_AddModule("clr._extras"); + IntPtr module_globals = Runtime.PyModule_GetDict(module); IntPtr builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(locals.Handle, "__builtins__", builtins); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); var assembly = Assembly.GetExecutingAssembly(); using (Stream stream = assembly.GetManifestResourceStream("Python.Runtime.resources.clr.py")) @@ -163,12 +164,15 @@ public static void Initialize() { { // add the contents of clr.py to the module string clr_py = reader.ReadToEnd(); - PyObject result = RunString(clr_py, globals, locals.Handle); + PyObject result = RunString(clr_py, module_globals, locals.Handle); if (null == result) throw new PythonException(); result.Dispose(); } + // add the imported module to the clr module, and copy the API functions + // and decorators into the main clr module. + Runtime.PyDict_SetItemString(clr_dict, "_extras", module); foreach (PyObject key in locals.Keys()) { if (!key.ToString().StartsWith("_")){ From 2019762596e9df1812249a5c80206e35a36b05fc Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 5 Nov 2014 10:31:05 +0000 Subject: [PATCH 39/77] add a couple of prints to unit test to see what's happening in travis-ci. --- src/tests/test_module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/test_module.py b/src/tests/test_module.py index c9a4a2f24..c1ed07192 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -62,7 +62,9 @@ def testModuleInterface(self): import System self.assertEquals(type(System.__dict__), type({})) self.assertEquals(System.__name__, 'System') + print (System.__file__) self.assertTrue(System.__file__.endswith("System.dll")) + print (System.__doc__) self.assertTrue(System.__doc__.startswith("Namespace containing types from the following assemblies:")) self.assertTrue(self.isCLRClass(System.String)) self.assertTrue(self.isCLRClass(System.Int32)) From 80b19e611bae491d093cd3e3d97de8c398a0e878 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 5 Nov 2014 10:45:08 +0000 Subject: [PATCH 40/77] change module.__file__ test as the filename can be any of the System dlls. --- src/tests/test_module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/test_module.py b/src/tests/test_module.py index c1ed07192..ce22e55a8 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -13,6 +13,7 @@ # testImplicitAssemblyLoad() passes on deprecation warning; perfect! # ##clr.AddReference('System.Windows.Forms') import sys, os, string, unittest, types, warnings +from fnmatch import fnmatch class ModuleTests(unittest.TestCase): @@ -62,9 +63,8 @@ def testModuleInterface(self): import System self.assertEquals(type(System.__dict__), type({})) self.assertEquals(System.__name__, 'System') - print (System.__file__) - self.assertTrue(System.__file__.endswith("System.dll")) - print (System.__doc__) + # the filename can be any module from the System namespace (eg System.Data.dll or System.dll) + self.assertTrue(fnmatch(System.__file__, "*System*.dll")) self.assertTrue(System.__doc__.startswith("Namespace containing types from the following assemblies:")) self.assertTrue(self.isCLRClass(System.String)) self.assertTrue(self.isCLRClass(System.Int32)) From f8db1b8906e07ed5e793df5b262f5df5d2ca6291 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 5 Nov 2014 17:22:22 +0000 Subject: [PATCH 41/77] Keep the clr module dictionary up to date in Python 3. Track assemblies with types in no namespace as well as those in namespaces. --- src/runtime/assemblymanager.cs | 6 ++--- src/runtime/importhook.cs | 41 +++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 07604ac00..35badb71a 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -291,8 +291,8 @@ internal static void ScanAssembly(Assembly assembly) { Type[] types = assembly.GetTypes(); for (int i = 0; i < types.Length; i++) { Type t = types[i]; - string ns = t.Namespace; - if ((ns != null) && (!namespaces.ContainsKey(ns))) { + string ns = t.Namespace ?? ""; + if (!namespaces.ContainsKey(ns)) { string[] names = ns.Split('.'); string s = ""; for (int n = 0; n < names.Length; n++) { @@ -367,7 +367,7 @@ public static List GetNames(string nsname) { Type[] types = a.GetTypes(); for (int i = 0; i < types.Length; i++) { Type t = types[i]; - if (t.Namespace == nsname) { + if ((t.Namespace ?? "") == nsname) { names.Add(t.Name); } } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index e6cedae44..670f24a2d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -92,9 +92,44 @@ internal static void Shutdown() { //=================================================================== // Return the clr python module (new reference) //=================================================================== - public static IntPtr GetCLRModule() { + public static IntPtr GetCLRModule(IntPtr? fromList=null) { root.InitializePreload(); #if (PYTHON32 || PYTHON33 || PYTHON34) + // update the module dictionary with the contents of the root dictionary + root.LoadNames(); + IntPtr py_mod_dict = Runtime.PyModule_GetDict(py_clr_module); + IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject** + clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr)); + Runtime.PyDict_Update(py_mod_dict, clr_dict); + + // find any items from the fromlist and get them from the root if they're not + // aleady in the module dictionary + if (fromList != null && fromList != IntPtr.Zero) { + if (Runtime.PyTuple_Check(fromList.GetValueOrDefault())) + { + Runtime.Incref(py_mod_dict); + PyDict mod_dict = new PyDict(py_mod_dict); + + Runtime.Incref(fromList.GetValueOrDefault()); + PyTuple from = new PyTuple(fromList.GetValueOrDefault()); + foreach (PyObject item in from) { + if (mod_dict.HasKey(item)) + continue; + + string s = item.AsManagedObject(typeof(string)) as string; + if (null == s) + continue; + + ManagedType attr = root.GetAttribute(s, true); + if (null == attr) + continue; + + Runtime.Incref(attr.pyHandle); + mod_dict.SetItem(s, new PyObject(attr.pyHandle)); + } + } + } + Runtime.Incref(py_clr_module); return py_clr_module; #else @@ -145,12 +180,12 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { // do the Incref()ed return here, since we've already found // the module. if (mod_name == "clr") { - return GetCLRModule(); + return GetCLRModule(fromList); } if (mod_name == "CLR") { Exceptions.deprecation("The CLR module is deprecated. " + "Please use 'clr'."); - return GetCLRModule(); + return GetCLRModule(fromList); } string realname = mod_name; if (mod_name.StartsWith("CLR.")) { From 4527dc94f99afd278e41d8a668f6c319dd13138c Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 5 Nov 2014 17:24:19 +0000 Subject: [PATCH 42/77] Add implementation of PyObject_Compare for Python 3. --- src/runtime/runtime.cs | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9900c96f1..0494485eb 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -290,8 +290,11 @@ internal static int AtExit() { #if (PYTHON32 || PYTHON33 || PYTHON34) internal static IntPtr PyBytesType; internal static IntPtr PyNotImplemented; - internal static int Py_EQ = 2; - internal static int Py_NE = 3; + internal const int Py_LT = 0; + internal const int Py_LE = 1; + internal const int Py_EQ = 2; + internal const int Py_NE = 3; + internal const int Py_GT = 4; internal static IntPtr _PyObject_NextNotImplemented; #endif @@ -849,10 +852,42 @@ internal unsafe static extern IntPtr internal unsafe static extern IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args); +#if (PYTHON32 || PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + internal unsafe static extern int + PyObject_RichCompareBool(IntPtr value1, IntPtr value2, int opid); + + internal static int PyObject_Compare(IntPtr value1, IntPtr value2) { + int res; + res = PyObject_RichCompareBool(value1, value2, Py_LT); + if (-1 == res) + return -1; + else if (1 == res) + return -1; + + res = PyObject_RichCompareBool(value1, value2, Py_EQ); + if (-1 == res) + return -1; + else if (1 == res) + return 0; + + res = PyObject_RichCompareBool(value1, value2, Py_GT); + if (-1 == res) + return -1; + else if (1 == res) + return 1; + + Exceptions.SetError(Exceptions.SystemError, "Error comparing objects"); + return -1; + } +#else [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] internal unsafe static extern int PyObject_Compare(IntPtr value1, IntPtr value2); +#endif + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] From 4bc6a60ef554dcda30472e0736a18823f649998c Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 15:02:52 +0000 Subject: [PATCH 43/77] update unittests for python 3 as well as python 2. --- src/tests/runtests.py | 4 +- src/tests/test_array.py | 60 ++++++++++++------- src/tests/test_class.py | 8 ++- src/tests/test_compat.py | 45 ++++++++++---- src/tests/test_constructors.py | 2 +- src/tests/test_conversion.py | 99 ++++++++++++++++--------------- src/tests/test_delegate.py | 8 ++- src/tests/test_enum.py | 27 +++++---- src/tests/test_exceptions.py | 34 +++++++---- src/tests/test_field.py | 20 ++++--- src/tests/test_generic.py | 103 +++++++++++++++++++-------------- src/tests/test_indexer.py | 33 ++++++----- src/tests/test_interface.py | 9 ++- src/tests/test_method.py | 66 ++++++++++++--------- src/tests/test_module.py | 47 ++++++++++----- src/tests/test_property.py | 12 +++- src/tests/test_thread.py | 12 +++- 17 files changed, 368 insertions(+), 221 deletions(-) diff --git a/src/tests/runtests.py b/src/tests/runtests.py index 452b701f8..60bf075bf 100644 --- a/src/tests/runtests.py +++ b/src/tests/runtests.py @@ -18,7 +18,7 @@ try: import System except ImportError: - print "Load clr import hook" + print("Load clr import hook") import clr test_modules = ( @@ -69,6 +69,6 @@ def main(verbosity=1): if __name__ == '__main__': main(1) if '--pause' in sys.argv: - print "Press enter to continue" + print("Press enter to continue") raw_input() diff --git a/src/tests/test_array.py b/src/tests/test_array.py index 3a2259e45..a545c1b4c 100644 --- a/src/tests/test_array.py +++ b/src/tests/test_array.py @@ -10,6 +10,11 @@ import sys, os, string, unittest, types import Python.Test as Test import System +import six + +if six.PY3: + long = int + unichr = chr class ArrayTests(unittest.TestCase): @@ -422,8 +427,8 @@ def testInt64Array(self): self.assertTrue(items[0] == 0) self.assertTrue(items[4] == 4) - max = 9223372036854775807L - min = -9223372036854775808L + max = long(9223372036854775807) + min = long(-9223372036854775808) items[0] = max self.assertTrue(items[0] == max) @@ -522,7 +527,7 @@ def testUInt32Array(self): self.assertTrue(items[0] == 0) self.assertTrue(items[4] == 4) - max = 4294967295L + max = long(4294967295) min = 0 items[0] = max @@ -572,7 +577,7 @@ def testUInt64Array(self): self.assertTrue(items[0] == 0) self.assertTrue(items[4] == 4) - max = 18446744073709551615L + max = long(18446744073709551615) min = 0 items[0] = max @@ -1056,7 +1061,7 @@ def testArrayIteration(self): empty = Test.NullArrayTest().empty for i in empty: - raise TypeError, 'iteration over empty array' + raise TypeError('iteration over empty array') def testTupleArrayConversion(self): @@ -1131,7 +1136,10 @@ def testSequenceArrayConversion(self): """Test conversion of sequence-like objects to array arguments.""" from Python.Test import ArrayConversionTest from Python.Test import Spam - from UserList import UserList + if six.PY3: + from collections import UserList + else: + from UserList import UserList items = UserList() for i in range(10): @@ -1146,7 +1154,10 @@ def testSequenceNestedArrayConversion(self): """Test conversion of sequences to array-of-array arguments.""" from Python.Test import ArrayConversionTest from Python.Test import Spam - from UserList import UserList + if six.PY3: + from collections import UserList + else: + from UserList import UserList items = UserList() for i in range(10): @@ -1235,7 +1246,10 @@ def testSequenceArrayConversionTypeChecking(self): """Test error handling for sequence conversion to array arguments.""" from Python.Test import ArrayConversionTest from Python.Test import Spam - from UserList import UserList + if six.PY3: + from collections import UserList + else: + from UserList import UserList # This should work, because null / None is a valid value in an # array of reference types. @@ -1353,9 +1367,9 @@ def testSpecialArrayCreation(self): self.assertTrue(value[1] == 127) self.assertTrue(value.Length == 2) - value = Array[System.Char]([u'A', u'Z']) - self.assertTrue(value[0] == u'A') - self.assertTrue(value[1] == u'Z') + value = Array[System.Char]([six.u('A'), six.u('Z')]) + self.assertTrue(value[0] == six.u('A')) + self.assertTrue(value[1] == six.u('Z')) self.assertTrue(value.Length == 2) value = Array[System.Char]([0, 65535]) @@ -1378,29 +1392,31 @@ def testSpecialArrayCreation(self): self.assertTrue(value[1] == 2147483647) self.assertTrue(value.Length == 2) - value = Array[System.Int64]([0, 9223372036854775807L]) + value = Array[System.Int64]([0, long(9223372036854775807)]) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 9223372036854775807L) + self.assertTrue(value[1] == long(9223372036854775807)) self.assertTrue(value.Length == 2) - value = Array[long]([0, 9223372036854775807L]) - self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 9223372036854775807L) - self.assertTrue(value.Length == 2) + # there's no explicit long type in python3, use System.Int64 instead + if not six.PY3: + value = Array[long]([0, long(9223372036854775807)]) + self.assertTrue(value[0] == 0) + self.assertTrue(value[1] == long(9223372036854775807)) + self.assertTrue(value.Length == 2) value = Array[System.UInt16]([0, 65000]) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 65000) + self.assertTrue(value[1] == 65000) self.assertTrue(value.Length == 2) - value = Array[System.UInt32]([0, 4294967295L]) + value = Array[System.UInt32]([0, long(4294967295)]) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 4294967295L) + self.assertTrue(value[1] == long(4294967295)) self.assertTrue(value.Length == 2) - value = Array[System.UInt64]([0, 18446744073709551615L]) + value = Array[System.UInt64]([0, long(18446744073709551615)]) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 18446744073709551615L) + self.assertTrue(value[1] == long(18446744073709551615)) self.assertTrue(value.Length == 2) value = Array[System.Single]([0.0, 3.402823e38]) diff --git a/src/tests/test_class.py b/src/tests/test_class.py index ecc335999..0f1085a6c 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -11,6 +11,12 @@ import sys, os, string, unittest, types import Python.Test as Test import System +import six + +if six.PY3: + DictProxyType = type(object.__dict__) +else: + DictProxyType = types.DictProxyType class ClassTests(unittest.TestCase): @@ -32,7 +38,7 @@ def testClassStandardAttrs(self): """Test standard class attributes.""" self.assertTrue(ClassTest.__name__ == 'ClassTest') self.assertTrue(ClassTest.__module__ == 'Python.Test') - self.assertTrue(type(ClassTest.__dict__) == types.DictProxyType) + self.assertTrue(type(ClassTest.__dict__) == DictProxyType) self.assertTrue(len(ClassTest.__doc__) > 0) diff --git a/src/tests/test_compat.py b/src/tests/test_compat.py index 7bb80d488..8bc75ddd9 100644 --- a/src/tests/test_compat.py +++ b/src/tests/test_compat.py @@ -8,6 +8,12 @@ # =========================================================================== import sys, os, string, unittest, types +import six + +if six.PY3: + ClassType = type +else: + ClassType = types.ClassType class CompatibilityTests(unittest.TestCase): @@ -19,8 +25,11 @@ def isCLRModule(self, object): return type(object).__name__ == 'ModuleObject' def isCLRRootModule(self, object): - return type(object).__name__ == 'CLRModule' - + if six.PY3: + # in Python 3 the clr module is a normal python module + return object.__name__ == "clr" + return type(object).__name__ == 'CLRModuleObject' + def isCLRClass(self, object): return type(object).__name__ == 'CLR Metatype' # for now @@ -36,9 +45,15 @@ def testSimpleImport(self): self.assertTrue(type(sys) == types.ModuleType) self.assertTrue(sys.__name__ == 'sys') - import httplib - self.assertTrue(type(httplib) == types.ModuleType) - self.assertTrue(httplib.__name__ == 'httplib') + if six.PY3: + import http.client + self.assertTrue(type(http.client) == types.ModuleType) + self.assertTrue(http.client.__name__ == 'http.client') + + else: + import httplib + self.assertTrue(type(httplib) == types.ModuleType) + self.assertTrue(httplib.__name__ == 'httplib') def testSimpleImportWithAlias(self): @@ -51,9 +66,15 @@ def testSimpleImportWithAlias(self): self.assertTrue(type(mySys) == types.ModuleType) self.assertTrue(mySys.__name__ == 'sys') - import httplib as myHttplib - self.assertTrue(type(myHttplib) == types.ModuleType) - self.assertTrue(myHttplib.__name__ == 'httplib') + if six.PY3: + import http.client as myHttplib + self.assertTrue(type(myHttplib) == types.ModuleType) + self.assertTrue(myHttplib.__name__ == 'http.client') + + else: + import httplib as myHttplib + self.assertTrue(type(myHttplib) == types.ModuleType) + self.assertTrue(myHttplib.__name__ == 'httplib') def testDottedNameImport(self): @@ -127,7 +148,7 @@ def testDottedNameImportFrom(self): self.assertTrue(pulldom.__name__ == 'xml.dom.pulldom') from xml.dom.pulldom import PullDOM - self.assertTrue(type(PullDOM) == types.ClassType) + self.assertTrue(type(PullDOM) == ClassType) self.assertTrue(PullDOM.__name__ == 'PullDOM') @@ -146,7 +167,7 @@ def testDottedNameImportFromWithAlias(self): self.assertTrue(myPulldom.__name__ == 'xml.dom.pulldom') from xml.dom.pulldom import PullDOM as myPullDOM - self.assertTrue(type(myPullDOM) == types.ClassType) + self.assertTrue(type(myPullDOM) == ClassType) self.assertTrue(myPullDOM.__name__ == 'PullDOM') @@ -178,7 +199,7 @@ def testExplicitAssemblyLoad(self): self.assertTrue(assembly != None) import CLR.System.Data - self.assertTrue(sys.modules.has_key('System.Data')) + self.assertTrue('System.Data' in sys.modules) assembly = Assembly.LoadWithPartialName('SpamSpamSpamSpamEggsAndSpam') self.assertTrue(assembly == None) @@ -265,7 +286,7 @@ def main(): try: import System except ImportError: - print "Load clr import hook" + print("Load clr import hook") import clr main() diff --git a/src/tests/test_constructors.py b/src/tests/test_constructors.py index 4486e50bd..593b8afd8 100644 --- a/src/tests/test_constructors.py +++ b/src/tests/test_constructors.py @@ -55,7 +55,7 @@ class sub(System.Exception): instance = sub() ob = SubclassConstructorTest(instance) - print ob + print(ob) self.assertTrue(isinstance(ob.value, System.Exception)) diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 8408f6fe3..961c7b9e8 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -10,6 +10,11 @@ import sys, os, string, unittest, types from Python.Test import ConversionTest import System +import six + +if six.PY3: + long = int + unichr = chr class ConversionTests(unittest.TestCase): @@ -176,16 +181,16 @@ def testCharConversion(self): self.assertTrue(System.Char.MinValue == unichr(0)) object = ConversionTest() - self.assertTrue(object.CharField == u'A') + self.assertTrue(object.CharField == six.u('A')) object.CharField = 'B' - self.assertTrue(object.CharField == u'B') + self.assertTrue(object.CharField == six.u('B')) - object.CharField = u'B' - self.assertTrue(object.CharField == u'B') + object.CharField = six.u('B') + self.assertTrue(object.CharField == six.u('B')) object.CharField = 67 - self.assertTrue(object.CharField == u'C') + self.assertTrue(object.CharField == six.u('C')) def test(): ConversionTest().CharField = 65536 @@ -307,23 +312,23 @@ def test(): def testInt64Conversion(self): """Test int64 conversion.""" - self.assertTrue(System.Int64.MaxValue == 9223372036854775807L) - self.assertTrue(System.Int64.MinValue == -9223372036854775808L) + self.assertTrue(System.Int64.MaxValue == long(9223372036854775807)) + self.assertTrue(System.Int64.MinValue == long(-9223372036854775808)) object = ConversionTest() self.assertTrue(object.Int64Field == 0) - object.Int64Field = 9223372036854775807L - self.assertTrue(object.Int64Field == 9223372036854775807L) + object.Int64Field = long(9223372036854775807) + self.assertTrue(object.Int64Field == long(9223372036854775807)) - object.Int64Field = -9223372036854775808L - self.assertTrue(object.Int64Field == -9223372036854775808L) + object.Int64Field = long(-9223372036854775808) + self.assertTrue(object.Int64Field == long(-9223372036854775808)) - object.Int64Field = System.Int64(9223372036854775807L) - self.assertTrue(object.Int64Field == 9223372036854775807L) + object.Int64Field = System.Int64(long(9223372036854775807)) + self.assertTrue(object.Int64Field == long(9223372036854775807)) - object.Int64Field = System.Int64(-9223372036854775808L) - self.assertTrue(object.Int64Field == -9223372036854775808L) + object.Int64Field = System.Int64(long(-9223372036854775808)) + self.assertTrue(object.Int64Field == long(-9223372036854775808)) def test(): ConversionTest().Int64Field = "spam" @@ -336,22 +341,22 @@ def test(): self.assertRaises(TypeError, test) def test(): - ConversionTest().Int64Field = 9223372036854775808L + ConversionTest().Int64Field = long(9223372036854775808) self.assertRaises(OverflowError, test) def test(): - ConversionTest().Int64Field = -9223372036854775809L + ConversionTest().Int64Field = long(-9223372036854775809) self.assertRaises(OverflowError, test) def test(): - value = System.Int64(9223372036854775808L) + value = System.Int64(long(9223372036854775808)) self.assertRaises(OverflowError, test) def test(): - value = System.Int64(-9223372036854775809L) + value = System.Int64(long(-9223372036854775809)) self.assertRaises(OverflowError, test) @@ -409,20 +414,20 @@ def test(): def testUInt32Conversion(self): """Test uint32 conversion.""" - self.assertTrue(System.UInt32.MaxValue == 4294967295L) + self.assertTrue(System.UInt32.MaxValue == long(4294967295)) self.assertTrue(System.UInt32.MinValue == 0) object = ConversionTest() self.assertTrue(object.UInt32Field == 0) - object.UInt32Field = 4294967295L - self.assertTrue(object.UInt32Field == 4294967295L) + object.UInt32Field = long(4294967295) + self.assertTrue(object.UInt32Field == long(4294967295)) object.UInt32Field = -0 self.assertTrue(object.UInt32Field == 0) - object.UInt32Field = System.UInt32(4294967295L) - self.assertTrue(object.UInt32Field == 4294967295L) + object.UInt32Field = System.UInt32(long(4294967295)) + self.assertTrue(object.UInt32Field == long(4294967295)) object.UInt32Field = System.UInt32(0) self.assertTrue(object.UInt32Field == 0) @@ -438,7 +443,7 @@ def test(): self.assertRaises(TypeError, test) def test(): - ConversionTest().UInt32Field = 4294967296L + ConversionTest().UInt32Field = long(4294967296) self.assertRaises(OverflowError, test) @@ -448,7 +453,7 @@ def test(): self.assertRaises(OverflowError, test) def test(): - value = System.UInt32(4294967296L) + value = System.UInt32(long(4294967296)) self.assertRaises(OverflowError, test) @@ -460,20 +465,20 @@ def test(): def testUInt64Conversion(self): """Test uint64 conversion.""" - self.assertTrue(System.UInt64.MaxValue == 18446744073709551615L) + self.assertTrue(System.UInt64.MaxValue == long(18446744073709551615)) self.assertTrue(System.UInt64.MinValue == 0) object = ConversionTest() self.assertTrue(object.UInt64Field == 0) - object.UInt64Field = 18446744073709551615L - self.assertTrue(object.UInt64Field == 18446744073709551615L) + object.UInt64Field = long(18446744073709551615) + self.assertTrue(object.UInt64Field == long(18446744073709551615)) object.UInt64Field = -0 self.assertTrue(object.UInt64Field == 0) - object.UInt64Field = System.UInt64(18446744073709551615L) - self.assertTrue(object.UInt64Field == 18446744073709551615L) + object.UInt64Field = System.UInt64(long(18446744073709551615)) + self.assertTrue(object.UInt64Field == long(18446744073709551615)) object.UInt64Field = System.UInt64(0) self.assertTrue(object.UInt64Field == 0) @@ -489,7 +494,7 @@ def test(): self.assertRaises(TypeError, test) def test(): - ConversionTest().UInt64Field = 18446744073709551616L + ConversionTest().UInt64Field = long(18446744073709551616) self.assertRaises(OverflowError, test) @@ -499,7 +504,7 @@ def test(): self.assertRaises(OverflowError, test) def test(): - value = System.UInt64(18446744073709551616L) + value = System.UInt64(long(18446744073709551616)) self.assertRaises(OverflowError, test) @@ -618,7 +623,7 @@ def testDecimalConversion(self): max_d = Decimal.Parse("79228162514264337593543950335") min_d = Decimal.Parse("-79228162514264337593543950335") - self.assertTrue(Decimal.ToInt64(Decimal(10)) == 10L) + self.assertTrue(Decimal.ToInt64(Decimal(10)) == long(10)) object = ConversionTest() self.assertTrue(object.DecimalField == Decimal(0)) @@ -659,25 +664,25 @@ def testStringConversion(self): object = ConversionTest() self.assertTrue(object.StringField == "spam") - self.assertTrue(object.StringField == u"spam") + self.assertTrue(object.StringField == six.u("spam")) object.StringField = "eggs" self.assertTrue(object.StringField == "eggs") - self.assertTrue(object.StringField == u"eggs") + self.assertTrue(object.StringField == six.u("eggs")) - object.StringField = u"spam" + object.StringField = six.u("spam") self.assertTrue(object.StringField == "spam") - self.assertTrue(object.StringField == u"spam") + self.assertTrue(object.StringField == six.u("spam")) - object.StringField = u'\uffff\uffff' - self.assertTrue(object.StringField == u'\uffff\uffff') + object.StringField = six.u('\uffff\uffff') + self.assertTrue(object.StringField == six.u('\uffff\uffff')) object.StringField = System.String("spam") self.assertTrue(object.StringField == "spam") - self.assertTrue(object.StringField == u"spam") + self.assertTrue(object.StringField == six.u("spam")) - object.StringField = System.String(u'\uffff\uffff') - self.assertTrue(object.StringField == u'\uffff\uffff') + object.StringField = System.String(six.u('\uffff\uffff')) + self.assertTrue(object.StringField == six.u('\uffff\uffff')) object.StringField = None self.assertTrue(object.StringField == None) @@ -829,11 +834,11 @@ def testByteArrayConversion(self): self.assertTrue(array[0] == 0) self.assertTrue(array[4] == 4) - value = "testing" + value = six.b("testing") object.ByteArrayField = value array = object.ByteArrayField for i in range(len(value)): - self.assertTrue(array[i] == ord(value[i])) + self.assertTrue(array[i] == six.indexbytes(value, i)) def testSByteArrayConversion(self): @@ -848,11 +853,11 @@ def testSByteArrayConversion(self): self.assertTrue(array[0] == 0) self.assertTrue(array[4] == 4) - value = "testing" + value = six.b("testing") object.SByteArrayField = value array = object.SByteArrayField for i in range(len(value)): - self.assertTrue(array[i] == ord(value[i])) + self.assertTrue(array[i] == six.indexbytes(value, i)) diff --git a/src/tests/test_delegate.py b/src/tests/test_delegate.py index 21c53ea3f..0d2315925 100644 --- a/src/tests/test_delegate.py +++ b/src/tests/test_delegate.py @@ -15,6 +15,12 @@ import sys, os, string, unittest, types import Python.Test as Test import System +import six + +if six.PY3: + DictProxyType = type(object.__dict__) +else: + DictProxyType = types.DictProxyType class DelegateTests(unittest.TestCase): @@ -24,7 +30,7 @@ def testDelegateStandardAttrs(self): """Test standard delegate attributes.""" self.assertTrue(PublicDelegate.__name__ == 'PublicDelegate') self.assertTrue(PublicDelegate.__module__ == 'Python.Test') - self.assertTrue(type(PublicDelegate.__dict__) == types.DictProxyType) + self.assertTrue(type(PublicDelegate.__dict__) == DictProxyType) self.assertTrue(PublicDelegate.__doc__ == None) diff --git a/src/tests/test_enum.py b/src/tests/test_enum.py index 98db3f3c6..26b14c274 100644 --- a/src/tests/test_enum.py +++ b/src/tests/test_enum.py @@ -10,6 +10,13 @@ import sys, os, string, unittest, types from System import DayOfWeek from Python import Test +import six + +if six.PY3: + DictProxyType = type(object.__dict__) + long = int +else: + DictProxyType = types.DictProxyType class EnumTests(unittest.TestCase): @@ -19,7 +26,7 @@ def testEnumStandardAttrs(self): """Test standard enum attributes.""" self.assertTrue(DayOfWeek.__name__ == 'DayOfWeek') self.assertTrue(DayOfWeek.__module__ == 'System') - self.assertTrue(type(DayOfWeek.__dict__) == types.DictProxyType) + self.assertTrue(type(DayOfWeek.__dict__) == DictProxyType) self.assertTrue(DayOfWeek.__doc__ == None) @@ -71,23 +78,23 @@ def testIntEnum(self): def testUIntEnum(self): """Test uint enum.""" - self.assertTrue(Test.UIntEnum.Zero == 0L) - self.assertTrue(Test.UIntEnum.One == 1L) - self.assertTrue(Test.UIntEnum.Two == 2L) + self.assertTrue(Test.UIntEnum.Zero == long(0)) + self.assertTrue(Test.UIntEnum.One == long(1)) + self.assertTrue(Test.UIntEnum.Two == long(2)) def testLongEnum(self): """Test long enum.""" - self.assertTrue(Test.LongEnum.Zero == 0L) - self.assertTrue(Test.LongEnum.One == 1L) - self.assertTrue(Test.LongEnum.Two == 2L) + self.assertTrue(Test.LongEnum.Zero == long(0)) + self.assertTrue(Test.LongEnum.One == long(1)) + self.assertTrue(Test.LongEnum.Two == long(2)) def testULongEnum(self): """Test ulong enum.""" - self.assertTrue(Test.ULongEnum.Zero == 0L) - self.assertTrue(Test.ULongEnum.One == 1L) - self.assertTrue(Test.ULongEnum.Two == 2L) + self.assertTrue(Test.ULongEnum.Zero == long(0)) + self.assertTrue(Test.ULongEnum.One == long(1)) + self.assertTrue(Test.ULongEnum.Two == long(2)) def testInstantiateEnumFails(self): diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index de6dd01e5..86a141403 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -9,6 +9,10 @@ import sys, os, string, unittest, types import System +import six + +if six.PY3: + unicode = str # Note: all of these tests are known to fail because Python currently # doesn't allow new-style classes to be used as exceptions. I'm leaving @@ -21,10 +25,11 @@ class ExceptionTests(unittest.TestCase): def testUnifiedExceptionSemantics(self): """Test unified exception semantics.""" from System import Exception, Object - import exceptions e = Exception('Something bad happened') - self.assertTrue(isinstance(e, exceptions.Exception)) + if not six.PY3: + import exceptions + self.assertTrue(isinstance(e, exceptions.Exception)) self.assertTrue(isinstance(e, Exception)) @@ -49,7 +54,6 @@ def testExtendedExceptionAttributes(self): """Test accessing extended exception attributes.""" from Python.Test import ExceptionTest, ExtendedException from System import Exception, OverflowException - import exceptions e = ExceptionTest.GetExtendedException() self.assertTrue(isinstance(e, ExtendedException)) @@ -93,7 +97,7 @@ def testRaiseClassExceptionWithValue(self): from System import NullReferenceException def test(): - raise NullReferenceException, 'Aiiieee!' + raise NullReferenceException('Aiiieee!') self.assertRaises(NullReferenceException, test) @@ -185,7 +189,8 @@ def testCatchExceptionFromManagedMethod(self): try: ExceptionTest().ThrowException() - except OverflowException, e: + except OverflowException: + e = sys.exc_info()[1] self.assertTrue(isinstance(e, OverflowException)) return @@ -199,13 +204,15 @@ def testCatchExceptionFromManagedProperty(self): try: v = ExceptionTest().ThrowProperty - except OverflowException, e: + except OverflowException: + e = sys.exc_info()[1] self.assertTrue(isinstance(e, OverflowException)) return try: ExceptionTest().ThrowProperty = 1 - except OverflowException, e: + except OverflowException: + e = sys.exc_info()[1] self.assertTrue(isinstance(e, OverflowException)) return @@ -227,7 +234,10 @@ def testCatchExceptionManagedClass(self): def testCatchExceptionPythonClass(self): """Test catching the python class of an exception.""" from System import OverflowException - from exceptions import Exception + if six.PY3: + from builtins import Exception + else: + from exceptions import Exception try: raise OverflowException('overflow') @@ -267,7 +277,8 @@ def testCatchExceptionWithAssignment(self): try: raise OverflowException('overflow') - except OverflowException, e: + except OverflowException: + e = sys.exc_info()[1] self.assertTrue(isinstance(e, OverflowException)) @@ -303,9 +314,10 @@ def testStrOfException(self): try: Convert.ToDateTime('this will fail') - except FormatException, e: + except FormatException: + e = sys.exc_info()[1] msg = unicode(e).encode("utf8") # fix for international installation - self.assertTrue(msg.find('System.Convert.ToDateTime') > -1, msg) + self.assertTrue(msg.find(unicode('System.Convert.ToDateTime').encode("utf8")) > -1, msg) def testPythonCompatOfManagedExceptions(self): diff --git a/src/tests/test_field.py b/src/tests/test_field.py index e266f65d1..1ec9c7744 100644 --- a/src/tests/test_field.py +++ b/src/tests/test_field.py @@ -11,6 +11,12 @@ from Python.Test import FieldTest from Python.Test import ShortEnum import System +import six + +if six.PY3: + IntType = int +else: + IntType = types.IntType class FieldTests(unittest.TestCase): @@ -212,15 +218,15 @@ def testFieldDescriptorGetSet(self): self.assertTrue(object.PublicStaticField == 0) descriptor = FieldTest.__dict__['PublicStaticField'] - self.assertTrue(type(descriptor) != types.IntType) + self.assertTrue(type(descriptor) != IntType) object.PublicStaticField = 0 descriptor = FieldTest.__dict__['PublicStaticField'] - self.assertTrue(type(descriptor) != types.IntType) + self.assertTrue(type(descriptor) != IntType) FieldTest.PublicStaticField = 0 descriptor = FieldTest.__dict__['PublicStaticField'] - self.assertTrue(type(descriptor) != types.IntType) + self.assertTrue(type(descriptor) != IntType) def testFieldDescriptorWrongType(self): @@ -286,15 +292,15 @@ def testByteField(self): def testCharField(self): """Test char fields.""" object = FieldTest() - self.assertTrue(object.CharField == u'A') + self.assertTrue(object.CharField == six.u('A')) self.assertTrue(object.CharField == 'A') object.CharField = 'B' - self.assertTrue(object.CharField == u'B') + self.assertTrue(object.CharField == six.u('B')) self.assertTrue(object.CharField == 'B') - object.CharField = u'C' - self.assertTrue(object.CharField == u'C') + object.CharField = six.u('C') + self.assertTrue(object.CharField == six.u('C')) self.assertTrue(object.CharField == 'C') diff --git a/src/tests/test_generic.py b/src/tests/test_generic.py index 256bca29a..d7ae2e26b 100644 --- a/src/tests/test_generic.py +++ b/src/tests/test_generic.py @@ -14,6 +14,13 @@ import sys, os, string, unittest, types import Python.Test as Test import System +import six + +if six.PY3: + long = int + unichr = chr + unicode = str + class GenericTests(unittest.TestCase): """Test CLR generics support.""" @@ -42,13 +49,13 @@ def testPythonTypeAliasing(self): dict = Dictionary[long, long]() self.assertEquals(dict.Count, 0) - dict.Add(1L, 1L) - self.assertTrue(dict[1L] == 1L) + dict.Add(long(1), long(1)) + self.assertTrue(dict[long(1)] == long(1)) dict = Dictionary[System.Int64, System.Int64]() self.assertEquals(dict.Count, 0) - dict.Add(1L, 1L) - self.assertTrue(dict[1L] == 1L) + dict.Add(long(1), long(1)) + self.assertTrue(dict[long(1)] == long(1)) dict = Dictionary[float, float]() self.assertEquals(dict.Count, 0) @@ -172,15 +179,17 @@ def testGenericTypeBinding(self): self._testGenericWrapperByType(bool, True) self._testGenericWrapperByType(System.Byte, 255) self._testGenericWrapperByType(System.SByte, 127) - self._testGenericWrapperByType(System.Char, u'A') + self._testGenericWrapperByType(System.Char, six.u('A')) self._testGenericWrapperByType(System.Int16, 32767) self._testGenericWrapperByType(System.Int32, 2147483647) self._testGenericWrapperByType(int, 2147483647) - self._testGenericWrapperByType(System.Int64, 9223372036854775807L) - self._testGenericWrapperByType(long, 9223372036854775807L) + self._testGenericWrapperByType(System.Int64, long(9223372036854775807)) + # Python 3 has no explicit long type, use System.Int64 instead + if not six.PY3: + self._testGenericWrapperByType(long, long(9223372036854775807)) self._testGenericWrapperByType(System.UInt16, 65000) - self._testGenericWrapperByType(System.UInt32, 4294967295L) - self._testGenericWrapperByType(System.UInt64, 18446744073709551615L) + self._testGenericWrapperByType(System.UInt32, long(4294967295)) + self._testGenericWrapperByType(System.UInt64, long(18446744073709551615)) self._testGenericWrapperByType(System.Single, 3.402823e38) self._testGenericWrapperByType(System.Double, 1.7976931348623157e308) self._testGenericWrapperByType(float, 1.7976931348623157e308) @@ -309,15 +318,17 @@ def testGenericMethodTypeHandling(self): self._testGenericMethodByType(bool, True) self._testGenericMethodByType(System.Byte, 255) self._testGenericMethodByType(System.SByte, 127) - self._testGenericMethodByType(System.Char, u'A') + self._testGenericMethodByType(System.Char, six.u('A')) self._testGenericMethodByType(System.Int16, 32767) self._testGenericMethodByType(System.Int32, 2147483647) self._testGenericMethodByType(int, 2147483647) - self._testGenericMethodByType(System.Int64, 9223372036854775807L) - self._testGenericMethodByType(long, 9223372036854775807L) + # Python 3 has no explicit long type, use System.Int64 instead + if not six.PY3: + self._testGenericMethodByType(System.Int64, long(9223372036854775807)) + self._testGenericMethodByType(long, long(9223372036854775807)) + self._testGenericMethodByType(System.UInt32, long(4294967295)) + self._testGenericMethodByType(System.Int64, long(1844674407370955161)) self._testGenericMethodByType(System.UInt16, 65000) - self._testGenericMethodByType(System.UInt32, 4294967295L) - self._testGenericMethodByType(System.Int64, 1844674407370955161L) self._testGenericMethodByType(System.Single, 3.402823e38) self._testGenericMethodByType(System.Double, 1.7976931348623157e308) self._testGenericMethodByType(float, 1.7976931348623157e308) @@ -439,9 +450,9 @@ def testMethodOverloadSelectionWithGenericTypes(self): self.assertTrue(value.value == 127) vtype = GenericWrapper[System.Char] - input = vtype(u'A') + input = vtype(six.u('A')) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value.value == u'A') + self.assertTrue(value.value == six.u('A')) vtype = GenericWrapper[System.Char] input = vtype(65535) @@ -464,14 +475,16 @@ def testMethodOverloadSelectionWithGenericTypes(self): self.assertTrue(value.value == 2147483647) vtype = GenericWrapper[System.Int64] - input = vtype(9223372036854775807L) + input = vtype(long(9223372036854775807)) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value.value == 9223372036854775807L) + self.assertTrue(value.value == long(9223372036854775807)) - vtype = GenericWrapper[long] - input = vtype(9223372036854775807L) - value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value.value == 9223372036854775807L) + # Python 3 has no explicit long type, use System.Int64 instead + if not six.PY3: + vtype = GenericWrapper[long] + input = vtype(long(9223372036854775807)) + value = MethodTest.Overloaded.__overloads__[vtype](input) + self.assertTrue(value.value == long(9223372036854775807)) vtype = GenericWrapper[System.UInt16] input = vtype(65000) @@ -479,14 +492,14 @@ def testMethodOverloadSelectionWithGenericTypes(self): self.assertTrue(value.value == 65000) vtype = GenericWrapper[System.UInt32] - input = vtype(4294967295L) + input = vtype(long(4294967295)) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value.value == 4294967295L) + self.assertTrue(value.value == long(4294967295)) vtype = GenericWrapper[System.UInt64] - input = vtype(18446744073709551615L) + input = vtype(long(18446744073709551615)) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value.value == 18446744073709551615L) + self.assertTrue(value.value == long(18446744073709551615)) vtype = GenericWrapper[System.Single] input = vtype(3.402823e38) @@ -580,9 +593,9 @@ def testOverloadSelectionWithArraysOfGenericTypes(self): gtype = GenericWrapper[System.Char] vtype = System.Array[gtype] - input = vtype([gtype(u'A'), gtype(u'A')]) + input = vtype([gtype(six.u('A')), gtype(six.u('A'))]) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0].value == u'A') + self.assertTrue(value[0].value == six.u('A')) self.assertTrue(value.Length == 2) gtype = GenericWrapper[System.Char] @@ -615,19 +628,21 @@ def testOverloadSelectionWithArraysOfGenericTypes(self): gtype = GenericWrapper[System.Int64] vtype = System.Array[gtype] - input = vtype([gtype(9223372036854775807L), - gtype(9223372036854775807L)]) - value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0].value == 9223372036854775807L) - self.assertTrue(value.Length == 2) - - gtype = GenericWrapper[long] - vtype = System.Array[gtype] - input = vtype([gtype(9223372036854775807L), - gtype(9223372036854775807L)]) + input = vtype([gtype(long(9223372036854775807)), + gtype(long(9223372036854775807))]) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0].value == 9223372036854775807L) + self.assertTrue(value[0].value == long(9223372036854775807)) self.assertTrue(value.Length == 2) + + # Python 3 has no explicit long type, use System.Int64 instead + if not six.PY3: + gtype = GenericWrapper[long] + vtype = System.Array[gtype] + input = vtype([gtype(long(9223372036854775807)), + gtype(long(9223372036854775807))]) + value = MethodTest.Overloaded.__overloads__[vtype](input) + self.assertTrue(value[0].value == long(9223372036854775807)) + self.assertTrue(value.Length == 2) gtype = GenericWrapper[System.UInt16] vtype = System.Array[gtype] @@ -638,17 +653,17 @@ def testOverloadSelectionWithArraysOfGenericTypes(self): gtype = GenericWrapper[System.UInt32] vtype = System.Array[gtype] - input = vtype([gtype(4294967295L), gtype(4294967295L)]) + input = vtype([gtype(long(4294967295)), gtype(long(4294967295))]) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0].value == 4294967295L) + self.assertTrue(value[0].value == long(4294967295)) self.assertTrue(value.Length == 2) gtype = GenericWrapper[System.UInt64] vtype = System.Array[gtype] - input = vtype([gtype(18446744073709551615L), - gtype(18446744073709551615L)]) + input = vtype([gtype(long(18446744073709551615)), + gtype(long(18446744073709551615))]) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0].value == 18446744073709551615L) + self.assertTrue(value[0].value == long(18446744073709551615)) self.assertTrue(value.Length == 2) gtype = GenericWrapper[System.Single] diff --git a/src/tests/test_indexer.py b/src/tests/test_indexer.py index 2b1d4e100..cb572e3a8 100644 --- a/src/tests/test_indexer.py +++ b/src/tests/test_indexer.py @@ -9,6 +9,11 @@ import sys, os, string, unittest, types import Python.Test as Test +import six + +if six.PY3: + long = int + unichr = chr class IndexerTests(unittest.TestCase): @@ -238,8 +243,8 @@ def test(): def testInt64Indexer(self): """Test Int64 indexers.""" object = Test.Int64IndexerTest() - max = 9223372036854775807L - min = -9223372036854775808L + max = long(9223372036854775807) + min = long(-9223372036854775808) self.assertTrue(object[max] == None) @@ -292,7 +297,7 @@ def test(): def testUInt32Indexer(self): """Test UInt32 indexers.""" object = Test.UInt32IndexerTest() - max = 4294967295L + max = long(4294967295) min = 0 self.assertTrue(object[max] == None) @@ -319,7 +324,7 @@ def test(): def testUInt64Indexer(self): """Test UInt64 indexers.""" object = Test.UInt64IndexerTest() - max = 18446744073709551615L + max = long(18446744073709551615) min = 0 self.assertTrue(object[max] == None) @@ -431,19 +436,19 @@ def testStringIndexer(self): object = Test.StringIndexerTest() self.assertTrue(object["spam"] == None) - self.assertTrue(object[u"spam"] == None) + self.assertTrue(object[six.u("spam")] == None) object["spam"] = "spam" self.assertTrue(object["spam"] == "spam") - self.assertTrue(object["spam"] == u"spam") - self.assertTrue(object[u"spam"] == "spam") - self.assertTrue(object[u"spam"] == u"spam") + self.assertTrue(object["spam"] == six.u("spam")) + self.assertTrue(object[six.u("spam")] == "spam") + self.assertTrue(object[six.u("spam")] == six.u("spam")) - object[u"eggs"] = u"eggs" + object[six.u("eggs")] = six.u("eggs") self.assertTrue(object["eggs"] == "eggs") - self.assertTrue(object["eggs"] == u"eggs") - self.assertTrue(object[u"eggs"] == "eggs") - self.assertTrue(object[u"eggs"] == u"eggs") + self.assertTrue(object["eggs"] == six.u("eggs")) + self.assertTrue(object[six.u("eggs")] == "eggs") + self.assertTrue(object[six.u("eggs")] == six.u("eggs")) def test(): object = Test.StringIndexerTest() @@ -509,8 +514,8 @@ def testObjectIndexer(self): object[1] = "one" self.assertTrue(object[1] == "one") - object[1L] = "long" - self.assertTrue(object[1L] == "long") + object[long(1)] = "long" + self.assertTrue(object[long(1)] == "long") def test(): class eggs: diff --git a/src/tests/test_interface.py b/src/tests/test_interface.py index 1e9c0ad96..4412aefb2 100644 --- a/src/tests/test_interface.py +++ b/src/tests/test_interface.py @@ -11,6 +11,13 @@ import sys, os, string, unittest, types import Python.Test as Test import System +import six + +if six.PY3: + DictProxyType = type(object.__dict__) +else: + DictProxyType = types.DictProxyType + class InterfaceTests(unittest.TestCase): """Test CLR interface support.""" @@ -20,7 +27,7 @@ def testInterfaceStandardAttrs(self): from Python.Test import IPublicInterface as ip self.assertTrue(ip.__name__ == 'IPublicInterface') self.assertTrue(ip.__module__ == 'Python.Test') - self.assertTrue(type(ip.__dict__) == types.DictProxyType) + self.assertTrue(type(ip.__dict__) == DictProxyType) def testGlobalInterfaceVisibility(self): diff --git a/src/tests/test_method.py b/src/tests/test_method.py index 03a23cf84..cdfc1e33b 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -13,6 +13,12 @@ from Python.Test import MethodTest, MethodTestSub import System +import six + +if six.PY3: + long = int + unichr = chr + class MethodTests(unittest.TestCase): """Test CLR method support.""" @@ -235,11 +241,11 @@ def testMethodCallStructConversion(self): def testSubclassInstanceConversion(self): """Test subclass instance conversion in method call.""" - class sub(System.Exception): + class TestSubException(System.Exception): pass object = MethodTest() - instance = sub() + instance = TestSubException() result = object.TestSubclassConversion(instance) self.assertTrue(isinstance(result, System.Exception)) @@ -501,8 +507,8 @@ def testExplicitOverloadSelection(self): value = MethodTest.Overloaded.__overloads__[System.SByte](127) self.assertTrue(value == 127) - value = MethodTest.Overloaded.__overloads__[System.Char](u'A') - self.assertTrue(value == u'A') + value = MethodTest.Overloaded.__overloads__[System.Char](six.u('A')) + self.assertTrue(value == six.u('A')) value = MethodTest.Overloaded.__overloads__[System.Char](65535) self.assertTrue(value == unichr(65535)) @@ -517,25 +523,27 @@ def testExplicitOverloadSelection(self): self.assertTrue(value == 2147483647) value = MethodTest.Overloaded.__overloads__[System.Int64]( - 9223372036854775807L + long(9223372036854775807) ) - self.assertTrue(value == 9223372036854775807L) + self.assertTrue(value == long(9223372036854775807)) - value = MethodTest.Overloaded.__overloads__[long]( - 9223372036854775807L - ) - self.assertTrue(value == 9223372036854775807L) + # Python 3 has no explicit long type, use System.Int64 instead + if not six.PY3: + value = MethodTest.Overloaded.__overloads__[long]( + long(9223372036854775807) + ) + self.assertTrue(value == long(9223372036854775807)) value = MethodTest.Overloaded.__overloads__[System.UInt16](65000) self.assertTrue(value == 65000) - value = MethodTest.Overloaded.__overloads__[System.UInt32](4294967295L) - self.assertTrue(value == 4294967295L) + value = MethodTest.Overloaded.__overloads__[System.UInt32](long(4294967295)) + self.assertTrue(value == long(4294967295)) value = MethodTest.Overloaded.__overloads__[System.UInt64]( - 18446744073709551615L + long(18446744073709551615) ) - self.assertTrue(value == 18446744073709551615L) + self.assertTrue(value == long(18446744073709551615)) value = MethodTest.Overloaded.__overloads__[System.Single](3.402823e38) self.assertTrue(value == 3.402823e38) @@ -617,10 +625,10 @@ def testOverloadSelectionWithArrayTypes(self): self.assertTrue(value[1] == 127) vtype = Array[System.Char] - input = vtype([u'A', u'Z']) + input = vtype([six.u('A'), six.u('Z')]) value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0] == u'A') - self.assertTrue(value[1] == u'Z') + self.assertTrue(value[0] == six.u('A')) + self.assertTrue(value[1] == six.u('Z')) vtype = Array[System.Char] input = vtype([0, 65535]) @@ -647,16 +655,18 @@ def testOverloadSelectionWithArrayTypes(self): self.assertTrue(value[1] == 2147483647) vtype = Array[System.Int64] - input = vtype([0, 9223372036854775807L]) + input = vtype([0, long(9223372036854775807)]) value = MethodTest.Overloaded.__overloads__[vtype](input) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 9223372036854775807L) + self.assertTrue(value[1] == long(9223372036854775807)) - vtype = Array[long] - input = vtype([0, 9223372036854775807L]) - value = MethodTest.Overloaded.__overloads__[vtype](input) - self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 9223372036854775807L) + # Python 3 has no explicit long type, use System.Int64 instead + if not six.PY3: + vtype = Array[long] + input = vtype([0, long(9223372036854775807)]) + value = MethodTest.Overloaded.__overloads__[vtype](input) + self.assertTrue(value[0] == 0) + self.assertTrue(value[1] == long(9223372036854775807)) vtype = Array[System.UInt16] input = vtype([0, 65000]) @@ -665,16 +675,16 @@ def testOverloadSelectionWithArrayTypes(self): self.assertTrue(value[1] == 65000) vtype = Array[System.UInt32] - input = vtype([0, 4294967295L]) + input = vtype([0, long(4294967295)]) value = MethodTest.Overloaded.__overloads__[vtype](input) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 4294967295L) + self.assertTrue(value[1] == long(4294967295)) vtype = Array[System.UInt64] - input = vtype([0, 18446744073709551615L]) + input = vtype([0, long(18446744073709551615)]) value = MethodTest.Overloaded.__overloads__[vtype](input) self.assertTrue(value[0] == 0) - self.assertTrue(value[1] == 18446744073709551615L) + self.assertTrue(value[1] == long(18446744073709551615)) vtype = Array[System.Single] input = vtype([0.0, 3.402823e38]) diff --git a/src/tests/test_module.py b/src/tests/test_module.py index ce22e55a8..b34e7b12b 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -14,6 +14,12 @@ ##clr.AddReference('System.Windows.Forms') import sys, os, string, unittest, types, warnings from fnmatch import fnmatch +import six + +if six.PY3: + ClassType = type +else: + ClassType = types.ClassType class ModuleTests(unittest.TestCase): @@ -23,7 +29,10 @@ def isCLRModule(self, object): return type(object).__name__ == 'ModuleObject' def isCLRRootModule(self, object): - return type(object).__name__ == 'CLRModule' + if six.PY3: + # in Python 3 the clr module is a normal python module + return object.__name__ == "clr" + return type(object).__name__ == 'CLRModuleObject' def isCLRClass(self, object): return type(object).__name__ == 'CLR Metatype' # for now @@ -80,9 +89,14 @@ def testSimpleImport(self): self.assertTrue(type(sys) == types.ModuleType) self.assertTrue(sys.__name__ == 'sys') - import httplib - self.assertTrue(type(httplib) == types.ModuleType) - self.assertTrue(httplib.__name__ == 'httplib') + if six.PY3: + import http.client as httplib + self.assertTrue(type(httplib) == types.ModuleType) + self.assertTrue(httplib.__name__ == 'http.client') + else: + import httplib + self.assertTrue(type(httplib) == types.ModuleType) + self.assertTrue(httplib.__name__ == 'httplib') def testSimpleImportWithAlias(self): @@ -95,9 +109,14 @@ def testSimpleImportWithAlias(self): self.assertTrue(type(mySys) == types.ModuleType) self.assertTrue(mySys.__name__ == 'sys') - import httplib as myHttplib - self.assertTrue(type(myHttplib) == types.ModuleType) - self.assertTrue(myHttplib.__name__ == 'httplib') + if six.PY3: + import http.client as myHttplib + self.assertTrue(type(myHttplib) == types.ModuleType) + self.assertTrue(myHttplib.__name__ == 'http.client') + else: + import httplib as myHttplib + self.assertTrue(type(myHttplib) == types.ModuleType) + self.assertTrue(myHttplib.__name__ == 'httplib') def testDottedNameImport(self): @@ -171,7 +190,7 @@ def testDottedNameImportFrom(self): self.assertTrue(pulldom.__name__ == 'xml.dom.pulldom') from xml.dom.pulldom import PullDOM - self.assertTrue(type(PullDOM) == types.ClassType) + self.assertTrue(type(PullDOM) == ClassType) self.assertTrue(PullDOM.__name__ == 'PullDOM') @@ -190,7 +209,7 @@ def testDottedNameImportFromWithAlias(self): self.assertTrue(myPulldom.__name__ == 'xml.dom.pulldom') from xml.dom.pulldom import PullDOM as myPullDOM - self.assertTrue(type(myPullDOM) == types.ClassType) + self.assertTrue(type(myPullDOM) == ClassType) self.assertTrue(myPullDOM.__name__ == 'PullDOM') @@ -235,7 +254,7 @@ def testExplicitAssemblyLoad(self): self.assertTrue(assembly != None) import System.Data - self.assertTrue(sys.modules.has_key('System.Data')) + self.assertTrue('System.Data' in sys.modules) assembly = Assembly.LoadWithPartialName('SpamSpamSpamSpamEggsAndSpam') self.assertTrue(assembly == None) @@ -342,10 +361,10 @@ def test_ClrListAssemblies(self): from clr import ListAssemblies verbose = list(ListAssemblies(True)) short = list(ListAssemblies(False)) - self.assertTrue(u'mscorlib' in short) - self.assertTrue(u'System' in short) - self.assertTrue('Culture=' in verbose[0]) - self.assertTrue('Version=' in verbose[0]) + self.assertTrue(six.u('mscorlib') in short) + self.assertTrue(six.u('System') in short) + self.assertTrue(six.u('Culture=') in verbose[0]) + self.assertTrue(six.u('Version=') in verbose[0]) def test_ClrAddReference(self): from clr import AddReference diff --git a/src/tests/test_property.py b/src/tests/test_property.py index 851ff8af0..4b00040ef 100644 --- a/src/tests/test_property.py +++ b/src/tests/test_property.py @@ -9,6 +9,12 @@ import sys, os, string, unittest, types from Python.Test import PropertyTest +import six + +if six.PY3: + IntType = int +else: + IntType = types.IntType class PropertyTests(unittest.TestCase): @@ -139,15 +145,15 @@ def testPropertyDescriptorGetSet(self): self.assertTrue(object.PublicStaticProperty == 0) descriptor = PropertyTest.__dict__['PublicStaticProperty'] - self.assertTrue(type(descriptor) != types.IntType) + self.assertTrue(type(descriptor) != IntType) object.PublicStaticProperty = 0 descriptor = PropertyTest.__dict__['PublicStaticProperty'] - self.assertTrue(type(descriptor) != types.IntType) + self.assertTrue(type(descriptor) != IntType) PropertyTest.PublicStaticProperty = 0 descriptor = PropertyTest.__dict__['PublicStaticProperty'] - self.assertTrue(type(descriptor) != types.IntType) + self.assertTrue(type(descriptor) != IntType) def testPropertyDescriptorWrongType(self): diff --git a/src/tests/test_thread.py b/src/tests/test_thread.py index 171efa3bb..22d4c9538 100644 --- a/src/tests/test_thread.py +++ b/src/tests/test_thread.py @@ -7,13 +7,19 @@ # FOR A PARTICULAR PURPOSE. # =========================================================================== -import sys, os, string, unittest, types, thread +import sys, os, string, unittest, types from Python.Test import ThreadTest +import six + +if six.PY3: + import _thread as thread +else: + import thread def dprint(msg): # Debugging helper to trace thread-related tests. - if 0: print msg + if 0: print(msg) class ThreadTests(unittest.TestCase): @@ -39,7 +45,7 @@ def testDoubleCallbackToPython(self): def testPythonThreadCallsToCLR(self): """Test calls by Python-spawned threads into managed code.""" # This test is very likely to hang if something is wrong ;) - import threading, thread, time + import threading, time from System import String done = [] From 3e522b85f4ba7bed419a7d85f81a4b6fa16707d7 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 15:08:10 +0000 Subject: [PATCH 44/77] Update .travis.yml add python 3 builds --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6355ce73d..597bf5a7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ language: python python: - 2.6 - 2.7 + - 3.2 + - 3.4 before_install: - sudo apt-get install software-properties-common - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ trusty main universe" @@ -12,6 +14,7 @@ before_install: - yes | sudo certmgr -ssl -m https://nugetgallery.blob.core.windows.net - yes | sudo certmgr -ssl -m https://nuget.org install: + - pip install six - python setup.py build_ext --inplace script: - export PYTHONPATH=`pwd` From fad7ba3bcc45249e364eb3255bfa5c61e7859f03 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 15:09:00 +0000 Subject: [PATCH 45/77] Update appveyor.yml install six (needed for tests) --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 3dd7c3c04..ebdfb5b83 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,6 +23,7 @@ install: - set PATH=C:\Python;%PATH% - C:\Python\python.exe c:\get-pip.py - C:\Python\Scripts\pip.exe install wheel + - C:\Python\Scripts\pip.exe install six build_script: - C:\python\python.exe setup.py bdist_wheel From c1faa646cfa7840b64af89b4f78a24f2ccd284c5 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 15:19:56 +0000 Subject: [PATCH 46/77] Update .travis.yml set PYTHONHOME so npython picks up the virtualenv --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 597bf5a7d..dbf7d1d31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,5 @@ install: - python setup.py build_ext --inplace script: - export PYTHONPATH=`pwd` + - export PYTHONHOME=`python -c "import sys; print(sys.exec_prefix)"` - ./npython src/tests/runtests.py From c73acfd2b018de05804a89c0fc41efcd9c21bd62 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 15:39:21 +0000 Subject: [PATCH 47/77] fix testing clr module type name in python 2 tests --- src/tests/test_compat.py | 2 +- src/tests/test_module.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_compat.py b/src/tests/test_compat.py index 8bc75ddd9..66c9cff16 100644 --- a/src/tests/test_compat.py +++ b/src/tests/test_compat.py @@ -28,7 +28,7 @@ def isCLRRootModule(self, object): if six.PY3: # in Python 3 the clr module is a normal python module return object.__name__ == "clr" - return type(object).__name__ == 'CLRModuleObject' + return type(object).__name__ == 'CLRModule' def isCLRClass(self, object): return type(object).__name__ == 'CLR Metatype' # for now diff --git a/src/tests/test_module.py b/src/tests/test_module.py index b34e7b12b..0d340652a 100644 --- a/src/tests/test_module.py +++ b/src/tests/test_module.py @@ -32,7 +32,7 @@ def isCLRRootModule(self, object): if six.PY3: # in Python 3 the clr module is a normal python module return object.__name__ == "clr" - return type(object).__name__ == 'CLRModuleObject' + return type(object).__name__ == 'CLRModule' def isCLRClass(self, object): return type(object).__name__ == 'CLR Metatype' # for now From b41eaaaf55c68bc3d7bc960bfb3bf18feeb79ac1 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 15:49:09 +0000 Subject: [PATCH 48/77] convert bytes to string in mono build when building for python 3 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 65657684b..17a07fcd8 100644 --- a/setup.py +++ b/setup.py @@ -267,6 +267,8 @@ def _check_output(*popenargs, **kwargs): if cmd is None: cmd = popenargs[0] raise CalledProcessError(retcode, cmd, output=output) + if sys.version_info[2] > 2: + return output.decode("ascii") return output From ce0545fc9e9b37a00169992f4fead677c9201e65 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 6 Nov 2014 16:45:07 +0000 Subject: [PATCH 49/77] Use unicode functions instead of string functions when building for python 3 --- src/monoclr/pynetinit.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/monoclr/pynetinit.c b/src/monoclr/pynetinit.c index eaa1d9c8b..2f8b75848 100644 --- a/src/monoclr/pynetinit.c +++ b/src/monoclr/pynetinit.c @@ -15,6 +15,10 @@ #include "dirent.h" #endif +#if PY_MAJOR_VERSION > 2 +#include +#endif + // initialize Mono and PythonNet PyNet_Args* PyNet_Init(int ext) { PyNet_Args *pn_args; @@ -102,11 +106,25 @@ void main_thread_handler (gpointer user_data) { int ii = 0; for (ii = 0; ii < PyList_Size(syspath); ++ii) { +#if PY_MAJOR_VERSION > 2 + Py_ssize_t wlen; + wchar_t *wstr = PyUnicode_AsWideCharString(PyList_GetItem(syspath, ii), &wlen); + char* pydir = (char*)malloc(wlen + 1); + size_t mblen = wcstombs(pydir, wstr, wlen + 1); + if (mblen > wlen) + pydir[wlen] = '\0'; + PyMem_Free(wstr); +#else const char* pydir = PyString_AsString(PyList_GetItem(syspath, ii)); +#endif char* curdir = (char*) malloc(1024); strncpy(curdir, strlen(pydir) > 0 ? pydir : ".", 1024); strncat(curdir, slash, 1024); +#if PY_MAJOR_VERSION > 2 + free(pydir); +#endif + //look in this directory for the pn_args->pr_file DIR* dirp = opendir(curdir); if (dirp != NULL) { From 6eb59ecb85a5afe605205c5fbc6abe01f543735f Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 7 Nov 2014 15:39:06 +0000 Subject: [PATCH 50/77] Fix Py_Main for python 3 --- src/runtime/runtime.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 0494485eb..cb304c461 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -546,10 +546,17 @@ internal unsafe static extern void internal unsafe static extern IntPtr PyGILState_GetThisThreadState(); +#if (PYTHON32 || PYTHON33 || PYTHON34) + [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, + ExactSpelling=true, CharSet=CharSet.Ansi)] + public unsafe static extern int + Py_Main(int argc, [MarshalAsAttribute(UnmanagedType.LPArray, ArraySubType=UnmanagedType.LPWStr)] string[] argv); +#else [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] public unsafe static extern int Py_Main(int argc, string[] argv); +#endif [DllImport(Runtime.dll, CallingConvention=CallingConvention.Cdecl, ExactSpelling=true, CharSet=CharSet.Ansi)] From 5062377296e3594d461ff9459b85c5eb9d660957 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 10 Nov 2014 10:32:19 +0000 Subject: [PATCH 51/77] Changes to the monoclr build so it builds for python 3. --- setup.py | 11 +++++ src/monoclr/clrmod.c | 57 ++++++++++++++++------ src/monoclr/python.c | 22 +++++++++ src/runtime/runtime.cs | 104 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 170 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 17a07fcd8..df3b68e5c 100644 --- a/setup.py +++ b/setup.py @@ -115,6 +115,17 @@ def build_extension(self, ext): if CONFIG == "Debug": defines.extend(["DEBUG", "TRACE"]) + if sys.platform != "win32" and DEVTOOLS == "Mono": + defines.append("MONO_LINUX") + + if hasattr(sys, "abiflags"): + if "d" in sys.abiflags: + defines.append("PYTHON_WITH_PYDEBUG") + if "m" in sys.abiflags: + defines.append("PYTHON_WITH_PYMALLOC") + if "u" in sys.abiflags: + defines.append("PYTHON_WITH_WIDE_UNICODE") + cmd = [ _xbuild, "pythonnet.sln", diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c index 8b809b28f..580dc7cc2 100644 --- a/src/monoclr/clrmod.c +++ b/src/monoclr/clrmod.c @@ -25,21 +25,50 @@ PyDoc_STRVAR(clr_module_doc, static PyNet_Args *pn_args; char** environ = NULL; +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef clrdef = { + PyModuleDef_HEAD_INIT, + "clr", /* m_name */ + clr_module_doc, /* m_doc */ + -1, /* m_size */ + clr_methods, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; +#endif + +static PyObject *_initclr() { + PyObject *m; + + /* Create the module and add the functions */ +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&clrdef); +#else + m = Py_InitModule3("clr", clr_methods, clr_module_doc); +#endif + if (m == NULL) + return; + PyModule_AddObject(m, "facade", Py_True); + Py_INCREF(Py_True); + + pn_args = PyNet_Init(1); + if (pn_args->error) { + return NULL; + } + return m; +} + +#if PY_MAJOR_VERSION >= 3 +PyMODINIT_FUNC +PyInit_clr(void) { + return _initclr(); +} +#else PyMODINIT_FUNC initclr(void) -{ - PyObject *m; - - /* Create the module and add the functions */ - m = Py_InitModule3("clr", clr_methods, clr_module_doc); - if (m == NULL) - return; - PyModule_AddObject(m, "facade", Py_True); - Py_INCREF(Py_True); - - pn_args = PyNet_Init(1); - if (pn_args->error) { - return; - } + _initclr(); } +#endif diff --git a/src/monoclr/python.c b/src/monoclr/python.c index aa340491f..f90b9b540 100644 --- a/src/monoclr/python.c +++ b/src/monoclr/python.c @@ -16,7 +16,29 @@ #include +#if (PY_MAJOR_VERSION > 2) +#include +#endif + int main(int argc, char **argv) { +#if (PY_MAJOR_VERSION > 2) + int i, result; + size_t len; + wchar_t **wargv = (wchar_t**)malloc(sizeof(wchar_t*)*argc); + for (i=0; i internal static void Initialize() { - + is32bit = IntPtr.Size == 4; if (0 == Runtime.Py_IsInitialized()) @@ -211,8 +293,10 @@ internal static void Initialize() { #if (PYTHON32 || PYTHON33 || PYTHON34) IntPtr dll = NativeMethods.LoadLibrary(Runtime.dll); _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dll, "_PyObject_NextNotImplemented"); +#if !MONO_LINUX NativeMethods.FreeLibrary(dll); #endif +#endif // Determine whether we need to wrap exceptions for versions of From 195513b5f6e062dfa1e8f367e520309cb510e34f Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 10 Nov 2014 11:10:43 +0000 Subject: [PATCH 52/77] Fix copy and paste error in python 2 branch of the mono code. --- src/monoclr/clrmod.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c index 580dc7cc2..dd68696bc 100644 --- a/src/monoclr/clrmod.c +++ b/src/monoclr/clrmod.c @@ -49,7 +49,7 @@ static PyObject *_initclr() { m = Py_InitModule3("clr", clr_methods, clr_module_doc); #endif if (m == NULL) - return; + return NULL; PyModule_AddObject(m, "facade", Py_True); Py_INCREF(Py_True); @@ -67,7 +67,7 @@ PyInit_clr(void) { } #else PyMODINIT_FUNC -initclr(void) +initclr(void) { _initclr(); } #endif From 66f044992f9925bc8d3570124ee5612f2588f40e Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 10 Nov 2014 12:56:23 +0000 Subject: [PATCH 53/77] bug in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df3b68e5c..37ac3a661 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def _check_output(*popenargs, **kwargs): if cmd is None: cmd = popenargs[0] raise CalledProcessError(retcode, cmd, output=output) - if sys.version_info[2] > 2: + if sys.version_info[0] > 2: return output.decode("ascii") return output From ee15bf434940c0895c3de770f458a46be0d4c2a7 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 26 Nov 2014 15:45:14 +0000 Subject: [PATCH 54/77] Fix reference counting bug in Converter.ToPython. --- src/runtime/converter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 8a62baf42..fdfc2a429 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -179,6 +179,7 @@ internal static IntPtr ToPython(Object value, Type type) { foreach (object o in (IEnumerable)value) { resultlist.Append(new PyObject(ToPython(o, o.GetType()))); } + Runtime.Incref(resultlist.Handle); return resultlist.Handle; } From 150a900b4ae816fc17a962b4e08f549009d501a9 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 27 Nov 2014 10:57:28 +0000 Subject: [PATCH 55/77] Make sure PyObject instances are disposed cleanly. Waiting for the GC can cause problems as unreachable objects may be collected by Python. --- src/runtime/classderived.cs | 251 +++++++++++++++++------------------- src/runtime/converter.cs | 14 +- 2 files changed, 128 insertions(+), 137 deletions(-) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ae18d8551..2d44467a7 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -164,18 +164,20 @@ internal static Type CreateDerivedType(string name, if (py_dict != IntPtr.Zero && Runtime.PyDict_Check(py_dict)) { Runtime.Incref(py_dict); - PyDict dict = new PyDict(py_dict); - - foreach (PyObject pyKey in dict.Keys()) + using (PyDict dict = new PyDict(py_dict)) + using (PyObject keys = dict.Keys()) { - PyObject value = dict[pyKey]; - if (value.HasAttr("_clr_property_type_")) + foreach (PyObject pyKey in keys) { - string propertyName = pyKey.ToString(); - pyProperties.Add(propertyName); + using (PyObject value = dict[pyKey]) + if (value.HasAttr("_clr_property_type_")) + { + string propertyName = pyKey.ToString(); + pyProperties.Add(propertyName); - // Add the property to the type - AddPythonProperty(propertyName, value, typeBuilder); + // Add the property to the type + AddPythonProperty(propertyName, value, typeBuilder); + } } } } @@ -204,21 +206,23 @@ internal static Type CreateDerivedType(string name, if (py_dict != IntPtr.Zero && Runtime.PyDict_Check(py_dict)) { Runtime.Incref(py_dict); - PyDict dict = new PyDict(py_dict); - - foreach (PyObject pyKey in dict.Keys()) + using (PyDict dict = new PyDict(py_dict)) + using (PyObject keys = dict.Keys()) { - PyObject value = dict[pyKey]; - if (value.HasAttr("_clr_return_type_") && value.HasAttr("_clr_arg_types_")) + foreach (PyObject pyKey in keys) { - string methodName = pyKey.ToString(); + using (PyObject value = dict[pyKey]) + if (value.HasAttr("_clr_return_type_") && value.HasAttr("_clr_arg_types_")) + { + string methodName = pyKey.ToString(); - // if this method has already been redirected to the python method skip it - if (virtualMethods.Contains(methodName)) - continue; + // if this method has already been redirected to the python method skip it + if (virtualMethods.Contains(methodName)) + continue; - // Add the method to the type - AddPythonMethod(methodName, value, typeBuilder); + // Add the method to the type + AddPythonMethod(methodName, value, typeBuilder); + } } } } @@ -386,62 +390,64 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde if (func.HasAttr("_clr_method_name_")) methodName = func.GetAttr("_clr_method_name_").ToString(); - PyObject pyReturnType = func.GetAttr("_clr_return_type_"); - Type returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; - if (returnType == null) - returnType = typeof(void); - - PyObject pyArgTypes = func.GetAttr("_clr_arg_types_"); - if (!pyArgTypes.IsIterable()) - throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types"); - - List argTypes = new List(); - foreach (PyObject pyArgType in pyArgTypes) + using (PyObject pyReturnType = func.GetAttr("_clr_return_type_")) + using (PyObject pyArgTypes = func.GetAttr("_clr_arg_types_")) { - Type argType = pyArgType.AsManagedObject(typeof(Type)) as Type; - if (argType == null) + Type returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; + if (returnType == null) + returnType = typeof(void); + + if (!pyArgTypes.IsIterable()) throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types"); - argTypes.Add(argType); - } - // add the method to call back into python - MethodAttributes methodAttribs = MethodAttributes.Public | - MethodAttributes.Virtual | - MethodAttributes.ReuseSlot | - MethodAttributes.HideBySig; + List argTypes = new List(); + foreach (PyObject pyArgType in pyArgTypes) + { + Type argType = pyArgType.AsManagedObject(typeof(Type)) as Type; + if (argType == null) + throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types"); + argTypes.Add(argType); + } - MethodBuilder methodBuilder = typeBuilder.DefineMethod(methodName, - methodAttribs, - returnType, - argTypes.ToArray()); + // add the method to call back into python + MethodAttributes methodAttribs = MethodAttributes.Public | + MethodAttributes.Virtual | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig; - ILGenerator il = methodBuilder.GetILGenerator(); - il.DeclareLocal(typeof(Object[])); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, methodName); - il.Emit(OpCodes.Ldnull); // don't fall back to the base type's method - il.Emit(OpCodes.Ldc_I4, argTypes.Count); - il.Emit(OpCodes.Newarr, typeof(System.Object)); - il.Emit(OpCodes.Stloc_0); - for (int i = 0; i < argTypes.Count; ++i) - { + MethodBuilder methodBuilder = typeBuilder.DefineMethod(methodName, + methodAttribs, + returnType, + argTypes.ToArray()); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.DeclareLocal(typeof(Object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, methodName); + il.Emit(OpCodes.Ldnull); // don't fall back to the base type's method + il.Emit(OpCodes.Ldc_I4, argTypes.Count); + il.Emit(OpCodes.Newarr, typeof(System.Object)); + il.Emit(OpCodes.Stloc_0); + for (int i = 0; i < argTypes.Count; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (argTypes[i].IsValueType) + il.Emit(OpCodes.Box, argTypes[i]); + il.Emit(OpCodes.Stelem, typeof(Object)); + } il.Emit(OpCodes.Ldloc_0); - il.Emit(OpCodes.Ldc_I4, i); - il.Emit(OpCodes.Ldarg, i + 1); - if (argTypes[i].IsValueType) - il.Emit(OpCodes.Box, argTypes[i]); - il.Emit(OpCodes.Stelem, typeof(Object)); - } - il.Emit(OpCodes.Ldloc_0); - if (returnType == typeof(void)) - { - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid")); - } - else - { - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(returnType)); + if (returnType == typeof(void)) + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid")); + } + else + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(returnType)); + } + il.Emit(OpCodes.Ret); } - il.Emit(OpCodes.Ret); } /// @@ -460,47 +466,49 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu MethodAttributes.HideBySig | MethodAttributes.SpecialName; - PyObject pyPropertyType = func.GetAttr("_clr_property_type_"); - Type propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; - if (propertyType == null) - throw new ArgumentException("_clr_property_type must be a CLR type"); - - PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, - PropertyAttributes.None, - propertyType, - null); - - if (func.HasAttr("fget") && func.GetAttr("fget").IsTrue()) + using (PyObject pyPropertyType = func.GetAttr("_clr_property_type_")) { - MethodBuilder methodBuilder = typeBuilder.DefineMethod("get_" + propertyName, - methodAttribs, - propertyType, - null); - - ILGenerator il = methodBuilder.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, propertyName); - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeGetProperty").MakeGenericMethod(propertyType)); - il.Emit(OpCodes.Ret); + Type propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; + if (propertyType == null) + throw new ArgumentException("_clr_property_type must be a CLR type"); - propertyBuilder.SetGetMethod(methodBuilder); - } - - if (func.HasAttr("fset") && func.GetAttr("fset").IsTrue()) - { - MethodBuilder methodBuilder = typeBuilder.DefineMethod("set_" + propertyName, - methodAttribs, - null, - new Type[]{propertyType}); + PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, + PropertyAttributes.None, + propertyType, + null); - ILGenerator il = methodBuilder.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, propertyName); - il.Emit(OpCodes.Ldarg_1); - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeSetProperty").MakeGenericMethod(propertyType)); - il.Emit(OpCodes.Ret); + if (func.HasAttr("fget") && func.GetAttr("fget").IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("get_" + propertyName, + methodAttribs, + propertyType, + null); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeGetProperty").MakeGenericMethod(propertyType)); + il.Emit(OpCodes.Ret); + + propertyBuilder.SetGetMethod(methodBuilder); + } - propertyBuilder.SetSetMethod(methodBuilder); + if (func.HasAttr("fset") && func.GetAttr("fset").IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("set_" + propertyName, + methodAttribs, + null, + new Type[]{propertyType}); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Ldarg_1); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeSetProperty").MakeGenericMethod(propertyType)); + il.Emit(OpCodes.Ret); + + propertyBuilder.SetSetMethod(methodBuilder); + } } } @@ -681,25 +689,16 @@ public static T InvokeGetProperty(IPythonDerivedType obj, string propertyName if (null == self) throw new NullReferenceException("Instance must be specified when getting a property"); - List disposeList = new List(); IntPtr gs = Runtime.PyGILState_Ensure(); try { Runtime.Incref(self.pyHandle); - PyObject pyself = new PyObject(self.pyHandle); - disposeList.Add(pyself); - - PyObject pyvalue = pyself.GetAttr(propertyName); - disposeList.Add(pyvalue); - return (T)pyvalue.AsManagedObject(typeof(T)); + using (PyObject pyself = new PyObject(self.pyHandle)) + using (PyObject pyvalue = pyself.GetAttr(propertyName)) + return (T)pyvalue.AsManagedObject(typeof(T)); } finally { - foreach (PyObject x in disposeList) - { - if (x != null) - x.Dispose(); - } Runtime.PyGILState_Release(gs); } } @@ -712,26 +711,16 @@ public static void InvokeSetProperty(IPythonDerivedType obj, string propertyN if (null == self) throw new NullReferenceException("Instance must be specified when setting a property"); - List disposeList = new List(); IntPtr gs = Runtime.PyGILState_Ensure(); try { Runtime.Incref(self.pyHandle); - PyObject pyself = new PyObject(self.pyHandle); - disposeList.Add(pyself); - - PyObject pyvalue = new PyObject(Converter.ToPythonImplicit(value)); - disposeList.Add(pyvalue); - - pyself.SetAttr(propertyName, pyvalue); + using (PyObject pyself = new PyObject(self.pyHandle)) + using (PyObject pyvalue = new PyObject(Converter.ToPythonImplicit(value))) + pyself.SetAttr(propertyName, pyvalue); } finally { - foreach (PyObject x in disposeList) - { - if (x != null) - x.Dispose(); - } Runtime.PyGILState_Release(gs); } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index fdfc2a429..5a64c586d 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -175,17 +175,19 @@ internal static IntPtr ToPython(Object value, Type type) { default: if (value is IEnumerable) { - var resultlist = new PyList(); - foreach (object o in (IEnumerable)value) { - resultlist.Append(new PyObject(ToPython(o, o.GetType()))); + using (var resultlist = new PyList()) { + foreach (object o in (IEnumerable)value) { + using (var p = new PyObject(ToPython(o, o.GetType()))) + resultlist.Append(p); + } + Runtime.Incref(resultlist.Handle); + return resultlist.Handle; } - Runtime.Incref(resultlist.Handle); - return resultlist.Handle; } - result = CLRObject.GetInstHandle(value, type); return result; } + } From 3a65dc178b396e55bb0e21279e15066faeb3d569 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 28 Jan 2015 13:28:59 +0000 Subject: [PATCH 56/77] Make sure temporary dictionary is disposed correctly. --- src/runtime/metatype.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 0c4f8c3b5..881723e8f 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -94,9 +94,10 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { // into python. if (IntPtr.Zero != dict) { Runtime.Incref(dict); - PyDict clsDict = new PyDict(dict); - if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) - return TypeManager.CreateSubType(name, base_type, dict); + using (PyDict clsDict = new PyDict(dict)) { + if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) + return TypeManager.CreateSubType(name, base_type, dict); + } } // otherwise just create a basic type without reflecting back into the managed side. From 8f8aff9074bc68b90759987c2514479734b08adc Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 28 Jan 2015 14:05:47 +0000 Subject: [PATCH 57/77] Fix a few more temporary objects to be disposed cleanly. --- src/runtime/importhook.cs | 39 ++++++++++++++++++---------------- src/runtime/pydict.cs | 3 ++- src/runtime/pyfloat.cs | 9 ++++---- src/runtime/pyobject.cs | 9 +++++--- src/runtime/pythonexception.cs | 9 ++++++-- 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 670f24a2d..33f737348 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -108,24 +108,27 @@ public static IntPtr GetCLRModule(IntPtr? fromList=null) { if (Runtime.PyTuple_Check(fromList.GetValueOrDefault())) { Runtime.Incref(py_mod_dict); - PyDict mod_dict = new PyDict(py_mod_dict); - - Runtime.Incref(fromList.GetValueOrDefault()); - PyTuple from = new PyTuple(fromList.GetValueOrDefault()); - foreach (PyObject item in from) { - if (mod_dict.HasKey(item)) - continue; - - string s = item.AsManagedObject(typeof(string)) as string; - if (null == s) - continue; - - ManagedType attr = root.GetAttribute(s, true); - if (null == attr) - continue; - - Runtime.Incref(attr.pyHandle); - mod_dict.SetItem(s, new PyObject(attr.pyHandle)); + using(PyDict mod_dict = new PyDict(py_mod_dict)) { + Runtime.Incref(fromList.GetValueOrDefault()); + using (PyTuple from = new PyTuple(fromList.GetValueOrDefault())) { + foreach (PyObject item in from) { + if (mod_dict.HasKey(item)) + continue; + + string s = item.AsManagedObject(typeof(string)) as string; + if (null == s) + continue; + + ManagedType attr = root.GetAttribute(s, true); + if (null == attr) + continue; + + Runtime.Incref(attr.pyHandle); + using (PyObject obj = new PyObject(attr.pyHandle)) { + mod_dict.SetItem(s, obj); + } + } + } } } } diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index cd85c7126..e2ceaa6d6 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -102,7 +102,8 @@ public bool HasKey(PyObject key) { /// public bool HasKey(string key) { - return HasKey(new PyString(key)); + using (PyString str = new PyString(key)) + return HasKey(str); } diff --git a/src/runtime/pyfloat.cs b/src/runtime/pyfloat.cs index 960892594..c6995887c 100644 --- a/src/runtime/pyfloat.cs +++ b/src/runtime/pyfloat.cs @@ -76,10 +76,11 @@ public PyFloat(double value) : base() { /// public PyFloat(string value) : base() { - PyString s = new PyString(value); - obj = Runtime.PyFloat_FromString(s.obj, IntPtr.Zero); - if (obj == IntPtr.Zero) { - throw new PythonException(); + using (PyString s = new PyString(value)) { + obj = Runtime.PyFloat_FromString(s.obj, IntPtr.Zero); + if (obj == IntPtr.Zero) { + throw new PythonException(); + } } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 37aa65e78..940fe3ea5 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -364,7 +364,8 @@ public virtual PyObject GetItem(PyObject key) { /// public virtual PyObject GetItem(string key) { - return GetItem(new PyString(key)); + using (PyString pyKey = new PyString(key)) + return GetItem(pyKey); } @@ -413,7 +414,8 @@ public virtual void SetItem(PyObject key, PyObject value) { /// public virtual void SetItem(string key, PyObject value) { - SetItem(new PyString(key), value); + using (PyString pyKey = new PyString(key)) + SetItem(pyKey, value); } @@ -461,7 +463,8 @@ public virtual void DelItem(PyObject key) { /// public virtual void DelItem(string key) { - DelItem(new PyString(key)); + using (PyString pyKey = new PyString(key)) + DelItem(pyKey); } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index e85c17651..007b88cf0 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -34,14 +34,19 @@ public PythonException() : base() Runtime.Incref(_pyTB); if ((_pyType != IntPtr.Zero) && (_pyValue != IntPtr.Zero)) { - string type = new PyObject(_pyType).GetAttr("__name__").ToString(); + string type; + using (PyObject pyType = new PyObject(_pyType)) { + type = pyType.GetAttr("__name__").ToString(); + } string message = Runtime.GetManagedString(_pyValue); _message = type + " : " + message; } if (_pyTB != IntPtr.Zero) { PyObject tb_module = PythonEngine.ImportModule("traceback"); - _tb = tb_module.InvokeMethod("format_tb", new PyObject(_pyTB)).ToString(); + using (PyObject pyTB = new PyObject(_pyTB)) { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } PythonEngine.ReleaseLock(gs); } From d8e46d1d1b33877b8edff0d8e483539d1a46739f Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 28 Jan 2015 15:01:21 +0000 Subject: [PATCH 58/77] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b00813b3..3b59c89b9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Python for .NET is a package that gives Python programmers nearly seamless integ [![Build Status](https://travis-ci.org/pythonnet/pythonnet.png?branch=develop)](https://travis-ci.org/pythonnet/pythonnet) -[![Build status](https://ci.appveyor.com/api/projects/status/65riiu1hvgaxsbwb)](https://ci.appveyor.com/project/davidanthoff/pythonnet) +[![Build status](https://ci.appveyor.com/api/projects/status/u3p6pkiqpgu0qoku/branch/python3)](https://ci.appveyor.com/project/TonyRoberts/pythonnet/branch/python3) **Features not yet integrated into the main branch**: From 0852becf0d514ddac57ac0eb1a8fe869697ffb6e Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 28 Jan 2015 15:03:20 +0000 Subject: [PATCH 59/77] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 3b59c89b9..f03190112 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,6 @@ pythonnet Python for .NET is a package that gives Python programmers nearly seamless integration with the .NET Common Language Runtime (CLR) and provides a powerful application scripting tool for .NET developers. -[![Build Status](https://travis-ci.org/pythonnet/pythonnet.png?branch=develop)](https://travis-ci.org/pythonnet/pythonnet) - [![Build status](https://ci.appveyor.com/api/projects/status/u3p6pkiqpgu0qoku/branch/python3)](https://ci.appveyor.com/project/TonyRoberts/pythonnet/branch/python3) From 42a7dcabb939b34cc32f70811bb614fdc3b2097f Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 5 Jun 2015 15:33:46 +0100 Subject: [PATCH 60/77] Dispose the owned 'current' object owned by PyIter instances when no longer needed. --- src/runtime/pyiter.cs | 53 ++++++++++++++++++++++++++--------------- src/runtime/pyobject.cs | 10 +++++--- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index 8d8ad44d7..c8599c2a3 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -21,42 +21,57 @@ public class PyIter : PyObject, IEnumerator private PyObject _current = null; /// - /// PyIter Constructor - /// - /// - /// - /// Creates a new PyIter from an existing iterator reference. Note - /// that the instance assumes ownership of the object reference. - /// The object reference is not checked for type-correctness. - /// + /// PyIter Constructor + /// + /// + /// + /// Creates a new PyIter from an existing iterator reference. Note + /// that the instance assumes ownership of the object reference. + /// The object reference is not checked for type-correctness. + /// - public PyIter(IntPtr ptr) : base(ptr) {} + public PyIter(IntPtr ptr) : base(ptr) {} /// - /// PyIter Constructor - /// - /// - /// - /// Creates a Python iterator from an iterable. Like doing "iter(iterable)" in python. - /// + /// PyIter Constructor + /// + /// + /// + /// Creates a Python iterator from an iterable. Like doing "iter(iterable)" in python. + /// - public PyIter(PyObject iterable) : base() + public PyIter(PyObject iterable) : base() { obj = Runtime.PyObject_GetIter(iterable.obj); if (obj == IntPtr.Zero) throw new PythonException(); } + protected override void Dispose(bool disposing) + { + if (null != _current) + { + _current.Dispose(); + _current = null; + } + base.Dispose(disposing); + } + #region IEnumerator Members public bool MoveNext() { + // dispose of the previous object, if there was one + if (null != _current) + { + _current.Dispose(); + _current = null; + } + IntPtr next = Runtime.PyIter_Next(obj); if (next == IntPtr.Zero) - { - _current = null; //release reference return false; - } + _current = new PyObject(next); return true; } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 940fe3ea5..76d30d211 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -118,19 +118,23 @@ public object AsManagedObject(Type t) { /// collection occurs. /// - public void Dispose() { + protected virtual void Dispose(bool disposing) { if (!disposed) { if (Runtime.Py_IsInitialized() > 0) { IntPtr gs = PythonEngine.AcquireLock(); Runtime.Decref(obj); - obj = IntPtr.Zero; + obj = IntPtr.Zero; PythonEngine.ReleaseLock(gs); } - GC.SuppressFinalize(this); disposed = true; } } + public void Dispose() { + Dispose(true); + GC.SuppressFinalize(this); + } + /// /// GetPythonType Method From 796611a1124625e02734980894d3a20febf38116 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 5 Jun 2015 15:36:08 +0100 Subject: [PATCH 61/77] Dispose PyObject instances after use. --- src/runtime/classderived.cs | 67 ++++++++++++++++++++-------------- src/runtime/pyobject.cs | 43 +++++++++++++++++----- src/runtime/pythonexception.cs | 6 ++- 3 files changed, 76 insertions(+), 40 deletions(-) diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 2d44467a7..685becef9 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -388,7 +388,10 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild private static void AddPythonMethod(string methodName, PyObject func, TypeBuilder typeBuilder) { if (func.HasAttr("_clr_method_name_")) - methodName = func.GetAttr("_clr_method_name_").ToString(); + { + using (PyObject pyMethodName = func.GetAttr("_clr_method_name_")) + methodName = pyMethodName.ToString(); + } using (PyObject pyReturnType = func.GetAttr("_clr_return_type_")) using (PyObject pyArgTypes = func.GetAttr("_clr_arg_types_")) @@ -477,37 +480,45 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu propertyType, null); - if (func.HasAttr("fget") && func.GetAttr("fget").IsTrue()) + if (func.HasAttr("fget")) { - MethodBuilder methodBuilder = typeBuilder.DefineMethod("get_" + propertyName, - methodAttribs, - propertyType, - null); - - ILGenerator il = methodBuilder.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, propertyName); - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeGetProperty").MakeGenericMethod(propertyType)); - il.Emit(OpCodes.Ret); - - propertyBuilder.SetGetMethod(methodBuilder); + using (PyObject pyfget = func.GetAttr("fget")) + if (pyfget.IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("get_" + propertyName, + methodAttribs, + propertyType, + null); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeGetProperty").MakeGenericMethod(propertyType)); + il.Emit(OpCodes.Ret); + + propertyBuilder.SetGetMethod(methodBuilder); + } } - if (func.HasAttr("fset") && func.GetAttr("fset").IsTrue()) + if (func.HasAttr("fset")) { - MethodBuilder methodBuilder = typeBuilder.DefineMethod("set_" + propertyName, - methodAttribs, - null, - new Type[]{propertyType}); - - ILGenerator il = methodBuilder.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldstr, propertyName); - il.Emit(OpCodes.Ldarg_1); - il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeSetProperty").MakeGenericMethod(propertyType)); - il.Emit(OpCodes.Ret); - - propertyBuilder.SetSetMethod(methodBuilder); + using (PyObject pyset = func.GetAttr("fset")) + if (pyset.IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("set_" + propertyName, + methodAttribs, + null, + new Type[]{propertyType}); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Ldarg_1); + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeSetProperty").MakeGenericMethod(propertyType)); + il.Emit(OpCodes.Ret); + + propertyBuilder.SetSetMethod(methodBuilder); + } } } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 76d30d211..33c716599 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -434,7 +434,9 @@ public virtual void SetItem(string key, PyObject value) { /// public virtual void SetItem(int index, PyObject value) { - SetItem(new PyInt(index), value); + using (PyInt pyindex = new PyInt(index)) { + SetItem(pyindex, value); + } } @@ -483,7 +485,8 @@ public virtual void DelItem(string key) { /// public virtual void DelItem(int index) { - DelItem(new PyInt(index)); + using (PyInt pyindex = new PyInt(index)) + DelItem(pyindex); } @@ -962,10 +965,20 @@ public override bool TryInvokeMember(InvokeMemberBinder binder, object[] args, o { if (this.HasAttr(binder.Name) && this.GetAttr(binder.Name).IsCallable()) { - PyTuple pyargs; - PyDict kwargs; - GetArgs(args, out pyargs, out kwargs); - result = InvokeMethod(binder.Name, pyargs, kwargs); + PyTuple pyargs = null; + PyDict kwargs = null; + try + { + GetArgs(args, out pyargs, out kwargs); + result = InvokeMethod(binder.Name, pyargs, kwargs); + } + finally + { + if (null != pyargs) + pyargs.Dispose(); + if (null != kwargs) + kwargs.Dispose(); + } return true; } else @@ -976,10 +989,20 @@ public override bool TryInvoke(InvokeBinder binder, object[] args, out object re { if (this.IsCallable()) { - PyTuple pyargs; - PyDict kwargs; - GetArgs(args, out pyargs, out kwargs); - result = Invoke(pyargs, kwargs); + PyTuple pyargs = null; + PyDict kwargs = null; + try + { + GetArgs(args, out pyargs, out kwargs); + result = Invoke(pyargs, kwargs); + } + finally + { + if (null != pyargs) + pyargs.Dispose(); + if (null != kwargs) + kwargs.Dispose(); + } return true; } else diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 007b88cf0..0d0b2e3e6 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -35,8 +35,10 @@ public PythonException() : base() if ((_pyType != IntPtr.Zero) && (_pyValue != IntPtr.Zero)) { string type; - using (PyObject pyType = new PyObject(_pyType)) { - type = pyType.GetAttr("__name__").ToString(); + using (PyObject pyType = new PyObject(_pyType)) + using (PyObject pyTypeName = pyType.GetAttr("__name__")) + { + type = pyTypeName.ToString(); } string message = Runtime.GetManagedString(_pyValue); _message = type + " : " + message; From 535babc31cfd2049141bad2d0f3812bfed440612 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 10 Jun 2015 14:59:45 +0100 Subject: [PATCH 62/77] Fix build issue with Python 3.3/3.4. Fixes #4 --- setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c40471ecc..f979a7d29 100644 --- a/setup.py +++ b/setup.py @@ -109,9 +109,17 @@ def build_extension(self, ext): if not os.path.exists(dest_dir): os.makedirs(dest_dir) + # Up to Python 3.2 sys.maxunicode is used to determine the size of Py_UNICODE + # but from 3.3 onwards Py_UNICODE is a typedef of wchar_t. + if sys.version_info[:2] <= (3, 2): + unicode_width = 2 if sys.maxunicode < 0x10FFFF else 4 + else: + import ctypes + unicode_width = ctypes.sizeof(ctypes.c_wchar) + defines = [ "PYTHON%d%s" % (sys.version_info[:2]), - "UCS2" if sys.maxunicode < 0x10FFFF else "UCS4", + "UCS%d" % unicode_width, ] if CONFIG == "Debug": From 824b5c772ea11930c2940678990b1e344598acea Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 15 Jun 2015 10:12:16 +0100 Subject: [PATCH 63/77] Fix issue copying npython to scripts. --- setup.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f979a7d29..a4ffce59a 100644 --- a/setup.py +++ b/setup.py @@ -7,10 +7,13 @@ from distutils.command.build_scripts import build_scripts from distutils.command.install_lib import install_lib from distutils.sysconfig import get_config_var +from distutils.util import convert_path +from distutils.dep_util import newer +from distutils import log from platform import architecture from subprocess import Popen, CalledProcessError, PIPE, check_call from glob import glob -import shutil +import stat import sys import os @@ -274,6 +277,60 @@ def finalize_options(self): scripts.append(script) self.scripts = scripts + def copy_scripts(self): + # Look for the npython script as it can't be copied by the base class' + # copy_scripts as it attempts to determine the file encoding as if it + # were a text file. + npython = None + for script in self.scripts: + if os.path.basename(script) == _npython_exe: + npython = script + + if npython is None: + return build_scripts.copy_scripts(self) + + # Call the base class copy_scripts with anything other than npython + scripts = self.scripts + self.scripts = [x for x in scripts if x != npython] + try: + base_result = build_scripts.copy_scripts(self) + finally: + self.scripts = scripts + + # Copy npython + outfiles = [] + updated_files = [] + + script = convert_path(npython) + outfile = os.path.join(self.build_dir, os.path.basename(script)) + outfiles.append(outfile) + + if not self.force and not newer(script, outfile): + log.debug("not copying %s (up-to-date)", script) + else: + updated_files.append(outfile) + self.copy_file(script, outfile) + + if os.name == 'posix': + for file in outfiles: + if self.dry_run: + log.info("changing mode of %s", file) + else: + oldmode = os.stat(file)[stat.ST_MODE] & 0o7777 + newmode = (oldmode | 0o555) & 0o7777 + if newmode != oldmode: + log.info("changing mode of %s from %o to %o", + file, oldmode, newmode) + os.chmod(file, newmode) + + # Some versions of build_command.copy_scripts return (outfiles, updated_files), + # older versions return None. + if base_result is not None: + base_outfiles, base_updated_files = base_result + outfiles.extend(base_outfiles) + updated_files.extend(base_updated_files) + return outfiles, updated_files + def _check_output(*popenargs, **kwargs): """subprocess.check_output from python 2.7. @@ -309,9 +366,9 @@ def _check_output(*popenargs, **kwargs): scripts=[_npython_exe], zip_safe=False, cmdclass={ - "build_ext" : PythonNET_BuildExt, - "build_scripts" : PythonNET_BuildScripts, - "install_lib" : PythonNET_InstallLib + "build_ext": PythonNET_BuildExt, + "build_scripts": PythonNET_BuildScripts, + "install_lib": PythonNET_InstallLib, } ) From 591b374217d5a7e9b3125f8422ba5b0ce0a7fdf8 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 15 Jun 2015 12:06:01 +0100 Subject: [PATCH 64/77] Copy pdb files to build folder if they exist. --- src/clrmodule/clrmodule.csproj | 7 ++++++- src/embed_tests/Python.EmbeddingTest.csproj | 7 ++++++- src/runtime/Python.Runtime.csproj | 7 ++++++- src/testing/Python.Test.csproj | 5 ++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj index 728683310..f6d1a41b5 100644 --- a/src/clrmodule/clrmodule.csproj +++ b/src/clrmodule/clrmodule.csproj @@ -119,8 +119,13 @@ + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + - + + diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 14f97f5fb..e4a9750b8 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -172,7 +172,12 @@ + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + - + + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 4179e707e..0b3f3a619 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -189,7 +189,12 @@ + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + - + + diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 672a86300..46ca484bc 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -141,8 +141,11 @@ $(SolutionDir) + $(TargetPath) + $(TargetDir)$(TargetName).pdb - + + From 95638e5f4a3dc928f6a20e269f5e6863e7226816 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 15 Jun 2015 17:58:48 +0100 Subject: [PATCH 65/77] Fix some issues in the import hook. Imported modules should be added to sys.modules. When importing something like 'CLR.X' the clr module should be returned, not X. --- src/runtime/importhook.cs | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 33f737348..208050b25 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -183,15 +183,31 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { // do the Incref()ed return here, since we've already found // the module. if (mod_name == "clr") { - return GetCLRModule(fromList); + IntPtr clr_module = GetCLRModule(fromList); + if (clr_module != IntPtr.Zero) { + IntPtr sys_modules = Runtime.PyImport_GetModuleDict(); + if (sys_modules != IntPtr.Zero) { + Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); + } + } + return clr_module; } if (mod_name == "CLR") { Exceptions.deprecation("The CLR module is deprecated. " + "Please use 'clr'."); - return GetCLRModule(fromList); + IntPtr clr_module = GetCLRModule(fromList); + if (clr_module != IntPtr.Zero) { + IntPtr sys_modules = Runtime.PyImport_GetModuleDict(); + if (sys_modules != IntPtr.Zero) { + Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); + } + } + return clr_module; } string realname = mod_name; + string clr_prefix = null; if (mod_name.StartsWith("CLR.")) { + clr_prefix = "CLR."; // prepend when adding the module to sys.modules realname = mod_name.Substring(4); string msg = String.Format("Importing from the CLR.* namespace "+ "is deprecated. Please import '{0}' directly.", realname); @@ -251,6 +267,9 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { Runtime.Incref(module); return module; } + if (clr_prefix != null) { + return GetCLRModule(fromList); + } module = Runtime.PyDict_GetItemString(modules, names[0]); Runtime.Incref(module); return module; @@ -286,9 +305,18 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { if (CLRModule.preload) { tail.LoadNames(); } - Runtime.PyDict_SetItemString(modules, tail.moduleName, - tail.pyHandle - ); + + // Add the module to sys.modules + Runtime.PyDict_SetItemString(modules, + tail.moduleName, + tail.pyHandle); + + // If imported from CLR add CLR. to sys.modules as well + if (clr_prefix != null) { + Runtime.PyDict_SetItemString(modules, + clr_prefix + tail.moduleName, + tail.pyHandle); + } } ModuleObject mod = fromlist ? tail : head; From d64273ec054b49d29cb75ba8f14697e02acea553 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Mon, 15 Jun 2015 18:00:45 +0100 Subject: [PATCH 66/77] Fix Int32 conversion for Python 3. --- src/runtime/converter.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 5a64c586d..4d0c06c11 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -405,6 +405,7 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, return true; case TypeCode.Int32: +#if !(PYTHON32 || PYTHON33 || PYTHON34) // Trickery to support 64-bit platforms. if (IntPtr.Size == 4) { op = Runtime.PyNumber_Int(value); @@ -427,6 +428,10 @@ static bool ToPrimitive(IntPtr value, Type obType, out Object result, return true; } else { +#else + // When using Python3 always use the PyLong API + { +#endif op = Runtime.PyNumber_Long(value); if (op == IntPtr.Zero) { if (Exceptions.ExceptionMatches(overflow)) { From cc7fe9a87306d45a7d3735a7d24a9a297291bd93 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 16 Jun 2015 11:40:29 +0100 Subject: [PATCH 67/77] When shutting down don't delete any Python objects after Py_Finalize has been called. --- src/runtime/exceptions.cs | 19 +++++++++++-------- src/runtime/importhook.cs | 16 +++++++++++----- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 5857c1e35..2abf1f29d 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -168,16 +168,19 @@ internal static void Initialize() { //=================================================================== internal static void Shutdown() { - Type type = typeof(Exceptions); - foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | - BindingFlags.Static)) { - IntPtr op = (IntPtr)fi.GetValue(type); - if (op != IntPtr.Zero) { - Runtime.Decref(op); + if (0 != Runtime.Py_IsInitialized()) { + Type type = typeof(Exceptions); + foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | + BindingFlags.Static)) { + IntPtr op = (IntPtr)fi.GetValue(type); + if (op != IntPtr.Zero) { + Runtime.Decref(op); + } } + Runtime.Decref(exceptions_module); + Runtime.PyObject_HasAttrString(warnings_module, "xx"); + Runtime.Decref(warnings_module); } - Runtime.Decref(exceptions_module); - Runtime.Decref(warnings_module); } /// diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 208050b25..9b44b240c 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -79,14 +79,20 @@ internal static void Initialize() { internal static void Shutdown() { #if (PYTHON32 || PYTHON33 || PYTHON34) - Runtime.Decref(py_clr_module); - Runtime.Decref(root.pyHandle); + if (0 != Runtime.Py_IsInitialized()) { + Runtime.Decref(py_clr_module); + Runtime.Decref(root.pyHandle); + } ModuleDefOffset.FreeModuleDef(module_def); #else - Runtime.Decref(root.pyHandle); - Runtime.Decref(root.pyHandle); + if (0 != Runtime.Py_IsInitialized()) { + Runtime.Decref(root.pyHandle); + Runtime.Decref(root.pyHandle); + } #endif - Runtime.Decref(py_import); + if (0 != Runtime.Py_IsInitialized()) { + Runtime.Decref(py_import); + } } //=================================================================== From 5e2e7080728652281b9b9eb160d61056bc49c036 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 17 Jun 2015 17:19:07 +0100 Subject: [PATCH 68/77] Fix decref of locals in RunString. Copy and paste bug. --- src/runtime/pythonengine.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index d7fb10774..dc36bd181 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -// ========================================================================== +work// ========================================================================== // This software is subject to the provisions of the Zope Public License, // Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. // THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED @@ -436,9 +436,7 @@ public static PyObject RunString(string code, IntPtr globals, IntPtr locals) { IntPtr flag = (IntPtr)257; /* Py_file_input */ IntPtr result = Runtime.PyRun_String(code, flag, globals, locals); - Runtime.Decref(locals); - if (result == IntPtr.Zero) - { + if (result == IntPtr.Zero) { return null; } return new PyObject(result); From 5b9e0dd765d7d58f6510db77fd6649af50c9736b Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 17 Jun 2015 17:21:12 +0100 Subject: [PATCH 69/77] Revert accidental change in last commit. --- src/runtime/pythonengine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index dc36bd181..50976f632 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -work// ========================================================================== +// ========================================================================== // This software is subject to the provisions of the Zope Public License, // Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. // THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED From c7db27340370458541f95725f529b87cabf74445 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 23 Jun 2015 17:09:29 +0100 Subject: [PATCH 70/77] Set LD_LIBRARY_PATH before starting Mono. Mono doesn't observe the rpath set when the lib is built, so get the folder the Python shared object has been loaded from and add that to LD_LIBRARY_PATH. Without this, if there are multiple Pythons installed the wrong object can be loaded. --- src/monoclr/pynetinit.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/monoclr/pynetinit.c b/src/monoclr/pynetinit.c index 2f8b75848..abc1de1d2 100644 --- a/src/monoclr/pynetinit.c +++ b/src/monoclr/pynetinit.c @@ -10,14 +10,15 @@ // Author: Christian Heimes #include "pynetclr.h" +#include "stdlib.h" #ifndef _WIN32 #include "dirent.h" +#include "dlfcn.h" +#include "libgen.h" +#include "alloca.h" #endif -#if PY_MAJOR_VERSION > 2 -#include -#endif // initialize Mono and PythonNet PyNet_Args* PyNet_Init(int ext) { @@ -97,6 +98,26 @@ void main_thread_handler (gpointer user_data) { MonoObject *exception = NULL; #ifndef _WIN32 + // Get the filename of the python shared object and set + // LD_LIBRARY_PATH so Mono can find it. + Dl_info dlinfo = {0}; + if (0 != dladdr(&Py_Initialize, &dlinfo)) { + char* fname = alloca(strlen(dlinfo.dli_fname) + 1); + strcpy(fname, dlinfo.dli_fname); + char* py_libdir = dirname(fname); + char* ld_library_path = getenv("LD_LIBRARY_PATH"); + if (NULL == ld_library_path) { + setenv("LD_LIBRARY_PATH", py_libdir, 1); + } else { + char* new_ld_library_path = alloca(strlen(py_libdir) + + strlen(ld_library_path) + + 2); + strcpy(new_ld_library_path, py_libdir); + strcat(new_ld_library_path, ":"); + strcat(new_ld_library_path, ld_library_path); + setenv("LD_LIBRARY_PATH", py_libdir, 1); + } + } //get python path system variable PyObject* syspath = PySys_GetObject("path"); From 9b2d6f266c46e59600016509187018218f6a0aa1 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Tue, 23 Jun 2015 18:14:39 +0100 Subject: [PATCH 71/77] Fix clr module to work without npython. The npython binary was a workaround to be able to use pythonnet on Linux. This is no longer necessary as the clr module can be imported directly from the standard Python interpreter. --- .travis.yml | 4 +- setup.py | 118 ++-------------------------------------- src/monoclr/clrpython.c | 29 ---------- src/monoclr/python.c | 44 --------------- src/runtime/runtime.cs | 4 ++ 5 files changed, 11 insertions(+), 188 deletions(-) delete mode 100644 src/monoclr/clrpython.c delete mode 100644 src/monoclr/python.c diff --git a/.travis.yml b/.travis.yml index dbf7d1d31..9a7471baf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,6 @@ install: - pip install six - python setup.py build_ext --inplace script: - - export PYTHONPATH=`pwd` + - export PYTHONPATH=`pwd`:$PYTHONPATH - export PYTHONHOME=`python -c "import sys; print(sys.exec_prefix)"` - - ./npython src/tests/runtests.py + - python src/tests/runtests.py diff --git a/setup.py b/setup.py index a4ffce59a..fab21726c 100644 --- a/setup.py +++ b/setup.py @@ -83,13 +83,11 @@ def _find_msbuild_tool(tool="msbuild.exe", use_windows_sdk=False): _xbuild = "\"%s\"" % _find_msbuild_tool("msbuild.exe") _defines_sep = ";" _config = "%sWin" % CONFIG - _npython_exe = "nPython.exe" elif DEVTOOLS == "Mono": _xbuild = "xbuild" _defines_sep = "," _config = "%sMono" % CONFIG - _npython_exe = "npython" else: raise NotImplementedError("DevTools %s not supported (use MsDev or Mono)" % DEVTOOLS) @@ -131,6 +129,11 @@ def build_extension(self, ext): if sys.platform != "win32" and DEVTOOLS == "Mono": defines.append("MONO_LINUX") + # Check if --enable-shared was set when Python was built + enable_shared = get_config_var("Py_ENABLE_SHARED") + if enable_shared == 0: + defines.append("PYTHON_WITHOUT_ENABLE_SHARED") + if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.append("PYTHON_WITH_PYDEBUG") @@ -191,44 +194,6 @@ def _build_monoclr(self, ext): build_ext.build_extension(self, clr_ext) - # build the clr python executable - sources = [ - "src/monoclr/pynetinit.c", - "src/monoclr/python.c", - ] - - macros = ext.define_macros[:] - for undef in ext.undef_macros: - macros.append((undef,)) - - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=ext.include_dirs, - debug=self.debug, - extra_postargs=cflags.split(" "), - depends=ext.depends) - - output_dir = os.path.dirname(self.get_ext_fullpath(ext.name)) - py_libs = get_config_var("BLDLIBRARY") - libs += " " + py_libs - - # Include the directories python's shared libs were installed to. This - # is case python was built with --enable-shared as then npython will need - # to be able to find libpythonX.X.so. - runtime_library_dirs = (get_config_var("DESTDIRS") or "").split(" ") - if ext.runtime_library_dirs: - runtime_library_dirs.extend(ext.runtime_library_dirs) - - self.compiler.link_executable(objects, - _npython_exe, - output_dir=output_dir, - libraries=self.get_libraries(ext), - library_dirs=ext.library_dirs, - runtime_library_dirs=runtime_library_dirs, - extra_postargs=libs.split(" "), - debug=self.debug) - def _install_packages(self): """install packages using nuget""" @@ -261,77 +226,6 @@ def install(self): self.copy_file(srcfile, destfile) -class PythonNET_BuildScripts(build_scripts): - - def finalize_options(self): - build_scripts.finalize_options(self) - - # fixup scripts to look in the build_ext output folder - if self.scripts: - build_ext = self.get_finalized_command("build_ext") - output_dir = os.path.dirname(build_ext.get_ext_fullpath("clr")) - scripts = [] - for script in self.scripts: - if os.path.exists(os.path.join(output_dir, script)): - script = os.path.join(output_dir, script) - scripts.append(script) - self.scripts = scripts - - def copy_scripts(self): - # Look for the npython script as it can't be copied by the base class' - # copy_scripts as it attempts to determine the file encoding as if it - # were a text file. - npython = None - for script in self.scripts: - if os.path.basename(script) == _npython_exe: - npython = script - - if npython is None: - return build_scripts.copy_scripts(self) - - # Call the base class copy_scripts with anything other than npython - scripts = self.scripts - self.scripts = [x for x in scripts if x != npython] - try: - base_result = build_scripts.copy_scripts(self) - finally: - self.scripts = scripts - - # Copy npython - outfiles = [] - updated_files = [] - - script = convert_path(npython) - outfile = os.path.join(self.build_dir, os.path.basename(script)) - outfiles.append(outfile) - - if not self.force and not newer(script, outfile): - log.debug("not copying %s (up-to-date)", script) - else: - updated_files.append(outfile) - self.copy_file(script, outfile) - - if os.name == 'posix': - for file in outfiles: - if self.dry_run: - log.info("changing mode of %s", file) - else: - oldmode = os.stat(file)[stat.ST_MODE] & 0o7777 - newmode = (oldmode | 0o555) & 0o7777 - if newmode != oldmode: - log.info("changing mode of %s from %o to %o", - file, oldmode, newmode) - os.chmod(file, newmode) - - # Some versions of build_command.copy_scripts return (outfiles, updated_files), - # older versions return None. - if base_result is not None: - base_outfiles, base_updated_files = base_result - outfiles.extend(base_outfiles) - updated_files.extend(base_updated_files) - return outfiles, updated_files - - def _check_output(*popenargs, **kwargs): """subprocess.check_output from python 2.7. Added here to support building for earlier versions @@ -363,11 +257,9 @@ def _check_output(*popenargs, **kwargs): ext_modules=[ Extension("clr", sources=[]) ], - scripts=[_npython_exe], zip_safe=False, cmdclass={ "build_ext": PythonNET_BuildExt, - "build_scripts": PythonNET_BuildScripts, "install_lib": PythonNET_InstallLib, } ) diff --git a/src/monoclr/clrpython.c b/src/monoclr/clrpython.c deleted file mode 100644 index 5fddabf22..000000000 --- a/src/monoclr/clrpython.c +++ /dev/null @@ -1,29 +0,0 @@ -// ========================================================================== -// This software is subject to the provisions of the Zope Public License, -// Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -// FOR A PARTICULAR PURPOSE. -// ========================================================================== -// -// Example how to integrate Python, PythonNet and Mono into a C application -// It provides a command prompt equal to PythonNet's console but using a -// different path. -// -// Author: Christian Heimes -// - -#include "pynetclr.h" - -int main(int argc, char **argv) { - PyNet_Args *pn_args; - pn_args = PyNet_Init(0); - if (pn_args->error) { - exit(1); - } - int rc = Py_Main(argc, argv); - PyNet_Finalize(pn_args); - exit(rc); -} - diff --git a/src/monoclr/python.c b/src/monoclr/python.c deleted file mode 100644 index f90b9b540..000000000 --- a/src/monoclr/python.c +++ /dev/null @@ -1,44 +0,0 @@ -// ========================================================================== -// This software is subject to the provisions of the Zope Public License, -// Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -// FOR A PARTICULAR PURPOSE. -// ========================================================================== -// -// python.c provides a python executable with is dynamically linked agaist -// libpython2.x.so. For example Ubuntu's python executables aren't linked -// against libpython :( -// -// Author: Christian Heimes -// - -#include - -#if (PY_MAJOR_VERSION > 2) -#include -#endif - -int main(int argc, char **argv) { -#if (PY_MAJOR_VERSION > 2) - int i, result; - size_t len; - wchar_t **wargv = (wchar_t**)malloc(sizeof(wchar_t*)*argc); - for (i=0; i Date: Tue, 23 Jun 2015 18:30:30 +0100 Subject: [PATCH 72/77] Update README.md Add travis-ci build status. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f03190112..316fe0e55 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Python for .NET is a package that gives Python programmers nearly seamless integ [![Build status](https://ci.appveyor.com/api/projects/status/u3p6pkiqpgu0qoku/branch/python3)](https://ci.appveyor.com/project/TonyRoberts/pythonnet/branch/python3) +[![Build Status](https://travis-ci.org/renshawbay/pythonnet.png?branch=python3)](https://travis-ci.org/renshawbay/pythonnet) **Features not yet integrated into the main branch**: - Python 3 support From 0ec51a3f614532cacd69f87491bd3a32980d05ef Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 24 Jun 2015 09:33:45 +0100 Subject: [PATCH 73/77] Update CI scripts now npython is no longer used. --- .travis.yml | 1 - appveyor.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a7471baf..5581f1329 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,5 +18,4 @@ install: - python setup.py build_ext --inplace script: - export PYTHONPATH=`pwd`:$PYTHONPATH - - export PYTHONHOME=`python -c "import sys; print(sys.exec_prefix)"` - python src/tests/runtests.py diff --git a/appveyor.yml b/appveyor.yml index ebdfb5b83..d7d46a7c0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -33,4 +33,3 @@ test_script: - mkdir c:\testdir - ps: copy-item (gci -path build -re -include Python.Test.dll)[0].FullName c:\testdir - c:\python\python.exe src\tests\runtests.py - - c:\python\scripts\npython.exe src\tests\runtests.py From 88aa3c4fd2d8ab8a1f981729f429c8948bc117e5 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 24 Jun 2015 17:03:02 +0100 Subject: [PATCH 74/77] Fixes for OSX. --- setup.py | 5 ++++- src/monoclr/clrmod.c | 4 ++++ src/monoclr/pynetclr.h | 1 + src/monoclr/pynetinit.c | 10 ++++++++- src/runtime/runtime.cs | 49 ++++++++++++++++++++++++++++++++++------- 5 files changed, 59 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index fab21726c..1f5cfcf43 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,10 @@ def build_extension(self, ext): defines.extend(["DEBUG", "TRACE"]) if sys.platform != "win32" and DEVTOOLS == "Mono": - defines.append("MONO_LINUX") + if sys.platform == "darwin": + defines.append("MONO_OSX") + else: + defines.append("MONO_LINUX") # Check if --enable-shared was set when Python was built enable_shared = get_config_var("Py_ENABLE_SHARED") diff --git a/src/monoclr/clrmod.c b/src/monoclr/clrmod.c index dd68696bc..c6de71eeb 100644 --- a/src/monoclr/clrmod.c +++ b/src/monoclr/clrmod.c @@ -57,6 +57,10 @@ static PyObject *_initclr() { if (pn_args->error) { return NULL; } + + if (NULL != pn_args->module) + return pn_args->module; + return m; } diff --git a/src/monoclr/pynetclr.h b/src/monoclr/pynetclr.h index c97db10cb..3a6a60c9c 100644 --- a/src/monoclr/pynetclr.h +++ b/src/monoclr/pynetclr.h @@ -32,6 +32,7 @@ typedef struct { char *error; char *init_name; char *shutdown_name; + PyObject* module; } PyNet_Args; PyNet_Args* PyNet_Init(int); diff --git a/src/monoclr/pynetinit.c b/src/monoclr/pynetinit.c index abc1de1d2..f6487802c 100644 --- a/src/monoclr/pynetinit.c +++ b/src/monoclr/pynetinit.c @@ -27,6 +27,7 @@ PyNet_Args* PyNet_Init(int ext) { pn_args->pr_file = PR_ASSEMBLY; pn_args->error = NULL; pn_args->shutdown = NULL; + pn_args->module = NULL; if (ext == 0) { pn_args->init_name = "Python.Runtime:Initialize()"; @@ -96,6 +97,7 @@ void main_thread_handler (gpointer user_data) { MonoImage *pr_image; MonoClass *pythonengine; MonoObject *exception = NULL; + MonoObject *init_result; #ifndef _WIN32 // Get the filename of the python shared object and set @@ -209,11 +211,17 @@ void main_thread_handler (gpointer user_data) { return; } - mono_runtime_invoke(init, NULL, NULL, &exception); + init_result = mono_runtime_invoke(init, NULL, NULL, &exception); if (exception) { pn_args->error = PyNet_ExceptionToString(exception); return; } + +#if PY_MAJOR_VERSION >= 3 + if (NULL != init_result) + pn_args->module = *(PyObject**)mono_object_unbox(init_result); +#endif + } // Get string from a Mono exception diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index e50bb7dd4..cce085223 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -25,7 +25,7 @@ namespace Python.Runtime { static class NativeMethods { -#if MONO_LINUX +#if (MONO_LINUX || MONO_OSX) static public IntPtr LoadLibrary(string fileName) { return dlopen(fileName, RTLD_NOW | RTLD_SHARED); } @@ -35,6 +35,10 @@ static public void FreeLibrary(IntPtr handle) { } static public IntPtr GetProcAddress(IntPtr dllHandle, string name) { + // look in the exe if dllHandle is NULL + if (IntPtr.Zero == dllHandle) + dllHandle = RTLD_DEFAULT; + // clear previous errors if any dlerror(); var res = dlsym(dllHandle, name); @@ -45,8 +49,26 @@ static public IntPtr GetProcAddress(IntPtr dllHandle, string name) { return res; } - const int RTLD_NOW = 2; - const int RTLD_SHARED = 20; +#if (MONO_OSX) + static int RTLD_NOW = 0x2; + static int RTLD_SHARED = 0x20; + static IntPtr RTLD_DEFAULT = new IntPtr(-2); + + [DllImport("__Internal")] + private static extern IntPtr dlopen(String fileName, int flags); + + [DllImport("__Internal")] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport("__Internal")] + private static extern int dlclose(IntPtr handle); + + [DllImport("__Internal")] + private static extern IntPtr dlerror(); +#else + static int RTLD_NOW = 0x2; + static int RTLD_SHARED = 0x20; + static IntPtr RTLD_DEFAULT = IntPtr.Zero; [DllImport("libdl.so")] private static extern IntPtr dlopen(String fileName, int flags); @@ -59,6 +81,8 @@ static public IntPtr GetProcAddress(IntPtr dllHandle, string name) { [DllImport("libdl.so")] private static extern IntPtr dlerror(); +#endif + #else [DllImport("kernel32.dll")] public static extern IntPtr LoadLibrary(string dllToLoad); @@ -139,7 +163,7 @@ public class Runtime { #if (PYTHON27) internal const string dllBase = "python27"; #endif -#if (MONO_LINUX) +#if (MONO_LINUX || MONO_OSX) #if (PYTHON32) internal const string dllBase = "python3.2"; #endif @@ -295,10 +319,15 @@ internal static void Initialize() { Error = new IntPtr(-1); #if (PYTHON32 || PYTHON33 || PYTHON34) - IntPtr dll = NativeMethods.LoadLibrary(Runtime.dll); + IntPtr dll = IntPtr.Zero; + if ("__Internal" != Runtime.dll) { + NativeMethods.LoadLibrary(Runtime.dll); + } _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dll, "_PyObject_NextNotImplemented"); -#if !MONO_LINUX - NativeMethods.FreeLibrary(dll); +#if !(MONO_LINUX || MONO_OSX) + if (IntPtr.Zero != dll) { + NativeMethods.FreeLibrary(dll); + } #endif #endif @@ -1751,7 +1780,11 @@ internal unsafe static string GetManagedString(IntPtr op) if (type == Runtime.PyUnicodeType) { IntPtr p = Runtime.PyUnicode_AsUnicode(op); - return UnixMarshal.PtrToString(p, Encoding.UTF32); + int length = Runtime.PyUnicode_GetSize(op); + int size = length * 4; + byte[] buffer = new byte[size]; + Marshal.Copy(p, buffer, 0, size); + return Encoding.UTF32.GetString(buffer, 0, size); } return null; From 3c3919de42325dfca8b74db247afee8e6ccad27b Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Wed, 24 Jun 2015 18:04:02 +0100 Subject: [PATCH 75/77] Set a logical name for the clr.py resource. --- src/runtime/Python.Runtime.csproj | 4 +++- src/runtime/pythonengine.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 0b3f3a619..80f911734 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -183,7 +183,9 @@ - + + clr.py + diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 50976f632..1a84eb198 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -159,7 +159,7 @@ public static void Initialize() { Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); var assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("Python.Runtime.resources.clr.py")) + using (Stream stream = assembly.GetManifestResourceStream("clr.py")) using (StreamReader reader = new StreamReader(stream)) { // add the contents of clr.py to the module From 1ee70319e7d09ca266af6735a92105a42ee21369 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Thu, 25 Jun 2015 14:54:53 +0100 Subject: [PATCH 76/77] Add Python 3 dlls to dll map. --- Python.Runtime.dll.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python.Runtime.dll.config b/Python.Runtime.dll.config index 11b4fb0fe..e9821a8a9 100644 --- a/Python.Runtime.dll.config +++ b/Python.Runtime.dll.config @@ -14,10 +14,16 @@ For more information read: + + + + + + From dd2dcd078af8ec7b4d5f024c6bf33e031b2bc505 Mon Sep 17 00:00:00 2001 From: Tony Roberts Date: Fri, 26 Jun 2015 11:18:23 +0100 Subject: [PATCH 77/77] Update README.md Update prior to merging back into the main repo. --- README.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 316fe0e55..f6196e5d9 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,30 @@ pythonnet ========= -Python for .NET is a package that gives Python programmers nearly seamless integration with the .NET Common Language Runtime (CLR) and provides a powerful application scripting tool for .NET developers. +Python for .NET is a package that gives Python programmers nearly seamless integration with the .NET Common Language Runtime (CLR) and provides a powerful application scripting tool for .NET developers. It allows Python code to interact with the CLR, and may also be used to embed Python into a .NET application. -[![Build status](https://ci.appveyor.com/api/projects/status/u3p6pkiqpgu0qoku/branch/python3)](https://ci.appveyor.com/project/TonyRoberts/pythonnet/branch/python3) +[![Build Status](https://travis-ci.org/pythonnet/pythonnet.png?branch=master)](https://travis-ci.org/pythonnet/pythonnet) -[![Build Status](https://travis-ci.org/renshawbay/pythonnet.png?branch=python3)](https://travis-ci.org/renshawbay/pythonnet) +[![Build status](https://ci.appveyor.com/api/projects/status/c8k0miljb3n1c7be/branch/master)](https://ci.appveyor.com/project/TonyRoberts/pythonnet-480xs) -**Features not yet integrated into the main branch**: -- Python 3 support -- Subclassing managed types in Python +**Calling .NET code from Python** --------------------------------------------------------------------------------------------------------- +Python for .NET allows CLR namespaces to be treated essentially as Python packages. + +```python + import clr + from System import String + from System.Collections import * +``` +To load an assembly, use the "AddReference" function in the "clr" module: + +```python + import clr + clr.AddReference("System.Windows.Forms") + from System.Windows.Forms import Form +``` + +**Embedding Python in .NET** + All calls to python should be inside a "using (Py.GIL()) {/* Your code here */}" block. + Import python modules using dynamic mod = Py.Import("mod"), then you can call functions as normal, eg mod.func(args).